Compare commits

..
39 Commits
Author SHA1 Message Date
c4ch3c4d3 ce372568e4 feat: add StatsCache for shared per-model usage polling 2026-08-14 12:52:40 -06:00
c4ch3c4d3 aac7416cd6 feat: add usage stats parsing and fetching from /api/metrics/stats 2026-08-14 12:47:53 -06:00
bzuccaro 93d1006356 plan: fix stats re-registration ordering after config change 2026-08-14 12:47:17 -06:00
bzuccaro 38e6db3f4f Add Usage Stats + GPU Combinations implementation plan 2026-08-14 12:46:54 -06:00
bzuccaro 0cbd9f5274 spec: clarify All models datasource handling 2026-08-14 12:45:28 -06:00
bzuccaro 60ca227be3 Add Usage Stats + GPU Combinations design 2026-08-14 12:45:21 -06:00
bzuccaro 6b98218356 style: use a constant dark background for the In-Flight Monitor key 2026-08-14 12:11:07 -06:00
bzuccaro ec83216884 fix: render in-flight count as static white (remove blink pulse) 2026-08-14 12:09:57 -06:00
bzuccaro c6e8a9ad11 fix: track in-flight requests by id; handle llama-swap upsert/remove events
llama-swap emits inflight events with operation 'upsert' (a single request,
re-emitted as elapsed_ms updates) and id-only 'remove' events. The decoder
ignored 'upsert' and dropped every id-only remove, so the tracker only ever
reflected the connect-time snapshot and the count got stuck (e.g. at '1').

The tracker now keys in-flight state by request id (snapshot/add replace by
id; remove deletes by id), and the decoder normalizes 'upsert' to 'add' and
extracts the id from bare removes.
2026-08-14 12:08:02 -06:00
bzuccaro 6053713020 Show in-flight request count + activity spark on In-Flight Monitor key
The key now renders the live request count (large) when a model is active,
plus a 1Hz count-trend sparkline of the last 60s in a bottom strip. IDLE/
LOADING/OFF/OFFLINE states unchanged. History lives in per-key state with
a sampler cleared on disappear.
2026-08-14 11:20:51 -06:00
bzuccaro 38c4c7eccd Add In-Flight Monitor count + activity spark design 2026-08-14 11:19:20 -06:00
bzuccaro c535b80b1f docs: document multi-key support and shared connection model 2026-08-14 11:01:32 -06:00
bzuccaro ccbf74c613 fix: support multiple keys per action and harden runtime/parsing 2026-08-14 10:59:25 -06:00
bzuccaro 11d9187177 docs: add README; verify plugin install and packaging 2026-08-14 10:53:26 -06:00
bzuccaro 1836de9bbd feat: wire entry point, manifest, property inspectors, and assets 2026-08-14 10:52:13 -06:00
bzuccaro f8b46e1721 fix: show NO MODEL placeholder instead of OFFLINE when unconfigured 2026-08-14 10:51:02 -06:00
bzuccaro 959ef08ee0 docs: make NO MODEL placeholder reachable in inflight action 2026-08-14 10:50:50 -06:00
bzuccaro 0574689bf2 feat: implement inflight monitor and GPU graph actions 2026-08-14 10:49:50 -06:00
bzuccaro 5c7bb2ec4c docs: restore plan code fence balance after datasource note 2026-08-14 10:49:06 -06:00
bzuccaro b71459c663 fix: satisfy JsonObject constraints for settings types 2026-08-14 10:47:38 -06:00
bzuccaro 9ae5279764 docs: fix CfgSettings/DataSourceItem to satisfy JsonObject in plan 2026-08-14 10:47:20 -06:00
bzuccaro a6b4085e47 feat: add shared runtime and PI datasource handlers 2026-08-14 10:46:42 -06:00
bzuccaro 1cde3f3c67 feat: add SVG key renderers for inflight and GPU graph actions 2026-08-14 10:44:06 -06:00
bzuccaro 77fa1fad4f feat: add SSE event feed for llama-swap api/events 2026-08-14 10:43:03 -06:00
bzuccaro 5812ad641b feat: add SSE parser and in-flight request tracker 2026-08-14 10:42:17 -06:00
bzuccaro a5ef90f4a1 fix: import LlamaSwapConfig from util in metrics poller 2026-08-14 10:41:28 -06:00
bzuccaro ed2615b7c8 docs: import LlamaSwapConfig from util in poller plan 2026-08-14 10:41:17 -06:00
bzuccaro ab68fcb86f feat: add metrics poller with ring buffers and GPU aggregates 2026-08-14 10:41:04 -06:00
bzuccaro a411edde28 feat: add llama-swap HTTP client 2026-08-14 10:40:09 -06:00
bzuccaro c4082baebc feat: add Prometheus metrics parser for llama-swap GPU series 2026-08-14 10:39:26 -06:00
bzuccaro fa46857ccb feat: add shared util helpers and config 2026-08-14 10:38:42 -06:00
bzuccaro 6f65cc4527 docs: correct shorten truncation expectation to verified output 2026-08-14 10:38:35 -06:00
bzuccaro f352e07ee3 docs: fix shorten test to use a name within the 14-char limit 2026-08-14 10:37:20 -06:00
bzuccaro 1ba4bec8fb docs: require Stream Deck 7.1+ for Nodejs 24 runtime 2026-08-14 10:36:31 -06:00
bzuccaro d9684c20aa fix: use tsx test runner discovery and ignore build output 2026-08-14 10:34:31 -06:00
bzuccaro 1b388b7adc chore: scaffold llama-watch Stream Deck plugin 2026-08-14 10:33:48 -06:00
bzuccaro fc8f80c019 Fix plan inconsistencies found in pre-flight review 2026-08-14 10:32:10 -06:00
bzuccaro 73eed57626 Add llama-watch implementation plan 2026-08-14 10:30:46 -06:00
bzuccaro d371429624 Add llama-watch Stream Deck plugin design spec 2026-08-14 10:25:41 -06:00
36 changed files with 3699 additions and 850 deletions
-2
View File
@@ -2,5 +2,3 @@ node_modules/
*.streamDeckPlugin
.DS_Store
com.bryce.llamawatch.sdPlugin/bin/
store-assets/*.png
store-assets/.gen/
+3 -18
View File
@@ -4,15 +4,8 @@ A Stream Deck plugin (macOS) that monitors a llama-swap instance.
## Actions
- **In-Flight Monitor** — per-model request activity. The default view shows
the live in-flight request count with a 60-second activity spark (states:
`OFF` / `LOADING` / `IDLE` / count). Switch **Display** to **Usage stats**
to show request/token totals and the generation-speed P95 from
`/api/metrics/stats`, per model or across **All models**, with a pickable
primary stat. Powered by the real-time `/api/events` SSE feed.
- **GPU Graph** — live line chart of a GPU metric (utilization %, VRAM %,
temperature, power draw, fan speed) for one GPU, all GPUs, or an arbitrary
combination (e.g. GPU combination `0,2`), sampled every 5 s.
- **In-Flight Monitor** — per-model color state: `OFF` / `LOADING` / `IDLE` / `ACTIVE` (pulsing red). Powered by the real-time `/api/events` SSE feed.
- **GPU Graph** — live line chart of a GPU metric (utilization %, VRAM %, temperature, power draw, fan speed) for one GPU or all GPUs, sampled every 5 s.
Pressing either key opens `http://<base-url>/ui` in your browser.
@@ -35,15 +28,7 @@ npm test # unit tests (node:test + tsx)
## Configure
Per-key settings: base URL (default `http://localhost:9292`), optional API key, and the model / GPU / metric to watch. The model and GPU dropdowns are populated live from the instance. Each action instance is independent, so you can place several In-Flight Monitor keys (one per model, or in either display mode) and several GPU Graph keys (one per GPU × metric, or per GPU combination) on the same profile. All keys share a single connection to the configured llama-swap instance, which is re-established automatically if you change the base URL or API key on any key.
## Notes
- Usage totals come from llama-swap's `/api/metrics/stats` and cover its
in-memory activity retention (default ~1000 most recent requests), not
lifetime counters. The P95 is a **tokens/sec** generation-speed percentile.
- Keys are SVG-rendered (`render.ts`), so no image assets or canvas are
needed at runtime.
Per-key settings: base URL (default `http://talos.milky.way:9292`), optional API key, and the model / GPU / metric to watch. The model and GPU dropdowns are populated live from the instance. Each action instance is independent, so you can place several In-Flight Monitor keys (one per model) and several GPU Graph keys (one per GPU × metric) on the same profile. All keys share a single connection to the configured llama-swap instance, which is re-established automatically if you change the base URL or API key on any key.
## Marketplace
+1 -1
View File
@@ -2,7 +2,7 @@
"$schema": "https://schemas.elgato.com/streamdeck/plugins/manifest.json",
"Name": "llama-watch",
"Version": "1.0.0.0",
"Author": "c4ch3c4d3",
"Author": "Bryce Zuccaro",
"Actions": [
{
"Name": "In-Flight Monitor",
+1 -7
View File
@@ -6,20 +6,14 @@
</head>
<body>
<sdpi-item label="Base URL">
<sdpi-textfield setting="baseUrl" value="http://localhost:9292" placeholder="http://host:port" />
<sdpi-textfield setting="baseUrl" value="http://talos.milky.way:9292" placeholder="http://host:port" />
</sdpi-item>
<sdpi-item label="API Key">
<sdpi-password setting="apiKey" placeholder="Optional" />
</sdpi-item>
<sdpi-item>
<sdpi-checkbox setting="insecure" label="Ignore certificate errors (self-signed TLS)"></sdpi-checkbox>
</sdpi-item>
<sdpi-item label="GPU">
<sdpi-select setting="gpuId" datasource="gpus" loading="Loading GPUs…" hot-reload default="all" placeholder="Select a GPU" />
</sdpi-item>
<sdpi-item label="GPU combination">
<sdpi-textfield setting="gpuCombo" placeholder="e.g. 0,2 (overrides GPU)" />
</sdpi-item>
<sdpi-item label="Metric">
<sdpi-select setting="metric" default="util_percent" placeholder="Select a metric">
<option value="util_percent">Utilization %</option>
+1 -18
View File
@@ -6,28 +6,11 @@
</head>
<body>
<sdpi-item label="Base URL">
<sdpi-textfield setting="baseUrl" value="http://localhost:9292" placeholder="http://host:port" />
<sdpi-textfield setting="baseUrl" value="http://talos.milky.way:9292" placeholder="http://host:port" />
</sdpi-item>
<sdpi-item label="API Key">
<sdpi-password setting="apiKey" placeholder="Optional" />
</sdpi-item>
<sdpi-item>
<sdpi-checkbox setting="insecure" label="Ignore certificate errors (self-signed TLS)"></sdpi-checkbox>
</sdpi-item>
<sdpi-item label="Display">
<sdpi-select setting="display" default="count" placeholder="Select a display mode">
<option value="count">Request count + activity</option>
<option value="usage">Usage stats</option>
</sdpi-select>
</sdpi-item>
<sdpi-item label="Primary stat (usage view)">
<sdpi-select setting="primaryStat" default="requests" placeholder="Select a primary stat">
<option value="requests">Requests</option>
<option value="input_tokens">Processed tokens</option>
<option value="output_tokens">Generated tokens</option>
<option value="gen_p95">Generation speed P95</option>
</sdpi-select>
</sdpi-item>
<sdpi-item label="Model">
<sdpi-select setting="modelId" datasource="models" loading="Loading models…" hot-reload placeholder="Select a model" />
</sdpi-item>
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,61 @@
# In-Flight Monitor — Count + Activity Spark
**Date:** 2026-08-14
**Status:** Approved (design)
## Summary
Enhance the In-Flight Monitor action so its key shows the **number of
in-flight requests** for the configured model plus a **count-trend spark**:
a tiny line/area chart of that count over the last ~60s, so activity is
visible even during a single long-running request.
## Layout (72×72 SVG, rendered via `renderInflight`)
- **Top:** model short name (small, white) — existing.
- **Center:** the status. When `ready`:
- count > 0 → the **count number**, large/bold (e.g. `3`), on the red
`ACTIVE` background, with the existing pulse.
- count === 0 → `IDLE` on green — existing.
- `loading``LOADING`, `stopped``OFF`, offline → `OFFLINE`,
unconfigured → `NO MODEL`, unknown → `…` — all unchanged.
- **Bottom strip (~14px):** the count-trend spark — a small polyline + faint
area fill of the model's in-flight count over the last 60 samples, in the
state color. Flat at the baseline when idle; spikes when requests come and go.
## Data Flow
- The feed side is unchanged — `InflightTracker` already exposes exact
per-model counts.
- Each key's per-key state (in `inflight-monitor.ts`, keyed by
`ev.action.id`) gains:
- `history: number[]` — a 60-sample ring buffer of the live count.
- `sampler?: ReturnType<typeof setInterval>` — a 1s interval that pushes
`runtime.tracker.count(modelId)` into `history` and re-renders.
Started in `onWillAppear`, cleared in `onWillDisappear`.
- Live feed events (`add`/`remove`/`snapshot`) still re-render the big
number immediately via the existing subscription; the spark refreshes at
1Hz.
- `renderInflight` gains an optional `history?: number[]` field. Spark
scale: yMax = `max(max(history), 1)` so a zero line sits at the baseline.
## Files
- `src/lib/render.ts``renderInflight` accepts `history` and draws the
spark; center label shows the count when active.
- `src/actions/inflight-monitor.ts` — per-key `history` ring buffer +
1s sampler.
- `tests/render.test.ts` — new/updated assertions for count rendering and
the spark.
## Error Handling / Edge Cases
- Spark with <2 samples renders no line (like the GPU chart).
- Sampler cleared on `onWillDisappear` (no leaks); multi-key safety
preserved (buffer + sampler live in the per-key map entry).
- History capped at 60 samples.
## Out of Scope
- No change to the GPU Graph action, the feed, or the tracker.
- No throughput-bar variant (count trend was chosen).
@@ -0,0 +1,166 @@
# llama-watch — Stream Deck Plugin Design
**Date:** 2026-08-14
**Status:** Approved (pending spec review)
## Summary
A Stream Deck plugin ("llama-watch") that monitors a llama-swap instance
(`http://talos.milky.way:9292`). Two key-action types:
1. **In-flight Monitor** — shows whether a specific model currently has a
request in flight, with a color-coded state (OFF / LOADING / IDLE / ACTIVE).
2. **GPU Graph** — renders a live line chart of a selected GPU metric
(utilization %, VRAM %, temperature, power draw, or fan speed) on the key,
for a specific GPU or the "All GPUs" aggregate.
Target device: standard Stream Deck (72x72 px keys), rendered at 144x144 for
crispness. macOS only. Local install, with eventual Elgato Marketplace
submission as a design goal.
## Approach
Official Elgato JS SDK (`streamdeck-jssdk`) + `streamdeck-cli` tooling, a
TypeScript Node.js plugin, and `@napi-rs/canvas` for image rendering. The
plugin runs locally (launched by the Stream Deck app) and talks directly to
the llama-swap instance.
## Data Sources
llama-swap exposes everything needed (verified live against
`talos.milky.way:9292`):
- **`GET /api/events`** — SSE stream used by the web UI. Emits `inflight`
events (operations `snapshot` / `add` / `remove`, per-request detail:
model, timestamp, elapsed, bytes) and `modelStatus` events (per-model
state: `stopped`, `loading`, `ready`). Also emits `logData`, `uiConfig`,
`profileChanged`, `activity` events — ignored.
- **`GET /metrics`** — Prometheus text format. Relevant series:
- `llamaswap_gpu_util_percent{id,name,uuid}`
- `llamaswap_gpu_memory_util_percent{id,name,uuid}`
- `llamaswap_gpu_temperature_celsius{id,name,uuid}`
- `llamaswap_gpu_power_draw_watts{id,name,uuid}`
- `llamaswap_gpu_fan_speed_percent{id,name,uuid}`
- **`GET /v1/models`** — model list (with `status.value` loaded/unloaded),
used to populate the model dropdown in the property inspector.
Upstream vLLM metrics (ports 10001/10003) are bound to localhost on the
server and are NOT reachable from the client Mac. Not used.
## Architecture
The plugin is a single Node process with three connections:
1. WebSocket to the Stream Deck software (via `streamdeck-jssdk`).
2. Persistent SSE connection to `/api/events` (the `EventFeed`).
3. 5s-interval polling of `/metrics` (the `MetricsPoller`).
### Modules
- **`MetricsPoller`** (pure logic, testable)
- Polls `GET /metrics` every **5 seconds** (fixed).
- Parses the Prometheus text format into a lookup keyed by metric name ×
GPU id.
- `getSnapshot(gpuSelector, metric)` resolves any selector × metric
combination into a number, including the "All GPUs" aggregate.
- Keeps a ring buffer of 60 samples (~5 minutes) per (gpu, metric).
- A missed/failed poll skips that sample (no gap in rendering beyond a
break in the line).
- **`EventFeed`** (pure logic, testable)
- Persistent SSE connection to `/api/events`.
- Reconnect with exponential backoff (1s → 30s max); the `inflight`
`snapshot` operation emitted on connect self-heals the tracker.
- Maintains per-model in-flight request counts and per-model state from
`modelStatus`.
- **Actions**
- `InflightMonitor` — one instance per monitored model.
- `GpuGraph` — one instance per GPU (or All GPUs) × metric.
- **Property inspectors** (`pi/*.html`) — settings UI for both actions.
- **Renderer** — draws 144x144 PNGs via `@napi-rs/canvas`.
### Data Flow
```
llama-swap ──SSE /api/events──▶ EventFeed ──▶ per-model inflight counts + state
──HTTP /metrics 5s──▶ MetricsPoller ──▶ ring buffers + aggregates
Stream Deck app ◀──jssdk WS── Actions ────────┘
◀──144x144 PNG + title── Renderer
```
### Settings (shared + per action)
- **Shared:** base URL (default `http://talos.milky.way:9292`), optional API
key (sent as a header when set; instance currently requires none).
- **InflightMonitor:** model (dropdown populated live from `/v1/models`).
- **GpuGraph:** GPU selector (dropdown from live GPU list, plus "All GPUs"),
metric (dropdown: Utilization %, VRAM %, Temperature, Power draw, Fan
speed).
## Action 1 — In-flight Monitor
- **States (rendered on key):**
- `stopped` / unloaded → grey background, model name, `OFF`
- `loading` → amber, `LOADING`
- `ready` + 0 requests → green, `IDLE`
- `ready` + ≥1 request → red, `ACTIVE` (subtle pulse, re-render ~2 Hz while
active)
- Model short name displayed on the key; if no model selected, placeholder
with an edit hint.
- **Press:** opens `http://<base-url>/ui` in the default browser (spawns
`open` on macOS).
## Action 2 — GPU Graph
- **Rendering (graph dominates the 72x72 key):**
- Big line chart of the last 60 samples (5 min @ 5s) filling the key.
- Dark background, bright line, subtle filled gradient under the curve.
- Small current-value label (e.g. `67%`, `336W`, `55°C`) at top; metric
name (e.g. `UTIL`, `PWR`) at bottom.
- **Scale:** % metrics fixed 0100; temperature auto 0100°C; power
auto-scaled to observed max.
- **Colors:** severity-based — green → amber → red. Temperature red at
≥80°C; util/power/fan scale with level.
- **Aggregates ("All GPUs"):** Utilization/VRAM/Fan → average; Temperature →
max; Power → sum.
- **Render-on-change:** idle/flat history does not re-render every poll.
- **Press:** opens the web UI (same as Action 1).
## Error Handling
- llama-swap unreachable → dark grey key with `!!` and `OFFLINE`; retry with
backoff; auto-recover when server returns.
- SSE drop → reconnect with backoff (1s → 30s); self-healing via `snapshot`.
- `/metrics` poll timeout → skip sample.
- Per-button error states — a bad setting on one key does not affect others.
## Packaging & Assets
- Package UUID: `com.bryce.llamawatch` (reverse-DNS, unique).
- `streamdeck-cli` scaffolds project, builds TypeScript, packages
`.streamDeckPlugin` (zip) → installs via double-click into
`~/Library/Application Support/com.elgato.StreamDeck/Plugins/`.
- Marketplace-ready from the start:
- 256×256 plugin icon + action icons (incl. pressed states) at marketplace
spec sizes.
- Name "llama-watch", category e.g. "System & Monitoring".
- Privacy note: reads GPU metrics + request status from the user's own
llama-swap server; only base URL + optional API key stored in Stream
Deck's local settings; no data leaves the machine.
- No hardcoded secrets/URLs — base URL is user-editable.
- Manifest declares macOS only initially.
## Testing
- Unit tests with `node:test` for the Prometheus parser, inflight tracker,
and aggregate math (avg/max/sum), using **real captured fixtures** from
the talos instance (metrics body, inflight snapshot, modelStatus payloads).
- Manual verification: both actions render correctly on the physical Stream
Deck; key press opens the web UI.
## Out of Scope (YAGNI)
- Marketplace submission itself (goal for later; assets/metadata prepared now).
- Auto-update plumbing.
- Windows/Linux support (later add-on; macOS declared in manifest).
- Additional metrics or display styles.
@@ -0,0 +1,131 @@
# Usage Stats + GPU Combinations
**Date:** 2026-08-14
**Status:** Approved (design)
## Summary
Two extensions to the existing llama-watch actions:
1. The In-Flight Monitor action gains a **Usage stats** display mode showing
token/request totals and the generation-speed P95 from llama-swap's
`/api/metrics/stats` endpoint.
2. The GPU Graph action gains **GPU combinations**: a key can aggregate an
arbitrary subset of GPUs (e.g. only the two Blackwells).
## Data Source (verified against llama-swap source)
- `GET /api/metrics/stats` (no query param) → global aggregates.
- `GET /api/metrics/stats?model=<id>` → per-model aggregates.
- Response fields used: `total_requests`, `total_input_tokens`,
`total_output_tokens`, `gen_histogram.p95`.
- `p95` is a **tokens/sec generation-speed percentile**, not latency.
- Totals cover llama-swap's in-memory activity retention (default ~1000
requests), not lifetime counters.
- The `/api/events` SSE `activity` event (payload `{"id": N}`) fires per
completed request and is used as a "re-poll stats" trigger. It carries no
data itself.
- The Prometheus `/metrics` endpoint exposes no token/request/latency data
and is not used for this feature.
## 1. Usage stats data layer
- **`src/lib/stats.ts`** (new):
- `interface UsageStats { totalRequests: number; totalInputTokens: number; totalOutputTokens: number; genP95: number }`.
- `parseStats(json: unknown): UsageStats | null` — defensive; returns
`null` on malformed input; missing fields default to `0`.
- `fetchStats(cfg: LlamaSwapConfig, modelId: string): Promise<UsageStats | null>`
fetches `/api/metrics/stats` (with `?model=` when `modelId` is not
`"all"`), returns parsed stats or `null` on HTTP error/parse failure.
- **`src/lib/stats-cache.ts`** (new):
- `StatsCache` holds `Set<string>` of registered model keys (`"all"` or a
model id) and a `Map<key, UsageStats | null>` of last-known values.
- One shared 5s interval polls every registered key via `fetchStats`.
- `register(key)` / `unregister(key)` start/stop the interval as needed
(no ref counting — each key registers/unregisters once).
- `get(key): UsageStats | undefined`.
- `refresh()` re-polls all registered keys.
- Change listener invoked after each poll completes (success or failure).
- Fetch failure keeps the last-known value.
- **`src/lib/event-feed.ts`**: `decodeEvent` learns the `activity` event —
`{ type: "activity", id: number }`. `FeedEvent` (in `inflight-tracker.ts`)
gains the `activity` variant.
- **`src/lib/runtime.ts`**: runtime owns a `StatsCache` (created alongside
feed/poller, reset on config change). Feed `activity` events trigger
`statsCache.refresh()` throttled to at most once per 2s. Runtime exposes:
- `getStats(modelId: string): UsageStats | undefined`
- `watchStats(modelId: string): () => void` (register; returns
unsubscribe).
- Stats changes flow through the existing `subscribe`/`emit` mechanism so
actions re-render on the normal tick.
## 2. In-Flight action modes
- New settings (per key, in `inflight-monitor.ts`):
- `display: "count" | "usage"` (default `count`).
- `primaryStat: "requests" | "input_tokens" | "output_tokens" | "gen_p95"`
(default `requests`).
- Model setting semantics:
- Model select (PI datasource) gains an **"All models"** option, value
`all`.
- Count mode: `all` → total in-flight across all models; label
`ALL MODELS`.
- Usage mode: `all` → global stats; otherwise per-model stats.
- **Count mode**: unchanged rendering (count number, spark, `IDLE`/
`LOADING`/`OFF`/`OFFLINE`). The 1s spark sampler runs only in count mode.
- **Usage mode** (`renderUsage` in `render.ts`, dark `#10131a` bg):
- top: model short name or `ALL MODELS` (7px).
- center (24px bold): the `primaryStat` value, compact-formatted —
`< 1000` integer, else `x.xk`, else `x.xM` (`genP95` shows integer t/s).
- three small rows (7px): the remaining three stats, labeled `REQ`,
`IN`, `OUT`, `P95`.
- offline → `OFFLINE` overlay; stats unknown → `--`.
- Press opens the UI (unchanged).
## 3. GPU combinations
- New setting `gpuCombo?: string` — comma-separated GPU ids, e.g. `"0,2"`.
When non-empty it overrides `gpuId`.
- Pure helper in `metrics-poller.ts` (exported, tested):
`combineSeries(histories: number[][], kind): { value?: number; history: number[] }`
— element-wise across the per-GPU rings, aligned by index, applying the
existing aggregate rules: `util_percent`/`memory_util_percent`/`fan`
average, `temperature` → max, `power` → sum. A trailing `undefined` value
from an empty ring leaves that element undefined.
- `gpu-graph.ts` render: for a combo, build per-GPU histories via
`poller.getHistory(id, metric)`, combine, use the combined value/history.
Unknown ids are filtered out; if none remain → `--`.
- Label: `GPU 0+2` (ids joined with `+`).
## 4. Property inspectors
- `ui/inflight.html`: add "Display" select (`count`/`usage`) and "Primary
stat" select. Both always visible; `primaryStat` has no effect in count
mode.
- `src/lib/datasources.ts`: the `models` datasource prepends
`{ label: "All models", value: "all" }` to its items (so the model select
offers both the aggregate and individual models).
- `ui/gpu.html`: add "GPU combination (comma-separated ids)" textfield,
optional, placeholder `e.g. 0,2`.
## Error Handling / Edge Cases
- Stats fetch error → keep last-known values; no `OFFLINE` flip (offline is
driven by feed status as today).
- Server offline → existing `OFFLINE` overlay in both modes.
- Stats not yet loaded → `--` placeholders.
- Activity-triggered refresh throttled to 1 per 2s.
- Combo with no valid ids → `--`.
- No manifest change (both features extend existing actions).
## Testing
- `parseStats`: happy path; malformed input; missing fields → `0`.
- `StatsCache`: register/unregister lifecycle (interval start/stop),
single poll per model, values cached, refresh on demand, failure keeps
last value. Uses an injected fetch function.
- `decodeEvent`: `activity` event decodes to `{ type: "activity", id }`.
- `renderUsage`: big primary stat, small labeled rows, compact formatting
(`52.1k`), `--` when no data, `ALL MODELS` label, offline overlay.
- `combineSeries`: avg/max/sum per metric; empty/mismatched histories.
- In-Flight action: `all` count sums models.
+1 -11
View File
@@ -8,8 +8,7 @@
"name": "llama-watch",
"version": "1.0.0",
"dependencies": {
"@elgato/streamdeck": "^2.1.1",
"undici": "^6.28.0"
"@elgato/streamdeck": "^2.1.1"
},
"devDependencies": {
"@elgato/cli": "^1.8.1",
@@ -2478,15 +2477,6 @@
"node": ">=14.17"
}
},
"node_modules/undici": {
"version": "6.28.0",
"resolved": "https://registry.npmjs.org/undici/-/undici-6.28.0.tgz",
"integrity": "sha512-LIY910g9TI13YS95lrMFrs8Rm/u/irgHeTWoKCoteeJ04CUJ92eEfj0rVn+7VKMPBpUPiUoBKfhNyLI23EE/KA==",
"license": "MIT",
"engines": {
"node": ">=18.17"
}
},
"node_modules/undici-types": {
"version": "6.20.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz",
+1 -2
View File
@@ -23,7 +23,6 @@
"typescript": "^5.7.3"
},
"dependencies": {
"@elgato/streamdeck": "^2.1.1",
"undici": "^6.28.0"
"@elgato/streamdeck": "^2.1.1"
}
}
-188
View File
@@ -1,188 +0,0 @@
import { mkdirSync, rmSync, writeFileSync } from "node:fs";
import { execFileSync } from "node:child_process";
import { fileURLToPath } from "node:url";
import path from "node:path";
import { renderGpuGraph, renderInflight, renderUsage } from "../src/lib/render";
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const OUT = path.join(ROOT, "store-assets");
const TMP = path.join(OUT, ".gen");
const FONT = "Helvetica, Arial, sans-serif";
const BG = "#0e1117";
const PANEL = "#161b26";
const STROKE = "#242b38";
const MUTED = "#8b93a5";
const WHITE = "#e6e9ef";
const GREEN = "#3fae5a";
const RED = "#e0453a";
const AMBER = "#d9a02a";
rmSync(TMP, { recursive: true, force: true });
mkdirSync(TMP, { recursive: true });
function convert(src: string, dst: string, w: number, h: number): void {
execFileSync("rsvg-convert", ["--unlimited", "-w", String(w), "-h", String(h), "-o", dst, src]);
}
function rasterizeKey(name: string, svg: string, scale = 3): string {
const svgPath = path.join(TMP, `${name}.svg`);
const pngPath = path.join(TMP, `${name}.png`);
writeFileSync(svgPath, svg);
convert(svgPath, pngPath, 72 * scale, 72 * scale);
return `${name}.png`;
}
function svg(w: number, h: number, body: string): string {
return `<svg xmlns="http://www.w3.org/2000/svg" width="${w}" height="${h}" viewBox="0 0 ${w} ${h}">
<rect width="${w}" height="${h}" fill="${BG}"/>
${body}
</svg>`;
}
function text(content: string, x: number, y: number, size: number, fill: string, weight = "normal", anchor = "middle"): string {
return ` <text x="${x}" y="${y}" text-anchor="${anchor}" font-family="${FONT}" font-size="${size}" font-weight="${weight}" fill="${fill}">${content}</text>`;
}
function keyImage(png: string, cx: number, cy: number, size: number): string {
return ` <image href="${png}" x="${cx - size / 2}" y="${cy - size / 2}" width="${size}" height="${size}"/>`;
}
function keyWithLabel(png: string, label: string, cx: number, cy: number, size: number): string {
return `${keyImage(png, cx, cy, size)}
<rect x="${cx - size / 2 - 6}" y="${cy + size / 2 + 14}" width="${size + 12}" height="1" fill="${STROKE}"/>
${text(label, cx, cy + size / 2 + 40, 26, WHITE)}`;
}
function header(title: string, subtitle: string): string {
return `${text(title, 960, 108, 56, WHITE, "bold")}
${text(subtitle, 960, 164, 28, MUTED)}`;
}
function footer(caption: string): string {
return `${text(caption, 960, 872, 24, MUTED)}`;
}
const keys: Record<string, string> = {
inflightIdle: renderInflight({ modelName: "DeepSeek-V4-Flash-0731", state: "ready", count: 0, offline: false }),
inflightActive: renderInflight({ modelName: "DeepSeek-V4-Flash-0731", state: "ready", count: 3, offline: false, history: [0, 1, 2, 3, 2, 3, 4, 3, 3, 2, 3, 2, 1] }),
usageP95: renderUsage({ modelName: "DeepSeek-V4-Flash-0731", stats: { totalRequests: 1950, totalInputTokens: 312177540, totalOutputTokens: 889193, genP95: 378.86 }, primaryStat: "gen_p95", offline: false }),
usageReq: renderUsage({ modelName: "All Models", stats: { totalRequests: 1985, totalInputTokens: 312327705, totalOutputTokens: 932710, genP95: 378.86 }, primaryStat: "requests", offline: false }),
gpuUtil: renderGpuGraph({ gpuName: "GPU 0 · RTX 5090", metric: "util_percent", value: 67, history: [10, 20, 40, 67, 55, 80, 72, 45, 60], offline: false }),
gpuPower: renderGpuGraph({ gpuName: "GPU 1 · RTX 5090", metric: "power", value: 312, history: [100, 150, 200, 312, 280, 350, 220, 190, 260], offline: false }),
gpuAll: renderGpuGraph({ gpuName: "ALL GPUS", metric: "temperature", value: 63, history: [55, 58, 61, 63, 60, 65, 64, 62, 66], offline: false }),
gpuCombo: renderGpuGraph({ gpuName: "GPU 0+2", metric: "util_percent", value: 71, history: [30, 45, 52, 71, 64, 78, 70, 58, 66], offline: false }),
};
const pngs: Record<string, string> = {};
for (const [name, svg] of Object.entries(keys)) {
pngs[name] = rasterizeKey(name, svg);
}
function writePng(name: string, w: number, h: number, body: string): void {
const svgPath = path.join(TMP, `${name}.svg`);
writeFileSync(svgPath, svg(w, h, body));
convert(svgPath, path.join(OUT, name), w, h);
console.log(`wrote store-assets/${name} (${w}x${h})`);
}
// Thumbnail — 1920x960
{
const s = 240;
const cx = [1250, 1570];
const cy = [360, 700];
const cols = [
keyWithLabel(pngs.inflightActive, "In-Flight Monitor", cx[0], cy[0], s),
keyWithLabel(pngs.usageReq, "Usage stats", cx[1], cy[0], s),
keyWithLabel(pngs.gpuUtil, "GPU Graph", cx[0], cy[1], s),
keyWithLabel(pngs.gpuCombo, "GPU combination", cx[1], cy[1], s),
];
const bullets = [
["#3fae5a", "In-flight request counts with a 60-second activity spark"],
["#d9a02a", "Usage stats: requests, tokens, and generation-speed P95"],
["#e0453a", "Live GPU graphs — per GPU, all GPUs, or a combination"],
]
.map(
([color, t], i) =>
` <circle cx="116" cy="${480 + i * 78}" r="10" fill="${color}"/>\n` +
text(t, 146, 488 + i * 78, 28, WHITE, "normal", "start"),
)
.join("\n");
writePng(
"thumbnail.png",
1920,
960,
`${text("llama-watch", 116, 250, 96, WHITE, "bold", "start")}
${text("Stream Deck keys for your llama-swap LLM server", 116, 322, 30, MUTED, "normal", "start")}
<rect x="116" y="350" width="620" height="3" fill="${GREEN}"/>
${bullets}
${cols.join("\n")}`,
);
}
// Gallery 1 — In-Flight Monitor
{
const s = 240;
const centers = [470, 810, 1150, 1490];
const items = [
[pngs.inflightIdle, "IDLE — no requests"],
[pngs.inflightActive, "3 in flight + 60s spark"],
[pngs.usageP95, "Usage · gen speed P95"],
[pngs.usageReq, "Usage · totals"],
];
writePng(
"gallery-1-inflight.png",
1920,
960,
`${header("In-Flight Monitor", "Live request counts, a 60-second activity spark, and usage stats")}
${items.map(([png, label], i) => keyWithLabel(png as string, label as string, centers[i], 460, s)).join("\n")}
${footer("Counts update instantly from the SSE feed; the spark shows the last 60 seconds of activity.")}`,
);
}
// Gallery 2 — GPU Graph
{
const s = 240;
const centers = [470, 810, 1150, 1490];
const items = [
[pngs.gpuUtil, "GPU 0 · utilization"],
[pngs.gpuPower, "GPU 1 · power draw"],
[pngs.gpuAll, "All GPUs · temperature"],
[pngs.gpuCombo, "GPU 0+2 · utilization combo"],
];
writePng(
"gallery-2-gpu.png",
1920,
960,
`${header("GPU Graph", "Live metric charts — one GPU, all GPUs, or a custom combination")}
${items.map(([png, label], i) => keyWithLabel(png as string, label as string, centers[i], 460, s)).join("\n")}
${footer("Sampled every 5 seconds: utilization, VRAM, temperature, power draw, and fan speed.")}`,
);
}
// Gallery 3 — Setup
{
const field = (label: string, value: string, y: number): string =>
`${text(label, 200, y, 22, MUTED, "normal", "start")}
<rect x="200" y="${y + 22}" width="700" height="44" rx="10" fill="${BG}" stroke="${STROKE}" stroke-width="2"/>
${text(value, 220, y + 22 + 29, 22, WHITE, "normal", "start")}`;
const panel = `${text("In-Flight Monitor settings", 200, 240, 30, WHITE, "bold", "start")}
${field("Base URL", "http://localhost:9292", 300)}
${field("API Key", "Optional", 395)}
${field("Model", "DeepSeek-V4-Flash-0731 (ready)", 490)}
${field("Display", "Usage stats", 585)}
${field("Primary stat", "Generation speed P95", 680)}
<rect x="160" y="170" width="760" height="640" rx="24" fill="${PANEL}" stroke="${STROKE}" stroke-width="3"/>`;
writePng(
"gallery-3-setup.png",
1920,
960,
`${header("Point it at your llama-swap instance", "Per-key settings in the property inspector")}
<g transform="translate(0,0)">${panel}</g>
${keyWithLabel(pngs.usageReq, "Usage stats key", 1420, 430, 300)}
${footer("Every key is independent: model, display mode, GPU, metric, and combination.")}`,
);
}
// App icon — 288x288
convert(path.join(ROOT, "com.bryce.llamawatch.sdPlugin/imgs/plugin/icon.svg"), path.join(OUT, "app-icon-288.png"), 288, 288);
console.log("wrote store-assets/app-icon-288.png (288x288)");
+4 -27
View File
@@ -8,7 +8,6 @@ import streamDeck, {
type WillDisappearEvent,
} from "@elgato/streamdeck";
import { type GpuMetricKind } from "../lib/metrics-parser";
import { combineSeries } from "../lib/metrics-poller";
import { renderGpuGraph, svgDataUrl } from "../lib/render";
import { runtime } from "../lib/runtime";
import { cfgFromSettings, type CfgSettings } from "../lib/util";
@@ -16,7 +15,6 @@ import { cfgFromSettings, type CfgSettings } from "../lib/util";
type GpuSettings = CfgSettings & {
gpuId?: string;
metric?: GpuMetricKind;
gpuCombo?: string;
};
type GpuState = {
@@ -73,28 +71,12 @@ export class GpuGraph extends SingletonAction<GpuSettings> {
const action = state.action;
if (!action) return;
const poller = runtime.pollerInstance;
const gpuId = state.settings.gpuId ?? "all";
const metric = state.settings.metric ?? "util_percent";
const combo = parseCombo(state.settings.gpuCombo);
const value = poller?.getValue(gpuId, metric);
const history = poller?.getHistory(gpuId, metric) ?? [];
const offline = !poller || poller.isOffline();
let value: number | undefined;
let history: number[] = [];
let gpuName: string;
if (combo) {
const known = new Set((poller?.gpus() ?? []).map((g) => g.id));
const ids = combo.filter((id) => known.has(id));
const histories = ids.map((id) => poller?.getHistory(id, metric) ?? []);
const combined = combineSeries(histories, metric);
value = combined.value;
history = combined.history;
gpuName = ids.length > 0 ? `GPU ${ids.join("+")}` : "NO GPUS";
} else {
const gpuId = state.settings.gpuId ?? "all";
value = poller?.getValue(gpuId, metric);
history = poller?.getHistory(gpuId, metric) ?? [];
gpuName = gpuId === "all" ? "ALL GPUS" : this.gpuLabel(gpuId);
}
const gpuName = gpuId === "all" ? "ALL GPUS" : this.gpuLabel(gpuId);
const svg = renderGpuGraph({ gpuName, metric, value, history, offline });
if (svg === state.lastSvg) return;
state.lastSvg = svg;
@@ -106,8 +88,3 @@ export class GpuGraph extends SingletonAction<GpuSettings> {
return gpu && gpu.name ? `GPU ${gpuId} · ${gpu.name}` : `GPU ${gpuId}`;
}
}
function parseCombo(raw: string | undefined): string[] | undefined {
const ids = [...new Set((raw ?? "").split(",").map((s) => s.trim()).filter(Boolean))];
return ids.length > 0 ? ids : undefined;
}
+13 -58
View File
@@ -7,24 +7,20 @@ import streamDeck, {
type WillAppearEvent,
type WillDisappearEvent,
} from "@elgato/streamdeck";
import { renderInflight, renderUsage, svgDataUrl } from "../lib/render";
import { renderInflight, svgDataUrl } from "../lib/render";
import { runtime } from "../lib/runtime";
import { cfgFromSettings, type CfgSettings } from "../lib/util";
type InflightSettings = CfgSettings & {
modelId?: string;
display?: "count" | "usage";
primaryStat?: "requests" | "input_tokens" | "output_tokens" | "gen_p95";
};
type InflightState = {
settings: InflightSettings;
action?: KeyAction<InflightSettings>;
unsubscribe?: () => void;
unwatchStats?: () => void;
history: number[];
sampler?: ReturnType<typeof setInterval>;
lastSvg?: string;
};
const HISTORY_LIMIT = 60;
@@ -49,11 +45,7 @@ export class InflightMonitor extends SingletonAction<InflightSettings> {
state.action = ev.action;
runtime.ensureConnections(cfgFromSettings(state.settings));
state.unsubscribe = runtime.subscribe(() => this.render(state));
if (this.displayOf(state) === "usage") {
state.unwatchStats = runtime.watchStats(this.modelKeyOf(state));
} else {
state.sampler = setInterval(() => this.sample(state), 1000);
}
state.sampler = setInterval(() => this.sample(state), 1000);
this.render(state);
}
@@ -61,8 +53,6 @@ export class InflightMonitor extends SingletonAction<InflightSettings> {
const state = this.states.get(ev.action.id);
if (!state) return;
state.unsubscribe?.();
state.unwatchStats?.();
state.unwatchStats = undefined;
if (state.sampler) clearInterval(state.sampler);
state.sampler = undefined;
this.states.delete(ev.action.id);
@@ -75,15 +65,6 @@ export class InflightMonitor extends SingletonAction<InflightSettings> {
state.action = ev.action;
state.history = [];
runtime.ensureConnections(cfgFromSettings(state.settings));
state.unwatchStats?.();
state.unwatchStats = undefined;
if (state.sampler) clearInterval(state.sampler);
state.sampler = undefined;
if (this.displayOf(state) === "usage") {
state.unwatchStats = runtime.watchStats(this.modelKeyOf(state));
} else {
state.sampler = setInterval(() => this.sample(state), 1000);
}
this.render(state);
}
@@ -91,55 +72,29 @@ export class InflightMonitor extends SingletonAction<InflightSettings> {
void streamDeck.system.openUrl(`${cfgFromSettings(ev.payload.settings).baseUrl}/ui`);
}
private displayOf(state: InflightState): "count" | "usage" {
return state.settings.display ?? "count";
}
private modelKeyOf(state: InflightState): string {
const modelId = state.settings.modelId ?? "";
return modelId === "" ? "all" : modelId;
}
private primaryStatOf(state: InflightState): "requests" | "input_tokens" | "output_tokens" | "gen_p95" {
return state.settings.primaryStat ?? "requests";
}
private render(state: InflightState): void {
const action = state.action;
if (!action) return;
let svg: string;
if (this.displayOf(state) === "usage") {
const modelKey = this.modelKeyOf(state);
svg = svgDataUrl(
renderUsage({
modelName: modelKey,
stats: runtime.getStats(modelKey),
primaryStat: this.primaryStatOf(state),
offline: runtime.offline,
}),
);
} else {
const modelId = state.settings.modelId ?? "";
const trackerState = modelId === "all" ? "ready" : runtime.tracker.state(modelId);
const count = modelId === "all" ? runtime.tracker.total() : runtime.tracker.count(modelId);
svg = svgDataUrl(
const modelId = state.settings.modelId ?? "";
const trackerState = runtime.tracker.state(modelId);
const count = runtime.tracker.count(modelId);
const offline = runtime.offline && modelId.length > 0;
void action.setImage(
svgDataUrl(
renderInflight({
modelName: modelId === "" ? "unset" : modelId,
modelName: modelId.length > 0 ? modelId : "unset",
state: trackerState,
count,
offline: runtime.offline && modelId.length > 0,
offline,
history: state.history,
}),
);
}
if (svg === state.lastSvg) return;
state.lastSvg = svg;
void action.setImage(svg);
),
);
}
private sample(state: InflightState): void {
const modelId = state.settings.modelId ?? "";
state.history.push(modelId === "all" ? runtime.tracker.total() : runtime.tracker.count(modelId));
state.history.push(runtime.tracker.count(modelId));
if (state.history.length > HISTORY_LIMIT) state.history.shift();
this.render(state);
}
+1 -1
View File
@@ -17,7 +17,7 @@ export function registerDataSources(): void {
let items: DataSourceItem[] = [];
try {
const models = await fetchModels(cfgFromSettings(settings));
items = [{ label: "All models", value: "all" }, ...models.map((m) => ({ label: `${m.id} (${m.status})`, value: m.id }))];
items = models.map((m) => ({ label: `${m.id} (${m.status})`, value: m.id }));
} catch {
items = [];
}
+2 -9
View File
@@ -1,4 +1,3 @@
import { fetchWith } from "./http";
import { type LlamaSwapConfig } from "./util";
import { parseSse, type SseMessage } from "./sse";
import { type FeedEvent, type InflightRequest, type ModelState } from "./inflight-tracker";
@@ -9,16 +8,10 @@ export function decodeEvent(msg: SseMessage): FeedEvent | null {
if (!outer.data) return null;
const inner = JSON.parse(outer.data) as Record<string, unknown>;
if (outer.type === "activity") {
const id = typeof inner.id === "number" ? inner.id : Number.NaN;
if (Number.isFinite(id)) return { type: "activity", id };
return null;
}
if (outer.type === "inflight") {
const operation = inner.operation as string;
if (operation === "remove") {
const id = typeof inner.id === "string" ? inner.id : typeof inner.id === "number" ? String(inner.id) : undefined;
const id = typeof inner.id === "string" ? inner.id : undefined;
if (id) return { type: "inflight", operation: "remove", id };
return null;
}
@@ -81,7 +74,7 @@ export class EventFeed {
try {
const headers: Record<string, string> = { Accept: "text/event-stream" };
if (this.cfg.apiKey) headers["Authorization"] = `Bearer ${this.cfg.apiKey}`;
const res = await fetchWith(this.cfg, `${this.cfg.baseUrl}/api/events`, {
const res = await fetch(`${this.cfg.baseUrl}/api/events`, {
headers,
signal: this.controller.signal,
});
-9
View File
@@ -1,9 +0,0 @@
import { Agent, fetch as undiciFetch } from "undici";
import { type LlamaSwapConfig } from "./util";
const INSECURE_AGENT = new Agent({ connect: { rejectUnauthorized: false } });
export function fetchWith(cfg: LlamaSwapConfig, input: string, init?: RequestInit): Promise<Response> {
if (cfg.insecure !== true) return fetch(input, init);
return undiciFetch(input, { ...(init ?? {}), dispatcher: INSECURE_AGENT }) as unknown as Promise<Response>;
}
+5 -18
View File
@@ -15,48 +15,35 @@ export interface ModelState {
export type FeedEvent =
| { type: "inflight"; operation: "snapshot" | "add"; requests: InflightRequest[] }
| { type: "inflight"; operation: "remove"; id: string }
| { type: "activity"; id: number }
| { type: "modelStatus"; models: ModelState[] };
export type ModelRuntimeState = "stopped" | "loading" | "ready";
export class InflightTracker {
private requests = new Map<string, { model: string; seen: number }>();
private requests = new Map<string, string>();
private states = new Map<string, ModelRuntimeState>();
constructor(private now: () => number = () => Date.now()) {}
apply(event: FeedEvent): void {
if (event.type === "inflight") {
if (event.operation === "snapshot") {
this.requests = new Map<string, { model: string; seen: number }>();
for (const r of event.requests) if (r.id) this.requests.set(r.id, { model: r.model, seen: this.now() });
this.requests = new Map<string, string>();
for (const r of event.requests) if (r.id) this.requests.set(r.id, r.model);
} else if (event.operation === "add") {
for (const r of event.requests) if (r.id) this.requests.set(r.id, { model: r.model, seen: this.now() });
for (const r of event.requests) if (r.id) this.requests.set(r.id, r.model);
} else if (event.operation === "remove") {
this.requests.delete(event.id);
}
this.prune();
} else if (event.type === "modelStatus") {
for (const m of event.models) this.states.set(m.id, normalizeState(m.state));
}
}
prune(maxAgeMs = 120_000): void {
const cutoff = this.now() - maxAgeMs;
for (const [id, entry] of this.requests) if (entry.seen < cutoff) this.requests.delete(id);
}
count(modelId: string): number {
let n = 0;
for (const entry of this.requests.values()) if (entry.model === modelId) n++;
for (const model of this.requests.values()) if (model === modelId) n++;
return n;
}
total(): number {
return this.requests.size;
}
state(modelId: string): ModelRuntimeState | undefined {
return this.states.get(modelId);
}
+2 -3
View File
@@ -1,4 +1,3 @@
import { fetchWith } from "./http";
import { type LlamaSwapConfig } from "./util";
const TIMEOUT_MS = 5000;
@@ -15,7 +14,7 @@ function headersFor(cfg: LlamaSwapConfig): Record<string, string> {
}
export async function fetchMetrics(cfg: LlamaSwapConfig): Promise<string> {
const res = await fetchWith(cfg, `${cfg.baseUrl}/metrics`, {
const res = await fetch(`${cfg.baseUrl}/metrics`, {
headers: headersFor(cfg),
signal: AbortSignal.timeout(TIMEOUT_MS),
});
@@ -24,7 +23,7 @@ export async function fetchMetrics(cfg: LlamaSwapConfig): Promise<string> {
}
export async function fetchModels(cfg: LlamaSwapConfig): Promise<ModelInfo[]> {
const res = await fetchWith(cfg, `${cfg.baseUrl}/v1/models`, {
const res = await fetch(`${cfg.baseUrl}/v1/models`, {
headers: headersFor(cfg),
signal: AbortSignal.timeout(TIMEOUT_MS),
});
-12
View File
@@ -22,18 +22,6 @@ const AGGREGATE: Record<GpuMetricKind, (values: number[]) => number> = {
fan: avg,
};
export function combineSeries(histories: number[][], kind: GpuMetricKind): { value?: number; history: number[] } {
const n = Math.max(...histories.map((h) => h.length), 0);
if (n === 0) return { value: undefined, history: [] };
const history: number[] = [];
for (let i = 0; i < n; i++) {
const at = histories.map((h) => h[i]).filter((v): v is number => v !== undefined);
if (at.length === 0) continue;
history.push(AGGREGATE[kind](at));
}
return { value: history.length > 0 ? history[history.length - 1] : undefined, history };
}
function key(gpuId: string, kind: GpuMetricKind): string {
return `${gpuId}|${kind}`;
}
+1 -68
View File
@@ -1,6 +1,5 @@
import { type GpuMetricKind } from "./metrics-parser";
import { type ModelRuntimeState } from "./inflight-tracker";
import { type UsageStats } from "./stats";
import { escapeXml, shorten } from "./util";
export function svgDataUrl(svg: string): string {
@@ -24,7 +23,7 @@ export interface InflightRenderOptions {
}
export function renderInflight(opts: InflightRenderOptions): string {
const name = escapeXml(shorten(opts.modelName === "all" ? "ALL MODELS" : opts.modelName));
const name = escapeXml(shorten(opts.modelName));
if (opts.offline) {
return frame("#10131a", [
@@ -189,69 +188,3 @@ function frame(bg: string, parts: string[]): string {
${parts.join("\n ")}
</svg>`;
}
export function formatCompact(n: number): string {
if (!Number.isFinite(n)) return "--";
if (n < 1000) return `${Math.round(n)}`;
if (n < 1_000_000) return `${(n / 1000).toFixed(1)}k`;
return `${(n / 1_000_000).toFixed(1)}M`;
}
export interface UsageRenderOptions {
modelName: string;
stats?: UsageStats;
primaryStat: "requests" | "input_tokens" | "output_tokens" | "gen_p95";
offline: boolean;
}
export function renderUsage(opts: UsageRenderOptions): string {
const name = escapeXml(shorten(opts.modelName === "all" ? "ALL MODELS" : opts.modelName));
if (opts.offline) {
return frame("#10131a", [
centerText("!!", 34, 20, "bold", "#e0e0e0"),
centerText("OFFLINE", 52, 9, "normal", "#bdbdbd"),
centerText(name, 64, 7, "normal", "#ffffff"),
]);
}
const s = opts.stats;
const big = s ? formatCompact(primaryValue(s, opts.primaryStat)) : "--";
const rows = usageRows(s, opts.primaryStat);
const parts: string[] = [
centerText(name, 10, 7, "normal", "#ffffff"),
centerText(big, 36, 24, "bold", "#ffffff"),
];
for (const row of rows) {
parts.push(
`<text x="4" y="${row.y}" font-family="Arial,sans-serif" font-size="7" fill="#8b93a5">${row.label}</text>`,
`<text x="68" y="${row.y}" text-anchor="end" font-family="Arial,sans-serif" font-size="7" fill="#ffffff">${row.value}</text>`,
);
}
return frame("#10131a", parts);
}
function primaryValue(s: UsageStats, primary: "requests" | "input_tokens" | "output_tokens" | "gen_p95"): number {
switch (primary) {
case "requests":
return s.totalRequests;
case "input_tokens":
return s.totalInputTokens;
case "output_tokens":
return s.totalOutputTokens;
case "gen_p95":
return s.genP95;
}
}
function usageRows(
s: UsageStats | undefined,
primary: "requests" | "input_tokens" | "output_tokens" | "gen_p95",
): { label: string; value: string; y: number }[] {
const items: { label: string; value: string }[] = [];
if (primary !== "requests") items.push({ label: "REQ", value: s ? formatCompact(s.totalRequests) : "--" });
if (primary !== "input_tokens") items.push({ label: "IN", value: s ? formatCompact(s.totalInputTokens) : "--" });
if (primary !== "output_tokens") items.push({ label: "OUT", value: s ? formatCompact(s.totalOutputTokens) : "--" });
if (primary !== "gen_p95") items.push({ label: "P95", value: s ? `${Math.round(s.genP95)} t/s` : "--" });
return items.map((it, i) => ({ ...it, y: 52 + i * 7 }));
}
+1 -29
View File
@@ -1,8 +1,6 @@
import { EventFeed } from "./event-feed";
import { InflightTracker } from "./inflight-tracker";
import { MetricsPoller } from "./metrics-poller";
import { StatsCache } from "./stats-cache";
import { type UsageStats } from "./stats";
import { type LlamaSwapConfig } from "./util";
class Runtime {
@@ -11,28 +9,20 @@ class Runtime {
private cfg?: LlamaSwapConfig;
private feed?: EventFeed;
private poller?: MetricsPoller;
private statsCache?: StatsCache;
private pruneTimer?: ReturnType<typeof setInterval>;
private listeners = new Set<() => void>();
ensureConnections(cfg: LlamaSwapConfig): void {
const changed =
!this.cfg ||
this.cfg.baseUrl !== cfg.baseUrl ||
this.cfg.apiKey !== cfg.apiKey ||
this.cfg.insecure !== cfg.insecure;
const changed = !this.cfg || this.cfg.baseUrl !== cfg.baseUrl || this.cfg.apiKey !== cfg.apiKey;
if (changed) {
this.feed?.stop();
this.poller?.stop();
this.feed = undefined;
this.poller = undefined;
this.statsCache?.setConfig(cfg);
this.cfg = cfg;
}
if (!this.feed) {
this.feed = new EventFeed(cfg, (ev) => {
this.tracker.apply(ev);
if (ev.type === "activity") this.statsCache?.scheduleRefresh();
this.emit();
});
this.feed.setStatusHandler((connected) => {
@@ -46,28 +36,12 @@ class Runtime {
this.poller.on(() => this.emit());
this.poller.start();
}
if (!this.statsCache) {
this.statsCache = new StatsCache(cfg);
this.statsCache.onChange(() => this.emit());
}
if (!this.pruneTimer) {
this.pruneTimer = setInterval(() => this.tracker.prune(), PRUNE_INTERVAL_MS);
}
}
get pollerInstance(): MetricsPoller | undefined {
return this.poller;
}
getStats(modelId: string): UsageStats | undefined {
return this.statsCache?.get(modelId);
}
watchStats(modelId: string): () => void {
this.statsCache?.register(modelId);
return () => this.statsCache?.unregister(modelId);
}
subscribe(listener: () => void): () => void {
this.listeners.add(listener);
return () => {
@@ -81,5 +55,3 @@ class Runtime {
}
export const runtime = new Runtime();
const PRUNE_INTERVAL_MS = 30_000;
+14 -27
View File
@@ -7,8 +7,7 @@ const ACTIVITY_THROTTLE_MS = 2000;
export type FetchFn = (cfg: LlamaSwapConfig, modelId: string) => Promise<UsageStats | null>;
export class StatsCache {
private refs = new Map<string, number>();
private refreshing = false;
private keys = new Set<string>();
private values = new Map<string, UsageStats | undefined>();
private listeners = new Set<() => void>();
private timer?: ReturnType<typeof setInterval>;
@@ -28,7 +27,8 @@ export class StatsCache {
}
register(key: string): void {
this.refs.set(key, (this.refs.get(key) ?? 0) + 1);
if (this.keys.has(key)) return;
this.keys.add(key);
if (!this.timer) {
void this.refresh();
this.timer = setInterval(() => void this.refresh(), this.pollMs);
@@ -36,18 +36,11 @@ export class StatsCache {
}
unregister(key: string): void {
const count = (this.refs.get(key) ?? 0) - 1;
if (count <= 0) {
this.refs.delete(key);
this.values.delete(key);
} else {
this.refs.set(key, count);
}
if (this.refs.size === 0) {
if (this.timer) clearInterval(this.timer);
this.keys.delete(key);
this.values.delete(key);
if (this.keys.size === 0 && this.timer) {
clearInterval(this.timer);
this.timer = undefined;
if (this.throttleTimer) clearTimeout(this.throttleTimer);
this.throttleTimer = undefined;
}
}
@@ -66,21 +59,15 @@ export class StatsCache {
}
async refresh(): Promise<void> {
if (this.refreshing) return;
this.refreshing = true;
try {
for (const key of this.refs.keys()) {
try {
const stats = await this.fetchFn(this.cfg, key);
if (stats) this.values.set(key, stats);
} catch {
}
this.lastRefresh = Date.now();
for (const key of this.keys) {
try {
const stats = await this.fetchFn(this.cfg, key);
if (stats) this.values.set(key, stats);
} catch {
}
} finally {
this.refreshing = false;
this.lastRefresh = Date.now();
this.emit();
}
this.emit();
}
onChange(listener: () => void): () => void {
+1 -2
View File
@@ -1,4 +1,3 @@
import { fetchWith } from "./http";
import { type LlamaSwapConfig } from "./util";
export interface UsageStats {
@@ -25,7 +24,7 @@ export async function fetchStats(cfg: LlamaSwapConfig, modelId: string): Promise
const query = modelId === "all" ? "" : `?model=${encodeURIComponent(modelId)}`;
const headers: Record<string, string> = {};
if (cfg.apiKey) headers["Authorization"] = `Bearer ${cfg.apiKey}`;
const res = await fetchWith(cfg, `${cfg.baseUrl}/api/metrics/stats${query}`, {
const res = await fetch(`${cfg.baseUrl}/api/metrics/stats${query}`, {
headers,
signal: AbortSignal.timeout(5000),
});
+1 -4
View File
@@ -1,24 +1,21 @@
import type { JsonObject } from "@elgato/utils";
export const DEFAULT_BASE_URL = "http://localhost:9292";
export const DEFAULT_BASE_URL = "http://talos.milky.way:9292";
export type CfgSettings = {
baseUrl?: string;
apiKey?: string;
insecure?: boolean;
} & JsonObject;
export interface LlamaSwapConfig {
baseUrl: string;
apiKey?: string;
insecure?: boolean;
}
export function cfgFromSettings(settings: CfgSettings): LlamaSwapConfig {
return {
baseUrl: normalizeBaseUrl(settings.baseUrl ?? DEFAULT_BASE_URL),
apiKey: settings.apiKey || undefined,
insecure: settings.insecure === true,
};
}
-75
View File
@@ -1,75 +0,0 @@
# Marketplace Listing — llama-watch
Copy the content below into Maker Console when creating the product.
## Product type
Stream Deck plugin
## Name
`llama-watch`
## Author / Organization
Organization: `c4ch3c4d3` (create this in Maker Console; sign the Maker Agreement)
## Description
> Copy exactly (254 chars first 250-char segment is unformatted; total ~900 chars, within the 2501500 limit):
```
llama-watch brings live monitoring of your llama-swap LLM server to your Stream Deck. Watch in-flight request counts per model with a real-time activity spark, switch to Usage stats to see request totals, processed and generated tokens, and the generation-speed P95 — per model or across all models. GPU Graph keys render live line charts of GPU utilization, VRAM, temperature, power draw, and fan speed — for one GPU, all GPUs, or a custom combination.
Features
- In-Flight Monitor: live per-model request count with a 60-second activity spark; Usage stats display (requests, processed/generated tokens, generation-speed P95) per model or all models.
- GPU Graph: live charts for utilization, VRAM, temperature, power draw, and fan speed — single GPU, all GPUs, or a custom combination.
- Per-key configuration: base URL and optional API key; every key is independent.
Requirements
- A running llama-swap instance (open source) and its base URL — default http://localhost:9292.
- Optional API key for protected instances.
- macOS 13 or later; Stream Deck 7.1 or later.
Privacy: the plugin reads data only from your configured llama-swap server and sends nothing elsewhere. No analytics, no telemetry.
```
## Tags
`monitoring`, `gpu`, `llm`, `llama-swap`, `developer tools`
## Price
Free
## Release notes (v1.0.0)
```
Initial release.
- In-Flight Monitor: live per-model request count with a 60-second activity spark; Usage stats display (requests, processed/generated tokens, generation-speed P95) per model or all models.
- GPU Graph: live charts for utilization, VRAM, temperature, power draw, and fan speed — single GPU, all GPUs, or a custom combination.
- Per-key base URL and optional API key settings.
```
## Media checklist (generated by scripts/store-assets.mts)
- `store-assets/app-icon-288.png` — app icon, 288×288 PNG
- `store-assets/thumbnail.png` — thumbnail, 1920×960 PNG
- `store-assets/gallery-1-inflight.png` — gallery, 1920×960 PNG
- `store-assets/gallery-2-gpu.png` — gallery, 1920×960 PNG
- `store-assets/gallery-3-setup.png` — gallery, 1920×960 PNG
## Submission steps (Maker Console)
1. maker.elgato.com → create organization `c4ch3c4d3` → sign the Maker Agreement.
2. Home → Create product → Stream Deck plugin.
3. Upload `com.bryce.llamawatch.streamDeckPlugin` (rebuilt with Author `c4ch3c4d3`).
4. Details: name `llama-watch`, description above, tags, price Free.
5. Media: upload the 5 PNGs above as app icon + thumbnail + 3 gallery items.
6. Release notes: v1.0.0 above. Submit.
7. Review: allow 410 business days; handle feedback by submitting a revision.
Notes
- The product name and monetization cannot be changed after creation without contacting maker@elgato.com — confirm `llama-watch` is still available when you create it.
- No demo video is required (the plugin integrates with the user's own software, not a paid service/hardware).
-18
View File
@@ -67,15 +67,6 @@ test("decodeEvent parses an inflight remove with a bare id", () => {
assert.deepEqual(ev, { type: "inflight", operation: "remove", id: "817" });
});
test("decodeEvent accepts a numeric id on remove", () => {
const msg: SseMessage = {
event: "message",
data: JSON.stringify({ type: "inflight", data: JSON.stringify({ operation: "remove", id: 820 }) }),
};
const ev = decodeEvent(msg);
assert.deepEqual(ev, { type: "inflight", operation: "remove", id: "820" });
});
test("decodeEvent parses modelStatus", () => {
const msg: SseMessage = {
event: "message",
@@ -126,12 +117,3 @@ test("decodeEvent handles inflight with a non-array requests field and no reques
assert.deepEqual(ev.requests, []);
}
});
test("decodeEvent parses an activity event with an id", () => {
const msg: SseMessage = {
event: "message",
data: JSON.stringify({ type: "activity", data: JSON.stringify({ id: 817 }) }),
};
const ev = decodeEvent(msg);
assert.deepEqual(ev, { type: "activity", id: 817 });
});
-19
View File
@@ -1,19 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIDCTCCAfGgAwIBAgIUXQ30Z78Fhhti2Ct6DXczY/jvDjcwDQYJKoZIhvcNAQEL
BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI2MDgyNzIxMTAyNloXDTM2MDgy
NDIxMTAyNlowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF
AAOCAQ8AMIIBCgKCAQEAtJNuhOYp4Qi7W17LRvph2EZH7WfQsbELROrT3ijJ2ys0
iA+Bee0ATpc/1R+NgSGfQZsPlAYh2TlYK0UWBIA1BGKm5tSHpBvfxs+GXcgmFM0k
nWFE4Kof6d8zvcd3l7aU+8w8Tz/LnhLpaYQiixZ4ZIoAC1SEWKO0wm+AoMSSeLjH
CBaGmLuD1MQ+I6kZPwDK2YluZ28LHMH9ZCbgfVKXR0FVjgF0fEw6645IzgWGUznN
8pcEisVG2Veyr83Q7uqEL3CtvrdYFjgFJP0Qzez2zgCGXKB+tiBsWIzL2rwVnDcV
ss2bwAVz9JMUxaj8aoMq0Bx5hJkzK9JipGRB9C58CQIDAQABo1MwUTAdBgNVHQ4E
FgQUCtzsN0GW6A2iKrJF28+UVJOY7zkwHwYDVR0jBBgwFoAUCtzsN0GW6A2iKrJF
28+UVJOY7zkwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEApZfd
QLXrtRfu6NG1BPfzaROjwTLs5O0tU8+/SSyi3WjoIZmYUBkP7nJm/sR2ZxpSUEfq
OFZwux8bDJTccS9Au0/OR9vYbHWmDoY1e14v2GehNRWXz8vvaD3AURluAYXcmgol
5WOnSB4W8rp6A5gEKX7n4hsHrkUx/Mt+uSg7KZv/fFnwQqyu9OthPKVNV1v79dgm
C0ZUNhpaDDX5Ae+khY88kGADwCKY9BTNzftQG0/4ZJgu7O34eHNgC+2MAgXDtLxm
TIk3XSEaFDoT+HOX38Lh3JYnzJCFO59tQfMMGrPuWfgOISxCe5tkZ22zTtPId+wB
+TptauhacNtwwpc1Jw==
-----END CERTIFICATE-----
-28
View File
@@ -1,28 +0,0 @@
-----BEGIN PRIVATE KEY-----
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC0k26E5inhCLtb
XstG+mHYRkftZ9CxsQtE6tPeKMnbKzSID4F57QBOlz/VH42BIZ9Bmw+UBiHZOVgr
RRYEgDUEYqbm1IekG9/Gz4ZdyCYUzSSdYUTgqh/p3zO9x3eXtpT7zDxPP8ueEulp
hCKLFnhkigALVIRYo7TCb4CgxJJ4uMcIFoaYu4PUxD4jqRk/AMrZiW5nbwscwf1k
JuB9UpdHQVWOAXR8TDrrjkjOBYZTOc3ylwSKxUbZV7KvzdDu6oQvcK2+t1gWOAUk
/RDN7PbOAIZcoH62IGxYjMvavBWcNxWyzZvABXP0kxTFqPxqgyrQHHmEmTMr0mKk
ZEH0LnwJAgMBAAECggEADgjFVqvixlwc36GS7+3Gy/3OWkuuwxis9QrBM6t84L1P
ZGG8IONEGleT/PbqUwZvb7Ri9hCx8cWMrjQ83VWviSs3qIoNDrqh3jxDx6cmGojF
Fzw3k7R1LYKM7WuCxnZIxvcdGtWs+BilLm+4FZJGAh5dmYPUk2UJx/DNkPEmJx6n
4aSEwUnV1QGPR7IjchGEHImBj8T8L3HBQnrsRsCRMwgUs3z0XMgfZoFm9/dq08YA
NPzyxAgSsQQrtVOyJMOFj+wi2NrUFtQ0OfS617cUMQXOyceQPpUECn2zPU1UIoPy
VgXfkxSXVdraetgNACELJvBI1bvvZctmkJYkEkY5lwKBgQDXcBDCHn4Any+T4gV8
Jp9XQf9lCWQnyegP7sSB9vLxzFKU49tuEDDG3m1wOPw17OV/Cs1sOzbiaFXSMM78
rD2ljRIODaewr3/Js8B6XbYmmkJqth8ByYuKV6kyq1aFPb4zTmEtcWbnnx45mUHr
yZr+8H33yl1iWhR7rdgAtvNP4wKBgQDWkw1C6qR9T8v8sf/vNbwJUHC5xD6B5d18
XIV8+Uax1tgmIupy/1hVllLBcaizBpE5eFPsWMMNw2pujnSKNER2hfADSFXYUlMO
Ror6e+Dfhr9MctiJjqhNW9kIPBvvnqzpOqwHun6P1GEC/qOi4GIdDDP95OqDaDsV
BvKNKVAwIwKBgQCS+iKEvNbLx85mvrFtRNA6cI0zuhd5Sbcnf4bS/845BmNkrpsa
WLNeSYsyH755b7gWVyFUcIV+Kx45uxDLsxqPolGqAsjfsqukyRxMnzhQ17buJHe8
+WpYpHuLVPc/CaOETznfDdndtWGifBtMKIu02A+oiIfzPG9y/WQ7AJW4bwKBgFmq
FW6TEq1yvPEZiLNzJuJVhOV7xgsN/SHMn9N7bzk9aBF3obTwUv9g07AWSMKWyfTT
/W3UIZ4MvNr6GGTwNnO4wHT+szC0JhTfEZBeV7fQXPwbObUxsc6xxN2WEK5vBh5n
8B9CpUSBIRDZS5PyY81znf5IvF6xHY9J2e13CBU1AoGARfyoIH6gF92bI1DNZ4cT
mtgPYEGbWYr28/ADF25CrlZ3HvWDqwt2Oe1EvWWyALLjTyKdZXuc0iFiKUSsgUSm
qlBjoh1lWXFrtm+uu8rtGaWrp9q0xjNhcmdM0TqjXpvifDXLALGUiRGvUjZHGo8J
mgw1KgJviTZzZu2cy7iFCqg=
-----END PRIVATE KEY-----
-55
View File
@@ -1,55 +0,0 @@
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { createServer, type Server } from "node:https";
import { after, before, test } from "node:test";
import { fetchWith } from "../src/lib/http";
const TLS_OPTIONS = {
cert: readFileSync(new URL("./fixtures/selfsigned-cert.pem", import.meta.url)),
key: readFileSync(new URL("./fixtures/selfsigned-key.pem", import.meta.url)),
};
let server: Server;
let base: string;
before(async () => {
server = createServer(TLS_OPTIONS, (req, res) => {
res.writeHead(200, { "content-type": "text/plain" });
res.end("ok");
});
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
base = `https://127.0.0.1:${(server.address() as { port: number }).port}`;
});
after(async () => new Promise((resolve) => server.close(() => resolve())));
test("fetchWith verifies certificates by default", async () => {
await assert.rejects(fetchWith({ baseUrl: base }, `${base}/`));
});
test("fetchWith accepts self-signed certificates when insecure", async () => {
const res = await fetchWith({ baseUrl: base, insecure: true }, `${base}/`);
assert.equal(res.status, 200);
assert.equal(await res.text(), "ok");
});
test("fetchWith honors init headers and signal when insecure", async () => {
let seen: string | undefined;
const spy = createServer(TLS_OPTIONS, (req, res) => {
seen = req.headers.authorization;
res.writeHead(200);
res.end("ok");
});
await new Promise<void>((resolve) => spy.listen(0, "127.0.0.1", resolve));
const spyBase = `https://127.0.0.1:${(spy.address() as { port: number }).port}`;
try {
const res = await fetchWith({ baseUrl: spyBase, insecure: true }, `${spyBase}/`, {
headers: { Authorization: "Bearer tok" },
signal: AbortSignal.timeout(3000),
});
assert.equal(res.status, 200);
assert.equal(seen, "Bearer tok");
} finally {
await new Promise<void>((resolve) => spy.close(() => resolve()));
}
});
-41
View File
@@ -63,44 +63,3 @@ test("modelStatus normalizes states", () => {
assert.equal(tracker.state("D"), "stopped");
assert.equal(tracker.state("missing"), undefined);
});
test("total sums in-flight requests across all models", () => {
const tracker = new InflightTracker();
assert.equal(tracker.total(), 0);
tracker.apply({ type: "inflight", operation: "add", requests: [{ model: "A", id: "1" }, { model: "B", id: "2" }] });
tracker.apply({ type: "inflight", operation: "add", requests: [{ model: "A", id: "3" }] });
assert.equal(tracker.total(), 3);
tracker.apply({ type: "inflight", operation: "remove", id: "1" });
assert.equal(tracker.total(), 2);
});
test("prune drops request ids that stopped receiving updates", () => {
let now = 0;
const tracker = new InflightTracker(() => now);
tracker.apply({ type: "inflight", operation: "add", requests: [{ model: "A", id: "1" }] });
now = 200_000;
tracker.apply({ type: "inflight", operation: "add", requests: [{ model: "A", id: "2" }] });
tracker.prune(60_000);
assert.equal(tracker.total(), 1);
assert.equal(tracker.count("A"), 1);
});
test("active requests refreshed by upserts survive pruning", () => {
let now = 0;
const tracker = new InflightTracker(() => now);
tracker.apply({ type: "inflight", operation: "add", requests: [{ model: "A", id: "1" }] });
now = 200_000;
tracker.apply({ type: "inflight", operation: "add", requests: [{ model: "A", id: "1" }] });
tracker.prune(60_000);
assert.equal(tracker.total(), 1);
});
test("prune is applied on every inflight event", () => {
let now = 0;
const tracker = new InflightTracker(() => now);
tracker.apply({ type: "inflight", operation: "add", requests: [{ model: "A", id: "stale" }] });
now = 200_000;
tracker.apply({ type: "inflight", operation: "add", requests: [{ model: "B", id: "live" }] });
assert.equal(tracker.total(), 1);
assert.equal(tracker.count("B"), 1);
});
+1 -29
View File
@@ -2,7 +2,7 @@ import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { test } from "node:test";
import { type LlamaSwapConfig } from "../src/lib/util";
import { combineSeries, MetricsPoller } from "../src/lib/metrics-poller";
import { MetricsPoller } from "../src/lib/metrics-poller";
const FIXTURE = readFileSync(new URL("./fixtures/metrics.txt", import.meta.url), "utf8");
@@ -61,31 +61,3 @@ test("poller reports offline after a fetch failure and recovers", async () => {
assert.equal(poller.isOffline(), false);
assert.ok(poller.getValue("0", "util_percent") !== undefined);
});
test("combineSeries averages util and sums power", () => {
const util = combineSeries(
[
[10, 20, 30],
[20, 40, 60],
],
"util_percent",
);
assert.deepEqual(util, { value: 45, history: [15, 30, 45] });
const power = combineSeries(
[
[100, 200],
[150, 250],
],
"power",
);
assert.deepEqual(power, { value: 450, history: [250, 450] });
});
test("combineSeries takes the max temperature and skips gaps", () => {
const temp = combineSeries([[50, 70], [60]], "temperature");
assert.deepEqual(temp, { value: 70, history: [60, 70] });
const empty = combineSeries([[], []], "temperature");
assert.deepEqual(empty, { value: undefined, history: [] });
});
+1 -35
View File
@@ -1,6 +1,6 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { renderGpuGraph, renderInflight, renderUsage, svgDataUrl, formatCompact } from "../src/lib/render";
import { renderGpuGraph, renderInflight, svgDataUrl } from "../src/lib/render";
test("svgDataUrl wraps an SVG as a base64 data URL", () => {
const url = svgDataUrl("<svg></svg>");
@@ -99,37 +99,3 @@ test("renderGpuGraph: single-point history avoids divide-by-zero", () => {
const svg = renderGpuGraph({ gpuName: "ALL GPUS", metric: "util_percent", value: 50, history: [50], offline: false });
assert.match(svg, /<svg/);
});
test("formatCompact renders integers, k, and M", () => {
assert.equal(formatCompact(0), "0");
assert.equal(formatCompact(999), "999");
assert.equal(formatCompact(52100), "52.1k");
assert.equal(formatCompact(1200000), "1.2M");
});
test("renderUsage shows the primary stat big and the rest small", () => {
const svg = renderUsage({
modelName: "DeepSeek-V4-Flash-0731",
stats: { totalRequests: 1950, totalInputTokens: 312177540, totalOutputTokens: 889193, genP95: 378.86 },
primaryStat: "gen_p95",
offline: false,
});
assert.match(svg, />379</);
assert.match(svg, /REQ/);
assert.match(svg, /IN/);
assert.match(svg, /OUT/);
assert.match(svg, /312.2M/);
assert.match(svg, /#10131a/);
});
test("renderUsage renders -- when stats are unknown and OFFLINE when offline", () => {
const none = renderUsage({ modelName: "A", stats: undefined, primaryStat: "requests", offline: false });
assert.match(none, />--</);
const off = renderUsage({ modelName: "A", stats: undefined, primaryStat: "requests", offline: true });
assert.match(off, /OFFLINE/);
});
test("renderUsage labels the aggregate view ALL MODELS", () => {
const svg = renderUsage({ modelName: "all", stats: { totalRequests: 1, totalInputTokens: 1, totalOutputTokens: 1, genP95: 1 }, primaryStat: "requests", offline: false });
assert.match(svg, /ALL MODELS/);
});
-28
View File
@@ -83,31 +83,3 @@ test("fetch failure keeps the last-known value", async () => {
assert.equal(cache.get("a"), undefined);
cache.unregister("a");
});
test("ref-counted register: a sibling unregister keeps the key active", async () => {
const calls = { count: 0 };
const cache = new StatsCache(cfg, stubFetch({ totalRequests: 5, totalInputTokens: 1, totalOutputTokens: 1, genP95: 1 }, calls), 10000, 30);
cache.register("all");
await flush();
assert.equal(cache.get("all")!.totalRequests, 5);
cache.register("all");
cache.unregister("all");
await flush();
assert.equal(cache.get("all")!.totalRequests, 5);
cache.unregister("all");
assert.equal(cache.get("all"), undefined);
});
test("setConfig keeps registrations but clears cached values", async () => {
const cache = new StatsCache(cfg, async (_c, key) => ({ totalRequests: 5, totalInputTokens: 1, totalOutputTokens: 1, genP95: 1 }), 10000, 30);
cache.register("all");
await flush();
assert.equal(cache.get("all")!.totalRequests, 5);
cache.setConfig({ baseUrl: "http://new" });
assert.equal(cache.get("all"), undefined);
cache.scheduleRefresh();
await flush();
await new Promise((r) => setTimeout(r, 60));
assert.equal(cache.get("all")!.totalRequests, 5);
cache.unregister("all");
});
+4 -7
View File
@@ -8,21 +8,18 @@ import {
} from "../src/lib/util";
test("normalizeBaseUrl strips trailing slashes", () => {
assert.equal(normalizeBaseUrl("http://localhost:9292/"), "http://localhost:9292");
assert.equal(normalizeBaseUrl("http://localhost:9292"), "http://localhost:9292");
assert.equal(normalizeBaseUrl("http://talos.milky.way:9292/"), "http://talos.milky.way:9292");
assert.equal(normalizeBaseUrl("http://talos.milky.way:9292"), "http://talos.milky.way:9292");
});
test("cfgFromSettings defaults baseUrl and omits empty apiKey", () => {
const cfg = cfgFromSettings({});
assert.equal(cfg.baseUrl, "http://localhost:9292");
assert.equal(cfg.baseUrl, "http://talos.milky.way:9292");
assert.equal(cfg.apiKey, undefined);
assert.equal(cfg.insecure, false);
assert.deepEqual(cfgFromSettings({ baseUrl: "http://x/", apiKey: "abc", insecure: true }), {
assert.deepEqual(cfgFromSettings({ baseUrl: "http://x/", apiKey: "abc" }), {
baseUrl: "http://x",
apiKey: "abc",
insecure: true,
});
assert.equal(cfgFromSettings({ insecure: "yes" }).insecure, false);
});
test("shorten keeps short names and truncates long ones", () => {