fix: track in-flight requests by id; handle llama-swap upsert/remove events

llama-swap emits inflight events with operation 'upsert' (a single request,
re-emitted as elapsed_ms updates) and id-only 'remove' events. The decoder
ignored 'upsert' and dropped every id-only remove, so the tracker only ever
reflected the connect-time snapshot and the count got stuck (e.g. at '1').

The tracker now keys in-flight state by request id (snapshot/add replace by
id; remove deletes by id), and the decoder normalizes 'upsert' to 'add' and
extracts the id from bare removes.
This commit is contained in:
2026-08-14 12:08:02 -06:00
parent 6053713020
commit c6e8a9ad11
4 changed files with 64 additions and 18 deletions
+10 -12
View File
@@ -13,29 +13,25 @@ export interface ModelState {
}
export type FeedEvent =
| { type: "inflight"; operation: "snapshot" | "add" | "remove"; requests: InflightRequest[] }
| { type: "inflight"; operation: "snapshot" | "add"; requests: InflightRequest[] }
| { type: "inflight"; operation: "remove"; id: string }
| { type: "modelStatus"; models: ModelState[] };
export type ModelRuntimeState = "stopped" | "loading" | "ready";
export class InflightTracker {
private counts = new Map<string, number>();
private requests = new Map<string, string>();
private states = new Map<string, ModelRuntimeState>();
apply(event: FeedEvent): void {
if (event.type === "inflight") {
if (event.operation === "snapshot") {
const counts = new Map<string, number>();
for (const r of event.requests) counts.set(r.model, (counts.get(r.model) ?? 0) + 1);
this.counts = counts;
this.requests = new Map<string, string>();
for (const r of event.requests) if (r.id) this.requests.set(r.id, r.model);
} else if (event.operation === "add") {
for (const r of event.requests) this.counts.set(r.model, (this.counts.get(r.model) ?? 0) + 1);
for (const r of event.requests) if (r.id) this.requests.set(r.id, r.model);
} else if (event.operation === "remove") {
for (const r of event.requests) {
const c = this.counts.get(r.model) ?? 0;
if (c <= 1) this.counts.delete(r.model);
else this.counts.set(r.model, c - 1);
}
this.requests.delete(event.id);
}
} else if (event.type === "modelStatus") {
for (const m of event.models) this.states.set(m.id, normalizeState(m.state));
@@ -43,7 +39,9 @@ export class InflightTracker {
}
count(modelId: string): number {
return this.counts.get(modelId) ?? 0;
let n = 0;
for (const model of this.requests.values()) if (model === modelId) n++;
return n;
}
state(modelId: string): ModelRuntimeState | undefined {