fix: track in-flight requests by id; handle llama-swap upsert/remove events

llama-swap emits inflight events with operation 'upsert' (a single request,
re-emitted as elapsed_ms updates) and id-only 'remove' events. The decoder
ignored 'upsert' and dropped every id-only remove, so the tracker only ever
reflected the connect-time snapshot and the count got stuck (e.g. at '1').

The tracker now keys in-flight state by request id (snapshot/add replace by
id; remove deletes by id), and the decoder normalizes 'upsert' to 'add' and
extracts the id from bare removes.
This commit is contained in:
2026-08-14 12:08:02 -06:00
parent a805195c49
commit 07fea457f1
4 changed files with 64 additions and 18 deletions
+8 -1
View File
@@ -9,6 +9,13 @@ export function decodeEvent(msg: SseMessage): FeedEvent | null {
const inner = JSON.parse(outer.data) as Record<string, unknown>;
if (outer.type === "inflight") {
const operation = inner.operation as string;
if (operation === "remove") {
const id = typeof inner.id === "string" ? inner.id : undefined;
if (id) return { type: "inflight", operation: "remove", id };
return null;
}
if (operation !== "snapshot" && operation !== "upsert" && operation !== "add") return null;
const requests: InflightRequest[] = Array.isArray(inner.requests)
? (inner.requests as InflightRequest[])
: Array.isArray(inner.request)
@@ -18,7 +25,7 @@ export function decodeEvent(msg: SseMessage): FeedEvent | null {
: [];
return {
type: "inflight",
operation: inner.operation as "snapshot" | "add" | "remove",
operation: operation === "snapshot" ? "snapshot" : "add",
requests,
};
}
+10 -12
View File
@@ -13,29 +13,25 @@ export interface ModelState {
}
export type FeedEvent =
| { type: "inflight"; operation: "snapshot" | "add" | "remove"; requests: InflightRequest[] }
| { type: "inflight"; operation: "snapshot" | "add"; requests: InflightRequest[] }
| { type: "inflight"; operation: "remove"; id: string }
| { type: "modelStatus"; models: ModelState[] };
export type ModelRuntimeState = "stopped" | "loading" | "ready";
export class InflightTracker {
private counts = new Map<string, number>();
private requests = new Map<string, string>();
private states = new Map<string, ModelRuntimeState>();
apply(event: FeedEvent): void {
if (event.type === "inflight") {
if (event.operation === "snapshot") {
const counts = new Map<string, number>();
for (const r of event.requests) counts.set(r.model, (counts.get(r.model) ?? 0) + 1);
this.counts = counts;
this.requests = new Map<string, string>();
for (const r of event.requests) if (r.id) this.requests.set(r.id, r.model);
} else if (event.operation === "add") {
for (const r of event.requests) this.counts.set(r.model, (this.counts.get(r.model) ?? 0) + 1);
for (const r of event.requests) if (r.id) this.requests.set(r.id, r.model);
} else if (event.operation === "remove") {
for (const r of event.requests) {
const c = this.counts.get(r.model) ?? 0;
if (c <= 1) this.counts.delete(r.model);
else this.counts.set(r.model, c - 1);
}
this.requests.delete(event.id);
}
} else if (event.type === "modelStatus") {
for (const m of event.models) this.states.set(m.id, normalizeState(m.state));
@@ -43,7 +39,9 @@ export class InflightTracker {
}
count(modelId: string): number {
return this.counts.get(modelId) ?? 0;
let n = 0;
for (const model of this.requests.values()) if (model === modelId) n++;
return n;
}
state(modelId: string): ModelRuntimeState | undefined {