fix: track in-flight requests by id; handle llama-swap upsert/remove events

llama-swap emits inflight events with operation 'upsert' (a single request,
re-emitted as elapsed_ms updates) and id-only 'remove' events. The decoder
ignored 'upsert' and dropped every id-only remove, so the tracker only ever
reflected the connect-time snapshot and the count got stuck (e.g. at '1').

The tracker now keys in-flight state by request id (snapshot/add replace by
id; remove deletes by id), and the decoder normalizes 'upsert' to 'add' and
extracts the id from bare removes.
This commit is contained in:
2026-08-14 12:08:02 -06:00
parent a805195c49
commit 07fea457f1
4 changed files with 64 additions and 18 deletions
+26
View File
@@ -41,6 +41,32 @@ test("decodeEvent parses an inflight add with a singular request field", () => {
}
});
test("decodeEvent parses an inflight upsert as an add", () => {
const msg: SseMessage = {
event: "message",
data: JSON.stringify({
type: "inflight",
data: JSON.stringify({ operation: "upsert", request: { id: "820", model: "DeepSeek-V4-Flash-0731", elapsed_ms: 103 } }),
}),
};
const ev = decodeEvent(msg);
assert.ok(ev && ev.type === "inflight");
if (ev && ev.type === "inflight") {
assert.equal(ev.operation, "add");
assert.equal(ev.requests[0].id, "820");
assert.equal(ev.requests[0].elapsed_ms, 103);
}
});
test("decodeEvent parses an inflight remove with a bare id", () => {
const msg: SseMessage = {
event: "message",
data: JSON.stringify({ type: "inflight", data: JSON.stringify({ operation: "remove", id: "817" }) }),
};
const ev = decodeEvent(msg);
assert.deepEqual(ev, { type: "inflight", operation: "remove", id: "817" });
});
test("decodeEvent parses modelStatus", () => {
const msg: SseMessage = {
event: "message",
+20 -5
View File
@@ -18,17 +18,32 @@ test("snapshot rebuilds counts for all in-flight requests", () => {
assert.equal(tracker.count("C"), 0);
});
test("add and remove adjust per-model counts", () => {
test("add (upsert) is idempotent per request id", () => {
const tracker = new InflightTracker();
tracker.apply({ type: "inflight", operation: "add", requests: [{ model: "A", id: "1" }] });
tracker.apply({ type: "inflight", operation: "add", requests: [{ model: "A", id: "1" }] });
tracker.apply({ type: "inflight", operation: "add", requests: [{ model: "A", id: "2" }] });
assert.equal(tracker.count("A"), 2);
tracker.apply({ type: "inflight", operation: "remove", requests: [{ model: "A", id: "1" }] });
});
test("remove is id-based and decrements the matching request", () => {
const tracker = new InflightTracker();
tracker.apply({ type: "inflight", operation: "add", requests: [{ model: "A", id: "1" }, { model: "B", id: "2" }] });
assert.equal(tracker.count("A"), 1);
tracker.apply({ type: "inflight", operation: "remove", requests: [{ model: "A", id: "2" }] });
assert.equal(tracker.count("A"), 0);
tracker.apply({ type: "inflight", operation: "remove", requests: [{ model: "A", id: "9" }] });
assert.equal(tracker.count("B"), 1);
tracker.apply({ type: "inflight", operation: "remove", id: "1" });
assert.equal(tracker.count("A"), 0);
assert.equal(tracker.count("B"), 1);
tracker.apply({ type: "inflight", operation: "remove", id: "999" });
assert.equal(tracker.count("B"), 1);
});
test("snapshot replaces prior in-flight state", () => {
const tracker = new InflightTracker();
tracker.apply({ type: "inflight", operation: "add", requests: [{ model: "A", id: "1" }] });
tracker.apply({ type: "inflight", operation: "snapshot", requests: [{ model: "A", id: "2" }] });
assert.equal(tracker.count("A"), 1);
assert.equal(tracker.count("B"), 0);
});
test("modelStatus normalizes states", () => {