diff --git a/README.md b/README.md index f25d159..4772b3c 100644 --- a/README.md +++ b/README.md @@ -55,15 +55,20 @@ Every 500ms, the plugin polls `GET /slots?model=` on each discovered serv ### State Classification -Each slot is classified as prefill or generation based on the `n_decoded` counter in `next_token[0]`. The plugin tracks a per-slot baseline value: +Each processing slot is classified as prefill (PP) or generation (TG) by comparing `n_decoded` and `n_prompt_tokens` against per-slot/per-task baselines. The plugin tracks two baseline maps keyed by slot ID: -1. When a slot first appears as processing, the current `n_decoded` is recorded as the baseline with `hasIncreased = false`. -2. If `n_decoded <= baseline` and `hasIncreased` is false, the slot is classified as prefilling. -3. If `n_decoded > baseline`, `hasIncreased` is set to true and the slot is classified as generating. +- `prevNdBySlot[slotId]` — stores `{ prevNd, idTask, classification }` +- `prevPromptTokensBySlot[slotId]` — stores `{ tokens, idTask }` -This approach handles the case where `n_decoded` drops when a new request starts on a reused slot, and prevents generation stalls (where `n_decoded` plateaus) from being misclassified as prefill. +The `idTask` field ensures baselines are invalidated when a slot starts a new task (task reuse). -When no slots are processing, all tracked state for those slots is cleared. +Classification logic: + +1. **First time or after task change/reset**: If we have no baseline for this slot+task, or `n_decoded` or `n_prompt_tokens` dropped below baseline (indicating a new request), use a heuristic: `n_decoded > 0` → TG, otherwise → PP. +2. **Compare against baseline**: If `n_decoded > baselineNd` → TG. If `n_prompt_tokens > baselineNt` → PP. +3. **Neither increased**: Persist the previous classification. This handles frequent polling where neither field increases on every poll cycle. + +When a slot stops being processed, all tracked state for that slot is cleared. ### Prefill Rate (PP) diff --git a/package.json b/package.json index 4e73c60..6614bda 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/package.json", "name": "@troed/oc-ls-stats", - "version": "1.1.0", + "version": "1.1.5", "type": "module", "exports": { "./tui": { diff --git a/tui.tsx b/tui.tsx index 294bc78..83b7137 100644 --- a/tui.tsx +++ b/tui.tsx @@ -143,16 +143,23 @@ function SessionPromptRight(props: { }) const text = createMemo(() => { + props.clock() if (props.tracker.failure) return props.tracker.failure const pp = prefillRate() const tps = liveTps() if (props.tracker.isPrefilling && pp) { - return `${formatPps(pp)} tps (PP)` + const result = `${formatPps(pp)} tps (PP)` + debug(`UI_TEXT="${result}" isPrefilling=${props.tracker.isPrefilling} isGenerating=${props.tracker.isGenerating} failure=${props.tracker.failure} pp=${pp} tps=${tps}`) + return result } if (props.tracker.isGenerating && tps) { - return `${formatTps(tps)} tps (TG)` + const result = `${formatTps(tps)} tps (TG)` + debug(`UI_TEXT="${result}" isPrefilling=${props.tracker.isPrefilling} isGenerating=${props.tracker.isGenerating} failure=${props.tracker.failure} pp=${pp} tps=${tps}`) + return result } - return `- tps (TG)` + const result = `- tps (TG)` + debug(`UI_TEXT="${result}" isPrefilling=${props.tracker.isPrefilling} isGenerating=${props.tracker.isGenerating} failure=${props.tracker.failure} pp=${pp} tps=${tps}`) + return result }) return <>{text() ? {text()} : null} @@ -256,6 +263,7 @@ const tui: TuiPlugin = async (api) => { bump() const result = processPoll(slotList, tracker) + bump() if (result.isPrefilling) { debug(`PP rate=${Math.round(result.lastPrefillRate)} tps`) @@ -270,10 +278,6 @@ const tui: TuiPlugin = async (api) => { debug(`TG ended rate=${Math.round(result.lastGeneratedTps)} tps`) } } - - if (tracker.lastPrefillRate > 0 || tracker.lastGeneratedTps > 0) { - bump() - } } const onDelta = api.event.on("message.part.delta", (evt) => { @@ -360,6 +364,7 @@ const tui: TuiPlugin = async (api) => { api.slots.register({ slots: { session_prompt_right(_ctx, value) { + debug(`slot_callback: session=${value.session_id}`) return }, },