diff --git a/AGENTS.md b/AGENTS.md index 87b0ae5..dfb44e5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,20 +63,21 @@ The plugin polls llama-server's `GET /slots` endpoint to track prompt processing - Falls back to `http://localhost:8080` default URL ### Slot Polling -- Polls `GET /slots?model=...` every 2 seconds (`SLOT_POLL_MS = 2000`) -- Prefill detection: `is_processing === true` AND `next_token[0].n_remain === -1` +- Polls `GET /slots?model=...` every 500ms (`SLOT_POLL_MS = 500`) +- Prefill detection: `is_processing === true` AND `next_token[0].n_decoded === 0` - Model parameter is required by the `/slots` endpoint - Falls back to `http://localhost:8080` default URL ### Display Format -`PP {pp} | TPS {tps} | AVG {avg}` -Example: `PP 1247 | TPS 25 | AVG 25` +`PP xxxx tps | GEN xxxx tps` +Example: `PP 1247 tps | GEN 25 tps` ### Display Rules - PP: integer only, no suffix, 4-char fixed-width padded -- TPS: integer only, 4-char fixed-width padded -- AVG: one decimal when < 100, integer when >= 100, 4-char fixed-width padded +- TPS: integer only, no suffix, 4-char fixed-width padded - "----" shown when no value available +- PP only shown during active prefill (`n_decoded === 0`) +- GEN only shown during active generation (`n_decoded > 0`) ### Limitations - Slot state tracked globally (not per-session) — slot IDs don't map to OpenCode sessions @@ -179,7 +180,7 @@ return 0` (e.g. 31943) | `0` | incrementing | | Transition | `true` | `>0` (e.g. 31999) | `0` | stable (final) | | Generation | `true` | `>0` (decreasing) | `>0` (incrementing) | stable (final) | -**Key insight:** `n_remain === -1` is the definitive prefill indicator. `n_decoded === 0` alone is not sufficient — a slot can have `n_decoded === 0` during generation start (transition phase). +**Key insight:** `n_decoded === 0` is the definitive prefill indicator (when `is_processing === true`). `n_remain === -1` only occurs when `n_predict == -1` (unlimited generation), which is rare in practice. ### PP Calculation Method PP is calculated from the per-slot `n_prompt_tokens` field during prefill: -1. Detect prefill: slot with `is_processing === true` AND `next_token[0].n_remain === -1` +1. Detect prefill: slot with `is_processing === true` AND `next_token[0].n_decoded === 0` 2. On first detection: capture `slot.id` and `slot.n_prompt_tokens` as baseline 3. On subsequent polls: `delta = current_n_prompt_tokens - baseline_tokens`, `dt = (now - startAt) / 1000`, `PP = delta / dt` -4. Reset when no slot has `n_remain === -1` (prefill ended) +4. Reset when no slot has `n_decoded === 0` (prefill ended) ### Why NOT `llamacpp:prompt_tokens_total` from `/metrics` @@ -220,14 +221,28 @@ PP is calculated from the per-slot `n_prompt_tokens` field during prefill: ### Slot Polling -- Polls `GET /slots?model=...` every 2 seconds (`SLOT_POLL_MS = 2000`) +- Polls `GET /slots?model=...` every 500ms (`SLOT_POLL_MS = 500`) - Model parameter is required by the `/slots` endpoint (returns error without it) -- Model discovered from: `tui.json` options → route session → OpenCode config provider parsing +- Model discovered from: route session → OpenCode config provider parsing - Falls back to `http://localhost:8080` default URL ### Display -PP only shown during active prefill (`n_remain === -1`). Format: integer only, no suffix (e.g., "1247"). Fixed-width 4 chars, space-padded. Shows "----" when no value. +PP only shown during active prefill (`n_decoded === 0`). Format: integer only, no suffix (e.g., "1247"). Fixed-width 4 chars, space-padded. Shows "----" when no value. + +### Why Progress Percentage Is Not Shown + +The `/slots` endpoint does not expose the **final prompt size** (`task->n_tokens()` from llama.cpp). During prefill, `n_prompt_tokens` and `n_prompt_tokens_processed` are always equal because every token is processed as it's added to the prompt. Progress percentage requires the ratio `n_prompt_tokens_processed / task->n_tokens()`, but `task->n_tokens()` is not returned in `/slots` JSON. + +**Required llama.cpp changes:** + +- **PR #14685** (merged as part of #14728): adds `prompt_progress` streaming to `/completion` endpoint with `total`, `cache`, `processed`, `time_ms`. This enables progress for streaming clients but not for `/slots` polling. +- **PR #23454** (merged May 2026): added `n_prompt_tokens`, `n_prompt_tokens_processed`, `n_prompt_tokens_cache` to `/slots` — but NOT the total prompt size. +- **Needed**: expose `n_prompt_tokens_total` (or `n_tokens` from `task->n_tokens()`) in `/slots` output so polling clients can compute `(n_prompt_tokens_processed / n_prompt_tokens_total) * 100`. + +### Why `n_remain === -1` Does NOT Indicate Prefill + +`n_remain === -1` only occurs when `n_predict == -1` (unlimited generation). When `n_predict` is set (e.g., 32000), `n_remain = n_predict - n_decoded`, which is always positive during prefill. Prefill detection must use `n_decoded === 0` instead. ## Debugging: PP Value Wrong @@ -238,7 +253,7 @@ PP shows 3634 when llama-server console shows ~1200-1300 tokens/sec during prefi Used `llamacpp:prompt_tokens_total` from `/metrics` endpoint which is a global counter across all slots. Delta between polls included tokens from other active slots. ### Fix -Track `n_prompt_tokens` from the slot data itself (per-slot field) during prefill state (`n_remain === -1`). +Track `n_prompt_tokens` from the slot data itself (per-slot field) during prefill state (`n_decoded === 0`). ### Debug Log Location `/tmp/oc-ls-stats-debug.log` — TUI plugin writes here because `console.log` in TUI doesn't reach main OpenCode log. diff --git a/package-lock.json b/package-lock.json index 91be0ef..8e7e4ee 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "oc-tps", - "version": "0.0.20", + "version": "0.0.66", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "oc-tps", - "version": "0.0.20", + "version": "0.0.66", "engines": { "opencode": ">=1.3.14" }, diff --git a/package.json b/package.json index 0b64a8d..921883f 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/package.json", "name": "@troed/oc-ls-stats", - "version": "0.0.57", + "version": "0.0.66", "type": "module", "exports": { "./tui": { diff --git a/tui.tsx b/tui.tsx index fbb7c44..9b13813 100644 --- a/tui.tsx +++ b/tui.tsx @@ -19,8 +19,6 @@ type StreamSample = { } const STREAM_WINDOW_MS = 5_000 -const LIVE_STALE_MS = 1_500 -const SINGLE_SAMPLE_MS = 1_000 const SLOT_POLL_MS = 500 type MessageTiming = { sessionID: string @@ -28,32 +26,22 @@ type MessageTiming = { firstResponseAt?: number firstTokenAt?: number lastTokenAt?: number - lastToolCallAt?: number -} - -type SessionAverage = { - totalTokens: number - totalDurationMs: number - totalTtftMs: number - messageCount: number } type TrackerState = { streamSamplesBySession: Record messageTimingByID: Record - sessionAverageByID: Record lastPrefillRate: number lastGeneratedTps: number isPrefilling: boolean prefillSlotId: number | null prefillCapturedTokens: number | null prefillStartAt: number - prefillTotalTokens: number - prefillProcessedTokens: number isGenerating: boolean generateSlotId: number | null generatePrevNd: number generateStartAt: number + prevNdBySlot: Record } function estimateStreamTokens(delta: string) { @@ -188,48 +176,6 @@ function formatTps(value: number) { return `${Math.round(value)}` } -function formatAvg(value: number) { - if (!Number.isFinite(value) || value <= 0) return undefined - if (value >= 100) return `${Math.round(value)}` - return `${value.toFixed(1)}` -} - -function formatPercent(value: number) { - if (!Number.isFinite(value) || value < 0) return undefined - if (value >= 100) return "100" - if (value >= 10) return `${Math.round(value)}` - return `${value.toFixed(1)}` -} - -function pad4(s: string | undefined) { - if (!s) return "----" - return s.padStart(4, " ") -} - -function pad3(s: string | undefined) { - if (!s) return "---" - return s.padStart(3, " ") -} - -function activeDurationMs(samples: StreamSample[], tailAt?: number) { - if (samples.length === 0) return 0 - if (samples.length === 1) { - const tailDuration = tailAt ? Math.max(0, tailAt - samples[0].at) : SINGLE_SAMPLE_MS - return Math.min(Math.max(tailDuration, 250), SINGLE_SAMPLE_MS) - } - - let duration = 0 - for (let i = 1; i < samples.length; i++) { - duration += Math.max(0, samples[i].at - samples[i - 1].at) - } - - if (tailAt) { - duration += Math.max(0, tailAt - samples[samples.length - 1].at) - } - - return Math.max(duration, SINGLE_SAMPLE_MS) -} - function SessionPromptRight(props: { api: Parameters[0] sessionID: string @@ -253,22 +199,16 @@ function SessionPromptRight(props: { return rate }) - const prefillProgress = createMemo(() => { - props.version() - if (!props.tracker.isPrefilling) return undefined - const total = props.tracker.prefillTotalTokens - const processed = props.tracker.prefillProcessedTokens - if (total <= 0) return undefined - return (processed / total) * 100 - }) - const text = createMemo(() => { const pp = prefillRate() const tps = liveTps() - const progress = prefillProgress() - const ppLabel = pp ? `PP ${pad4(formatPps(pp))} tps (${pad3(formatPercent(progress))}%)` : `PP ${pad4(undefined)} tps (${pad3(undefined)}%)` - const tpsLabel = tps ? `GEN ${pad4(formatTps(tps))} tps` : `GEN ${pad4(undefined)} tps` - return `${ppLabel} | ${tpsLabel}` + if (props.tracker.isPrefilling && pp) { + return `${formatPps(pp)} tps (PP)` + } + if (props.tracker.isGenerating && tps) { + return `${formatTps(tps)} tps (TG)` + } + return `- tps (TG)` }) return <>{text() ? {text()} : null} @@ -279,19 +219,17 @@ const tui: TuiPlugin = async (api) => { const tracker: TrackerState = { streamSamplesBySession: {}, messageTimingByID: {}, - sessionAverageByID: {}, lastPrefillRate: 0, lastGeneratedTps: 0, isPrefilling: false, prefillSlotId: null, prefillCapturedTokens: null, prefillStartAt: 0, - prefillTotalTokens: 0, - prefillProcessedTokens: 0, isGenerating: false, generateSlotId: null, generatePrevNd: 0, generateStartAt: 0, + prevNdBySlot: {}, } const [version, setVersion] = createSignal(0) const [clock, setClock] = createSignal(Date.now()) @@ -360,55 +298,80 @@ const tui: TuiPlugin = async (api) => { let anyGenerating = false let genEnded = false + const prevProcessingSlotIds = new Set(Object.keys(tracker.prevNdBySlot).map(Number)) + const currProcessingSlotIds = new Set() + for (const baseUrl of llamaServerUrls) { const slots = await fetchSlots(baseUrl, model) const slotList = Array.isArray(slots) ? slots : slots ? Object.values(slots) : [] debug(`poll baseUrl=${baseUrl} slots=${JSON.stringify(slotList)}`) + for (const slot of slotList) { + if (slot?.is_processing) { + currProcessingSlotIds.add(slot.id) + } + } + + for (const slotId of prevProcessingSlotIds) { + if (!currProcessingSlotIds.has(slotId)) { + delete tracker.prevNdBySlot[slotId] + } + } + let prefillSlot: any = null let generateSlot: any = null for (const slot of slotList) { if (!slot?.is_processing) continue - if (slot.next_token?.[0]?.n_remain === -1) { + const slotId = slot.id + const nd = slot.next_token?.[0]?.n_decoded ?? 0 + const entry = tracker.prevNdBySlot[slotId] + const baseline = entry?.baseline ?? nd + const hasIncreased = entry?.hasIncreased ?? false + + if (!hasIncreased && nd <= baseline) { prefillSlot = slot break - } - if (slot.next_token?.[0]?.n_decoded > 0) { + } else if (nd > baseline) { + tracker.prevNdBySlot[slotId] = { baseline, hasIncreased: true } generateSlot = slot } } + for (const slot of slotList) { + if (slot?.is_processing) { + const slotId = slot.id + const nd = slot.next_token?.[0]?.n_decoded ?? 0 + const entry = tracker.prevNdBySlot[slotId] + if (entry === undefined) { + tracker.prevNdBySlot[slotId] = { baseline: nd, hasIncreased: false } + } + } + } + if (prefillSlot) { anyPrefilling = true const slotId = prefillSlot.id const now = Date.now() const nt = prefillSlot.n_prompt_tokens ?? 0 - const npp = prefillSlot.n_prompt_tokens_processed ?? 0 - - tracker.prefillTotalTokens = nt - tracker.prefillProcessedTokens = npp if (tracker.prefillSlotId === slotId) { - if (tracker.prefillCapturedTokens !== null) { - const dt = (now - tracker.prefillStartAt) / 1000 - const delta = nt - tracker.prefillCapturedTokens - if (dt > 0 && delta > 0) { - tracker.lastPrefillRate = delta / dt - } + const dt = (now - tracker.prefillStartAt) / 1000 + const delta = nt - tracker.prefillCapturedTokens! + if (dt > 0 && delta > 0) { + tracker.lastPrefillRate = delta / dt } } else { tracker.prefillSlotId = slotId tracker.prefillCapturedTokens = nt tracker.prefillStartAt = now tracker.lastPrefillRate = 0 + debug(`prefill new slot id=${slotId} nt=${nt}`) } } else { if (tracker.isPrefilling) { tracker.prefillSlotId = null tracker.prefillCapturedTokens = null tracker.prefillStartAt = 0 - tracker.prefillTotalTokens = 0 - tracker.prefillProcessedTokens = 0 } } @@ -419,23 +382,14 @@ const tui: TuiPlugin = async (api) => { const nd = generateSlot.next_token?.[0]?.n_decoded ?? 0 if (tracker.generateSlotId === slotId) { - if (nd > tracker.generatePrevNd) { - const dt = (now - tracker.generateStartAt) / 1000 - const delta = nd - tracker.generatePrevNd - if (dt > 0 && delta > 0) { - tracker.lastGeneratedTps = delta / dt - debug(`gen slot=${slotId} nd=${nd} prev=${tracker.generatePrevNd} delta=${delta} dt=${dt.toFixed(2)} tps=${tracker.lastGeneratedTps.toFixed(1)}`) - } - tracker.generatePrevNd = nd - tracker.generateStartAt = now - } else if (nd < tracker.generatePrevNd) { - tracker.generatePrevNd = nd - tracker.generateStartAt = now - tracker.lastGeneratedTps = 0 - debug(`gen slot=${slotId} nd=${nd} prev=${tracker.generatePrevNd} counter reset`) - } else { - debug(`gen slot=${slotId} nd=${nd} prev=${tracker.generatePrevNd} stalled`) + const dt = (now - tracker.generateStartAt) / 1000 + const delta = nd - tracker.generatePrevNd + if (dt > 0 && delta > 0) { + tracker.lastGeneratedTps = delta / dt + debug(`gen slot=${slotId} nd=${nd} prev=${tracker.generatePrevNd} delta=${delta} dt=${dt.toFixed(2)} tps=${tracker.lastGeneratedTps.toFixed(1)}`) } + tracker.generatePrevNd = nd + tracker.generateStartAt = now } else { tracker.generateSlotId = slotId tracker.generatePrevNd = nd @@ -486,36 +440,11 @@ const tui: TuiPlugin = async (api) => { firstResponseAt: existing?.firstResponseAt, firstTokenAt: existing?.firstTokenAt, lastTokenAt: existing?.lastTokenAt, - lastToolCallAt: existing?.lastToolCallAt, } bump() return } - const timing = tracker.messageTimingByID[evt.properties.info.id] - if (timing?.sessionID === evt.properties.sessionID && typeof timing.firstResponseAt === "number") { - const totalTokens = evt.properties.info.tokens.output + evt.properties.info.tokens.reasoning - const endAt = - evt.properties.info.finish === "tool-calls" - ? timing.lastToolCallAt - : evt.properties.info.time.completed - const durationMs = typeof endAt === "number" ? Math.max(endAt - timing.firstResponseAt, 1) : undefined - const ttftMs = Math.max(timing.firstResponseAt - timing.requestStartAt, 0) - if (totalTokens > 0 && durationMs) { - const totals = tracker.sessionAverageByID[evt.properties.sessionID] ?? { - totalTokens: 0, - totalDurationMs: 0, - totalTtftMs: 0, - messageCount: 0, - } - tracker.sessionAverageByID[evt.properties.sessionID] = { - totalTokens: totals.totalTokens + totalTokens, - totalDurationMs: totals.totalDurationMs + durationMs, - totalTtftMs: totals.totalTtftMs + ttftMs, - messageCount: totals.messageCount + 1, - } - } - } delete tracker.messageTimingByID[evt.properties.info.id] pruneSamples(evt.properties.info.time.completed) bump() @@ -541,10 +470,6 @@ const tui: TuiPlugin = async (api) => { return } if (evt.properties.part.state.status !== "running") return - tracker.messageTimingByID[evt.properties.part.messageID] = { - ...timing, - lastToolCallAt: evt.properties.part.state.time.start, - } bump() })