From 04baa5b053a76fea47ba7d2ce99c8ac23510acb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Troed=20S=C3=A5ngberg?= Date: Wed, 8 Jul 2026 13:10:02 +0200 Subject: [PATCH] =?UTF-8?q?chore:=20v1.0.5=20=E2=80=94=20detect=20PP/TG=20?= =?UTF-8?q?from=20n=5Fdecoded=20vs=20baseline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 111 +++++++++++ package.json | 20 ++ tui.tsx | 525 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 656 insertions(+) create mode 100644 README.md create mode 100644 package.json create mode 100644 tui.tsx diff --git a/README.md b/README.md new file mode 100644 index 0000000..7eb13b4 --- /dev/null +++ b/README.md @@ -0,0 +1,111 @@ +# oc-ls-stats + +A TUI plugin for OpenCode that displays live prefill rate (PP) and generation rate (TG) from a llama.cpp-based llama-server. + +There are many plugins to show tokens per second, but the reason for this one is that when running a local model I often found myself not knowing what the server was currently doing, which meant constantly switching over to a console where I could see its output. Especially during prefill/prompt processing which can take more than a minute with no feedback in the UI from other plugins I tried. + +Another motivation was to display the data of interest, but in a non-intrusive way with no UI elements jumping around. This plugin thus only shows a single numeric value, tokens per second, with an indicator as to whether the model is currently doing prompt processing or inference (token generation). + +I'm using the llama-server /slots endpoint to get the needed data, which means if you connect opencode to another provider the plugin will just display "-" since it's not getting any data to display. + +Note: As explained in further detail below the data that's displayed has to be deduced from llama-server's output. Sometimes the plugin might display PP for prompt processing while in reality the model is doing TG. If additional developments are made to the llama-server output data the plugin might be able to discern between them in a better way, but I think is as good as it gets for now. + +I made this for my own usage. If you find it useful as well I'm just happy. + +/Troed + +_thanks to Tarquinen for their [oc-tps](https://github.com/Tarquinen/oc-tps), which I used as a base although I guess most of the code has now been replaced_ + +## Display Format + +The plugin renders a single line in the session prompt right slot: + +``` +1247 tps (PP) -- during prefill + 25 tps (TG) -- during generation +- tps (TG) -- idle +``` + +## Detection and Calculation + +### Server Discovery + +The plugin discovers the llama-server URL by reading the OpenCode configuration file (parsed as JSONC) and extracting `baseURL`/`base_url` fields from provider options that contain "localhost". Falls back to `http://localhost:8080`. + +### Slot Polling + +Every 500ms, the plugin polls `GET /slots?model=` on each discovered server. The model parameter is required by the `/slots` endpoint. If the model cannot be discovered from the current route's session, the plugin skips polling. + +### State Classification + +Each slot is classified as prefill or generation based on the `n_decoded` counter in `next_token[0]`. The plugin tracks a per-slot baseline value: + +1. When a slot first appears as processing, the current `n_decoded` is recorded as the baseline with `hasIncreased = false`. +2. If `n_decoded <= baseline` and `hasIncreased` is false, the slot is classified as prefilling. +3. If `n_decoded > baseline`, `hasIncreased` is set to true and the slot is classified as generating. + +This approach handles the case where `n_decoded` drops when a new request starts on a reused slot, and prevents generation stalls (where `n_decoded` plateaus) from being misclassified as prefill. + +When no slots are processing, all tracked state for those slots is cleared. + +### Prefill Rate (PP) + +During prefill, the plugin calculates the instantaneous prompt processing rate: + +1. On first detection of a prefill slot, the current `n_prompt_tokens` is captured as the baseline. +2. On subsequent polls, the delta in `n_prompt_tokens` is divided by the elapsed time in seconds. +3. The rate is updated only when both `dt > 0` and `delta > 0`. + +The per-slot `n_prompt_tokens` field is used instead of the global `llamacpp:prompt_tokens_total` from `/metrics` because the global counter includes tokens from all slots, producing inflated values when multiple slots are active simultaneously. + +### Generation Rate (TG) + +During generation, the plugin calculates the instantaneous token generation rate: + +1. On first detection of a generation slot, the current `n_decoded` is captured as the baseline. +2. On subsequent polls (same slot ID), the delta in `n_decoded` is divided by the elapsed time in seconds. +3. The rate is updated only when both `dt > 0` and `delta > 0`. + +Slot reuse is tracked via `generateSlotId` to detect when a new generation starts on a different slot. + +## Limitations + +### Progress Percentage + +The plugin cannot display prefill progress percentage. The `/slots` endpoint returns `n_prompt_tokens` (current prompt size) and `n_prompt_tokens_processed` (tokens processed), but not the final prompt size (`task->n_tokens()` from llama.cpp). Progress requires the ratio `n_prompt_tokens_processed / task->n_tokens()`. + +### What Would Improve Compatibility + +The following changes to the `/slots` endpoint would improve the plugin's functionality: + +1. **Expose final prompt size**: Add `n_prompt_tokens_total` (or `n_tokens`) to the `/slots` output, representing `task->n_tokens()` from llama.cpp. This would enable prefill progress percentage calculation as `(n_prompt_tokens_processed / n_prompt_tokens_total) * 100`. + +2. **Per-slot metrics endpoints**: Currently, the `/metrics` endpoint provides only global counters (`llamacpp:prompt_tokens_total`, `llamacpp:prompt_tokens_seconds`). Per-slot metrics would allow independent rate tracking without relying on slot state classification. + +3. **Slot transition notifications**: The plugin polls every 500ms to detect state transitions. A WebSocket or SSE-based notification system for slot state changes would reduce polling overhead and improve detection latency. + +4. **Stall detection**: When generation stalls (e.g., due to context window limits), `n_decoded` remains constant while `n_remain` stops decreasing. The plugin detects this via zero delta but has no way to distinguish a stall from normal generation. An explicit `stalled` flag in the slot output would help. + +5. **Model-agnostic slot data**: The `/slots` endpoint requires a model parameter. Returning all slots without model filtering, or supporting `*` as a wildcard, would simplify discovery when multiple models are loaded. + +## Installation + +```bash +opencode plugin @troed/oc-ls-stats@latest --global +``` + +Requires `opencode` `1.3.14` or newer. + +TUI plugins are loaded from `~/.config/opencode/tui.json`: + +```json +{"plugin": ["@troed/oc-ls-stats@latest"]} +``` + +## Debug Logging + +Debug logging is controlled by the `DEBUG_ENABLED` constant in `tui.tsx`. When enabled, full slot state data is written to `/tmp/oc-ls-stats-debug.log` on every poll. + +## License + +Creative Commons Zero (CC0 1.0 Universal) diff --git a/package.json b/package.json new file mode 100644 index 0000000..6e2e424 --- /dev/null +++ b/package.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://json.schemastore.org/package.json", + "name": "@troed/oc-ls-stats", + "version": "1.0.5", + "type": "module", + "exports": { + "./tui": { + "import": "./tui.tsx" + } + }, + "engines": { + "opencode": ">=1.3.14" + }, + "peerDependencies": { + "@opencode-ai/plugin": "*", + "@opentui/core": "*", + "@opentui/solid": "*", + "solid-js": "*" + } +} diff --git a/tui.tsx b/tui.tsx new file mode 100644 index 0000000..14cffe9 --- /dev/null +++ b/tui.tsx @@ -0,0 +1,525 @@ +/** @jsxImportSource @opentui/solid */ +import type { TuiPlugin, TuiPluginModule } from "@opencode-ai/plugin/tui" +import { createMemo, createSignal } from "solid-js" +import { appendFileSync } from "fs" + +const DEBUG_ENABLED = false +const DEBUG_LOG = "/tmp/oc-ls-stats-debug.log" +function debug(...args: unknown[]) { + if (!DEBUG_ENABLED) return + try { + const line = `${args.map(a => typeof a === "object" ? JSON.stringify(a) : String(a)).join(" ")}\n` + appendFileSync(DEBUG_LOG, line) + } catch {} +} + +type StreamSample = { + at: number + tokens: number +} + +const STREAM_WINDOW_MS = 5_000 +const SLOT_POLL_MS = 500 +type MessageTiming = { + sessionID: string + requestStartAt: number + firstResponseAt?: number + firstTokenAt?: number + lastTokenAt?: number +} + +type TrackerState = { + streamSamplesBySession: Record + messageTimingByID: Record + lastPrefillRate: number + lastGeneratedTps: number + isPrefilling: boolean + prefillSlotId: number | null + prefillCapturedTokens: number | null + prefillStartAt: number + isGenerating: boolean + generateSlotId: number | null + generatePrevNd: number + generateStartAt: number + prevNdBySlot: Record + failure: string | null +} + +function estimateStreamTokens(delta: string) { + return Math.max(1, Math.ceil(Buffer.byteLength(delta, "utf8") / 5)) +} + +function parseJSONC(text: string): unknown { + let result = "" + let inString = false + let escape = false + let i = 0 + + while (i < text.length) { + const ch = text[i] + + if (escape) { + result += ch + escape = false + i++ + continue + } + + if (ch === "\\") { + result += ch + escape = true + i++ + continue + } + + if (ch === '"') { + result += ch + inString = !inString + i++ + continue + } + + if (inString) { + result += ch + i++ + continue + } + + if (ch === "/" && text[i + 1] === "/") { + while (i < text.length && text[i] !== "\n") i++ + continue + } + + if (ch === "/" && text[i + 1] === "*") { + i += 2 + while (i < text.length - 1 && !(text[i] === "*" && text[i + 1] === "/")) i++ + i += 2 + result += " " + continue + } + + if (ch === ",") { + const next = text.indexOf(/[\s}\]]/, i + 1) + if (next !== -1 && /[\s}\]]/.test(text[next])) { + i++ + continue + } + } + + result += ch + i++ + } + + return JSON.parse(result) +} + +function stripBaseUrlPath(baseURL: string): string { + try { + const url = new URL(baseURL) + return `${url.origin}` + } catch { + return baseURL.replace(/\/v1(\/.*)?$/, "").replace(/\/$/, "") || baseURL + } +} + +function extractProviderUrls(config: unknown): string[] { + if (!config || typeof config !== "object") return [] + const obj = config as Record + const provider = obj.provider + if (!provider || typeof provider !== "object") return [] + + const urls: string[] = [] + for (const [_key, val] of Object.entries(provider as Record)) { + if (val && typeof val === "object") { + const opts = (val as Record).options + if (opts && typeof opts === "object") { + const baseURL = (opts as Record).baseURL || (opts as Record).base_url + if (typeof baseURL === "string" && baseURL.includes("localhost")) { + urls.push(stripBaseUrlPath(baseURL)) + } + } + } + } + return urls +} + +async function loadConfigUrls(api: Parameters[0]): Promise { + try { + const configPath = api.state.path.config + const resp = await fetch(`file://${configPath}`) + if (!resp.ok) return [] + const text = await resp.text() + const config = parseJSONC(text) + return extractProviderUrls(config) + } catch { + return [] + } +} + +async function fetchSlots(baseUrl: string, model?: string): Promise { + try { + const url = model ? `${baseUrl}/slots?model=${encodeURIComponent(model)}` : `${baseUrl}/slots` + const resp = await fetch(url) + if (!resp.ok) return null + return await resp.json() + } catch { + return null + } +} + +function formatPps(value: number) { + if (!Number.isFinite(value) || value <= 0) return undefined + return `${Math.round(value)}` +} + +function formatTps(value: number) { + if (!Number.isFinite(value) || value <= 0) return undefined + return `${Math.round(value)}` +} + +function SessionPromptRight(props: { + api: Parameters[0] + sessionID: string + tracker: TrackerState + version: () => number + clock: () => number +}) { + const liveTps = createMemo(() => { + props.version() + props.clock() + const rate = props.tracker.lastGeneratedTps + if (!Number.isFinite(rate) || rate <= 0) return undefined + return rate + }) + + const prefillRate = createMemo(() => { + props.version() + if (!props.tracker.isPrefilling) return undefined + const rate = props.tracker.lastPrefillRate + if (!Number.isFinite(rate) || rate <= 0) return undefined + return rate + }) + + const text = createMemo(() => { + if (props.tracker.failure) return props.tracker.failure + const pp = prefillRate() + const tps = liveTps() + if (props.tracker.isPrefilling && pp) { + return `${formatPps(pp)} tps (PP)` + } + if (props.tracker.isGenerating && tps) { + return `${formatTps(tps)} tps (TG)` + } + return `- tps (TG)` + }) + + return <>{text() ? {text()} : null} +} + +const tui: TuiPlugin = async (api) => { + console.log("[oc-ls-stats] TUI plugin loaded!") + const tracker: TrackerState = { + streamSamplesBySession: {}, + messageTimingByID: {}, + lastPrefillRate: 0, + lastGeneratedTps: 0, + isPrefilling: false, + prefillSlotId: null, + prefillCapturedTokens: null, + prefillStartAt: 0, + isGenerating: false, + generateSlotId: null, + generatePrevNd: 0, + generateStartAt: 0, + prevNdBySlot: {}, + failure: null, + } + const [version, setVersion] = createSignal(0) + const [clock, setClock] = createSignal(Date.now()) + + const bump = () => setVersion((value) => value + 1) + + const pruneSamples = (now = Date.now()) => { + let changed = false + + for (const [sessionID, samples] of Object.entries(tracker.streamSamplesBySession)) { + const next = samples.filter((sample) => now - sample.at <= STREAM_WINDOW_MS) + if (next.length !== samples.length) { + changed = true + if (next.length > 0) tracker.streamSamplesBySession[sessionID] = next + else delete tracker.streamSamplesBySession[sessionID] + } + } + + if (changed) bump() + } + + const clearLiveSamples = (sessionID: string) => { + if (!tracker.streamSamplesBySession[sessionID]?.length) return + delete tracker.streamSamplesBySession[sessionID] + bump() + } + + const appendSample = (sessionID: string, messageID: string, sample: StreamSample) => { + const now = sample.at + tracker.streamSamplesBySession[sessionID] = [ + ...(tracker.streamSamplesBySession[sessionID] ?? []).filter((item) => now - item.at <= STREAM_WINDOW_MS), + sample, + ] + const timing = tracker.messageTimingByID[messageID] + if (timing) { + tracker.messageTimingByID[messageID] = timing.firstTokenAt + ? { ...timing, lastTokenAt: now } + : { + ...timing, + firstResponseAt: timing.firstResponseAt ?? now, + firstTokenAt: now, + lastTokenAt: now, + } + } + bump() + } + + let llamaServerUrls: string[] = ["http://localhost:8080"] + let llamaServerModel: string | undefined + + const pollMetrics = async () => { + if (!llamaServerUrls.length) return + + let model = llamaServerModel + if (!model) { + const routeSessionID = api.route.current.params?.sessionID + if (routeSessionID) { + const session = api.state.session.get(routeSessionID) + if (session?.model?.id) { + model = session.model.id + } + } + } + + let anyPrefilling = false + let anyGenerating = false + let genEnded = false + + const prevProcessingSlotIds = new Set(Object.keys(tracker.prevNdBySlot).map(Number)) + const currProcessingSlotIds = new Set() + + for (const baseUrl of llamaServerUrls) { + const slots = await fetchSlots(baseUrl, model) + const slotList = Array.isArray(slots) ? slots : slots ? Object.values(slots) : [] + if (slotList.length === 0) { + tracker.failure = "n/a" + bump() + continue + } + tracker.failure = null + + debug(`poll baseUrl=${baseUrl} slots=${JSON.stringify(slotList)}`) + + for (const slot of slotList) { + if (slot?.is_processing) { + currProcessingSlotIds.add(slot.id) + } + } + + for (const slotId of prevProcessingSlotIds) { + if (!currProcessingSlotIds.has(slotId)) { + delete tracker.prevNdBySlot[slotId] + } + } + + let prefillSlot: any = null + let generateSlot: any = null + for (const slot of slotList) { + if (!slot?.is_processing) continue + const slotId = slot.id + const nd = slot.next_token?.[0]?.n_decoded ?? 0 + const entry = tracker.prevNdBySlot[slotId] + const baseline = entry?.baseline ?? nd + const hasIncreased = entry?.hasIncreased ?? false + + if (!hasIncreased && nd <= baseline) { + prefillSlot = slot + break + } else if (nd > baseline) { + tracker.prevNdBySlot[slotId] = { baseline, hasIncreased: true } + generateSlot = slot + } + } + + for (const slot of slotList) { + if (slot?.is_processing) { + const slotId = slot.id + const nd = slot.next_token?.[0]?.n_decoded ?? 0 + const entry = tracker.prevNdBySlot[slotId] + if (entry === undefined) { + tracker.prevNdBySlot[slotId] = { baseline: nd, hasIncreased: false } + } + } + } + + if (prefillSlot) { + anyPrefilling = true + const slotId = prefillSlot.id + const now = Date.now() + const nt = prefillSlot.n_prompt_tokens ?? 0 + + if (tracker.prefillSlotId === slotId) { + const dt = (now - tracker.prefillStartAt) / 1000 + const delta = nt - tracker.prefillCapturedTokens! + if (dt > 0 && delta > 0) { + tracker.lastPrefillRate = delta / dt + } + } else { + tracker.prefillSlotId = slotId + tracker.prefillCapturedTokens = nt + tracker.prefillStartAt = now + tracker.lastPrefillRate = 0 + debug(`prefill new slot id=${slotId} nt=${nt}`) + } + } else { + if (tracker.isPrefilling) { + tracker.prefillSlotId = null + tracker.prefillCapturedTokens = null + tracker.prefillStartAt = 0 + } + } + + if (generateSlot) { + anyGenerating = true + const slotId = generateSlot.id + const now = Date.now() + const nd = generateSlot.next_token?.[0]?.n_decoded ?? 0 + + if (tracker.generateSlotId === slotId) { + const dt = (now - tracker.generateStartAt) / 1000 + const delta = nd - tracker.generatePrevNd + if (dt > 0 && delta > 0) { + tracker.lastGeneratedTps = delta / dt + debug(`gen slot=${slotId} nd=${nd} prev=${tracker.generatePrevNd} delta=${delta} dt=${dt.toFixed(2)} tps=${tracker.lastGeneratedTps.toFixed(1)}`) + } + tracker.generatePrevNd = nd + tracker.generateStartAt = now + } else { + tracker.generateSlotId = slotId + tracker.generatePrevNd = nd + tracker.generateStartAt = now + tracker.lastGeneratedTps = 0 + debug(`gen new slot id=${slotId} nd=${nd}`) + } + } else { + if (tracker.isGenerating) { + tracker.generateSlotId = null + tracker.generatePrevNd = 0 + tracker.generateStartAt = 0 + tracker.lastGeneratedTps = 0 + genEnded = true + debug(`gen no slot found ending`) + } + } + + tracker.isPrefilling = anyPrefilling + tracker.isGenerating = anyGenerating + } + + if (tracker.lastPrefillRate > 0 || tracker.lastGeneratedTps > 0 || genEnded) { + bump() + } + } + + const onDelta = api.event.on("message.part.delta", (evt) => { + if (evt.properties.field !== "text") return + const parts = api.state.part(evt.properties.messageID) + const part = parts.find((item) => item.id === evt.properties.partID) + if (!part) return + if (part.type !== "text" && part.type !== "reasoning") return + appendSample(evt.properties.sessionID, evt.properties.messageID, { + at: Date.now(), + tokens: estimateStreamTokens(evt.properties.delta), + }) + }) + + const onMessage = api.event.on("message.updated", (evt) => { + if (evt.properties.info.role !== "assistant") return + + if (!evt.properties.info.time.completed) { + const existing = tracker.messageTimingByID[evt.properties.info.id] + tracker.messageTimingByID[evt.properties.info.id] = { + sessionID: evt.properties.sessionID, + requestStartAt: evt.properties.info.time.created, + firstResponseAt: existing?.firstResponseAt, + firstTokenAt: existing?.firstTokenAt, + lastTokenAt: existing?.lastTokenAt, + } + bump() + return + } + + delete tracker.messageTimingByID[evt.properties.info.id] + pruneSamples(evt.properties.info.time.completed) + bump() + }) + + const onPart = api.event.on("message.part.updated", (evt) => { + if (evt.properties.part.type !== "tool") return + if ( + evt.properties.part.state.status === "running" || + evt.properties.part.state.status === "completed" || + evt.properties.part.state.status === "error" + ) { + clearLiveSamples(evt.properties.sessionID) + } + const timing = tracker.messageTimingByID[evt.properties.part.messageID] + if (!timing) return + if (evt.properties.part.state.status === "pending") { + tracker.messageTimingByID[evt.properties.part.messageID] = { + ...timing, + firstResponseAt: timing.firstResponseAt ?? evt.properties.time, + } + bump() + return + } + if (evt.properties.part.state.status !== "running") return + bump() + }) + + const timer = setInterval(() => { + setClock(Date.now()) + pruneSamples() + }, 1000) + + const slotTimer = setInterval(async () => { + await pollMetrics() + }, SLOT_POLL_MS) + + api.lifecycle.onDispose(() => { + onDelta() + onMessage() + onPart() + clearInterval(timer) + clearInterval(slotTimer) + }) + + try { + if (!llamaServerUrls.length) { + llamaServerUrls = await loadConfigUrls(api) + } + } catch (e) { + console.error("[oc-ls-stats] loadConfigUrls error:", e) + } + + api.slots.register({ + slots: { + session_prompt_right(_ctx, value) { + return + }, + }, + }) +} + +const plugin: TuiPluginModule & { id: string } = { + id: "@troed/oc-ls-stats", + tui, +} + +export default plugin