diff --git a/.gitignore b/.gitignore index ebac6b9..98307ba 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ node_modules/ .npmrc .opencode/ +.vibe/ +assets/ +*.tgz diff --git a/.npmignore b/.npmignore new file mode 100644 index 0000000..75cb0d5 --- /dev/null +++ b/.npmignore @@ -0,0 +1,3 @@ +.vibe/ +assets/ +*.tgz diff --git a/AGENTS.md b/AGENTS.md index f91eb5b..87b0ae5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -56,21 +56,27 @@ OpenCode's `file://` path loader does not resolve `package.json` exports field f The plugin polls llama-server's `GET /slots` endpoint to track prompt processing (prefill) timing. ### URL Discovery -- First checks `tui.json` plugin options for `url` field - Falls back to reading OpenCode config from `api.state.path.config` - Parses JSONC (strips `//`, `/* */` comments and trailing commas) - Extracts `baseURL`/`base_url` from `provider.*.options` where value contains "localhost" - Strips `/v1` path suffix to get the llama-server base URL +- Falls back to `http://localhost:8080` default URL ### Slot Polling -- Polls `GET /slots` every 500ms (only when session is busy) -- Tracks slot state transitions: `prompt_processing` → non-prompt_processing -- Extracts `t_prompt_processing` (ms) and `n_prompt_tokens` from final state -- Falls back to wall-clock timing if `t_prompt_processing` unavailable +- Polls `GET /slots?model=...` every 2 seconds (`SLOT_POLL_MS = 2000`) +- Prefill detection: `is_processing === true` AND `next_token[0].n_remain === -1` +- Model parameter is required by the `/slots` endpoint +- Falls back to `http://localhost:8080` default URL ### Display Format -`TPS {live} | AVG {avg} | TTFT {ttft} | P {duration} @ {rate}` -Example: `TPS 42.3 TPS | AVG 38.1 TPS | TTFT 1.2s | P 850ms @ 118PPS` +`PP {pp} | TPS {tps} | AVG {avg}` +Example: `PP 1247 | TPS 25 | AVG 25` + +### Display Rules +- PP: integer only, no suffix, 4-char fixed-width padded +- TPS: integer only, 4-char fixed-width padded +- AVG: one decimal when < 100, integer when >= 100, 4-char fixed-width padded +- "----" shown when no value available ### Limitations - Slot state tracked globally (not per-session) — slot IDs don't map to OpenCode sessions @@ -83,7 +89,7 @@ Example: `TPS 42.3 TPS | AVG 38.1 TPS | TTFT 1.2s | P 850ms @ 118PPS` - Package name: `oc-ls-stats` (llama-server stats, distinct from `oc-tps` on npmjs.org) - Auth: token in `~/.npmrc` - Git remotes: `origin` → Forgejo, `upstream` → GitHub -- To update: bump version in `package.json`, run `npm publish --registry https://git.sync.wtf/api/packages/troed/npm/`, restart OpenCode +- To update: bump version in `package.json`, run `npm publish --registry https://git.sync.wtf/api/packages/troed/npm/`, clear cache (`rm -rf ~/.cache/opencode/packages/@troed`), restart OpenCode ## Debugging: Plugin Not Loading / No Output @@ -157,3 +163,85 @@ return 0` (e.g. 31999) | `0` | stable (final) | +| Generation | `true` | `>0` (decreasing) | `>0` (incrementing) | stable (final) | + +**Key insight:** `n_remain === -1` is the definitive prefill indicator. `n_decoded === 0` alone is not sufficient — a slot can have `n_decoded === 0` during generation start (transition phase). + +### PP Calculation Method + +PP is calculated from the per-slot `n_prompt_tokens` field during prefill: + +1. Detect prefill: slot with `is_processing === true` AND `next_token[0].n_remain === -1` +2. On first detection: capture `slot.id` and `slot.n_prompt_tokens` as baseline +3. On subsequent polls: `delta = current_n_prompt_tokens - baseline_tokens`, `dt = (now - startAt) / 1000`, `PP = delta / dt` +4. Reset when no slot has `n_remain === -1` (prefill ended) + +### Why NOT `llamacpp:prompt_tokens_total` from `/metrics` + +`llamacpp:prompt_tokens_total` is a **global counter across all slots**, not per-model. When multiple slots are active simultaneously (e.g., slot A prefilling while slot B generating), the delta includes tokens from all slots, producing wildly inflated PP values. + +**Example from debug data (`test-debug-log.jsonl`):** +- Line 15: slot 3 prefilling, `prompt_tokens_total` delta = 49 tokens in 1815ms → PP = 27 (correct but tiny delta) +- Line 31: slot 2 prefilling, slot 3 still generating, `prompt_tokens_total` delta = 13793 tokens in 668ms → PP = 20,648 (wrong — includes slot 3 tokens) +- Actual server console showed ~1200-1300 tokens/sec during prefill + +### Why NOT `llamacpp:prompt_tokens_seconds` from `/metrics` + +`llamacpp:prompt_tokens_seconds` is an **average since server start**, not an instantaneous rate. It smooths out all prefill events into a single running average, making it useless for displaying current prefill speed. + +### Slot Polling + +- Polls `GET /slots?model=...` every 2 seconds (`SLOT_POLL_MS = 2000`) +- Model parameter is required by the `/slots` endpoint (returns error without it) +- Model discovered from: `tui.json` options → route session → OpenCode config provider parsing +- Falls back to `http://localhost:8080` default URL + +### Display + +PP only shown during active prefill (`n_remain === -1`). Format: integer only, no suffix (e.g., "1247"). Fixed-width 4 chars, space-padded. Shows "----" when no value. + +## Debugging: PP Value Wrong + +### Symptom +PP shows 3634 when llama-server console shows ~1200-1300 tokens/sec during prefill. + +### Root Cause (fixed in v0.0.40) +Used `llamacpp:prompt_tokens_total` from `/metrics` endpoint which is a global counter across all slots. Delta between polls included tokens from other active slots. + +### Fix +Track `n_prompt_tokens` from the slot data itself (per-slot field) during prefill state (`n_remain === -1`). + +### Debug Log Location +`/tmp/oc-ls-stats-debug.log` — TUI plugin writes here because `console.log` in TUI doesn't reach main OpenCode log. + +### Debug Data +Sample debug log with slot states and metrics values: `test-debug-log.jsonl` diff --git a/TUIPLUGIN.md b/TUIPLUGIN.md new file mode 100644 index 0000000..b059181 --- /dev/null +++ b/TUIPLUGIN.md @@ -0,0 +1,120 @@ +# TUI Plugin Investigation - oc-ls-stats + +## Problem +The `@troed/oc-ls-stats` TUI plugin installs successfully via `opencode plugin` but produces no visible output in the OpenCode TUI. + +## Key Findings + +### 1. OpenCode Plugin Architecture +- **Server plugins**: Configured in `opencode.jsonc` under `plugin` array, shown in `/status` +- **TUI plugins**: Configured in `~/.config/opencode/tui.json` under `plugin` array, NOT shown in `/status` +- These are two completely separate plugin loaders + +### 2. TUI Plugin Installation +- Command: `opencode plugin "@latest" --global` +- Installs to: `~/.cache/opencode/packages/@latest/node_modules//` +- For scoped packages: `~/.cache/opencode/packages/@scope/package@latest/node_modules/@scope/package/` +- Config updated: `~/.config/opencode/tui.json` +- Peer deps installed to: `~/.config/opencode/node_modules/` + +### 3. TUI Plugin Loading +- TUI config loaded in a separate process from the main OpenCode instance +- Logs show only: `"loading tui config"` and `"applying tui config"` +- No plugin loading errors appear in logs (same as working oc-tps) +- Plugin loading happens silently - no success/error logs + +### 4. Registry Configuration +- `~/.npmrc`: + ``` + registry=https://registry.npmjs.org/ + //git.sync.wtf/api/packages/troed/npm/:_authToken=590922c3fe27e44cd5a216f14375cff575e52b5c + @troed:registry=https://git.sync.wtf/api/packages/troed/npm/ + ``` +- Public npm for `@opencode-ai/*` and other unscoped packages +- Forgejo registry for `@troed/*` scoped packages + +### 5. Package Structure (working vs non-working) +Both plugins have identical structure: +```json +{ + "name": "@troed/oc-ls-stats", + "version": "0.0.x", + "type": "module", + "exports": { "./tui": { "import": "./tui.tsx" } }, + "peerDependencies": { + "@opencode-ai/plugin": "*", + "@opentui/core": "*", + "@opentui/solid": "*", + "solid-js": "*" + } +} +``` + +Export format (identical): +```typescript +const plugin: TuiPluginModule & { id: string } = { + id: "oc-tps", + tui, +} +export default plugin +``` + +### 6. What We Tried (all failed) +- Pointing to file directly (`file:///path/to/tui.tsx`) - didn't work +- Adding `oc-plugin` field to package.json - didn't work +- Adding `main` field to package.json - didn't work +- Tuple format in tui.json - rejected by OpenCode +- Copying file to `~/.config/opencode/plugins/` - didn't work +- Publishing to Forgejo without scope - peer deps resolve from Forgejo (404) +- Publishing with scope (`@troed/`) - installs but no output +- Manual npm install to `~/.config/opencode/node_modules/` - no output +- Adding console.log - not visible (TUI runs in separate process) +- Adding early slot registration - no output +- Wrapping loadConfigUrls in try-catch - no output +- Simplified slot handler to static text - no output +- Multiple version bumps and reinstalls - no output + +### 7. Stale Cache Issues +- OpenCode caches plugins in `~/.cache/opencode/packages/` +- Stale entries from manual installs can persist +- Multiple stale directories created during debugging: + - `~/.cache/opencode/packages/oc-ls-stats@latest` (empty) + - `~/.cache/opencode/packages/@troed/oc-ls-stats:latest` (colon in name) + - `~/.cache/opencode/packages/troed/oc-ls-stats@latest` (missing @) +- Must manually clear cache before reinstalling: `rm -rf ~/.cache/opencode/packages/@troed/oc-ls-stats@latest` + +### 8. Background Dependency Install Error +- `~/.opencode/package.json` has `"@opencode-ai/plugin": "1.4.9"` as dependency +- OpenCode tries to install this from Forgejo (404) during startup +- This is a separate issue from TUI plugin loading +- The package is already installed in `~/.opencode/node_modules/` + +### 9. Working Reference: oc-tps +- Installed via: `opencode plugin oc-tps@latest --global` +- Cache location: `~/.cache/opencode/packages/oc-tps@latest/node_modules/oc-tps/` +- tui.json: `"oc-tps@latest"` (unscoped) +- Shows TPS output correctly +- NOT listed in `/status` (expected for TUI plugins) +- No log output (same as oc-ls-stats) + +### 10. Current State +- `@troed/oc-ls-stats@0.0.8` published to Forgejo +- Installed in cache with correct version +- tui.json: `["@troed/oc-ls-stats@latest"]` +- Peer deps installed in `~/.config/opencode/node_modules/` +- Slot registration added at start of tui function +- Static text output in slot handler +- **Still no visible output** + +## Conclusion +The plugin installs and appears to load (same log pattern as working oc-tps), but produces no output. The root cause is unknown. Possible causes: +1. OpenCode's TUI plugin loader has an issue with scoped packages +2. There's a runtime error that's not being logged +3. The slot registration is not being triggered for some reason +4. There's a difference in how the TUI process resolves modules for scoped vs unscoped packages + +## Next Steps +- Compare the exact module resolution path between oc-tps and oc-ls-stats +- Check if the TUI process can actually import the scoped package +- Verify the slot registration is being called (maybe via a side effect) +- Check if there's a TUI-specific debug mode or log output diff --git a/package.json b/package.json index a6c1a72..ffe7552 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/package.json", "name": "@troed/oc-ls-stats", - "version": "0.0.20", + "version": "0.0.40", "type": "module", "exports": { "./tui": { diff --git a/test-debug-log.jsonl b/test-debug-log.jsonl new file mode 100644 index 0000000..9bf634e --- /dev/null +++ b/test-debug-log.jsonl @@ -0,0 +1,46 @@ +{"metrics":{"predictedTps":24.9122,"promptTokensTotal":97244,"promptTokensSeconds":1111.56},"lastPromptTokensTotal":97244,"lastPromptTokensTime":1781383392976,"now":1781383395633} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":true,"next_token":{"has_next_token":true,"has_new_line":false,"n_remain":31999,"n_decoded":1}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9122,"promptTokensTotal":97559,"promptTokensSeconds":1101.94},"lastPromptTokensTotal":97244,"lastPromptTokensTime":1781383395633,"now":1781383396983} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":true,"next_token":{"has_next_token":true,"has_new_line":false,"n_remain":31969,"n_decoded":31}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9122,"promptTokensTotal":97559,"promptTokensSeconds":1101.94},"lastPromptTokensTotal":97559,"lastPromptTokensTime":1781383396983,"now":1781383399028} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":true,"next_token":{"has_next_token":true,"has_new_line":true,"n_remain":31921,"n_decoded":79}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9031,"promptTokensTotal":97559,"promptTokensSeconds":1101.94},"lastPromptTokensTotal":97559,"lastPromptTokensTime":1781383399028,"now":1781383401001} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31893,"n_decoded":107}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9031,"promptTokensTotal":97559,"promptTokensSeconds":1101.94},"lastPromptTokensTotal":97559,"lastPromptTokensTime":1781383401001,"now":1781383403001} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31893,"n_decoded":107}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9031,"promptTokensTotal":97559,"promptTokensSeconds":1101.94},"lastPromptTokensTotal":97559,"lastPromptTokensTime":1781383403001,"now":1781383405013} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31893,"n_decoded":107}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9031,"promptTokensTotal":97559,"promptTokensSeconds":1101.94},"lastPromptTokensTotal":97559,"lastPromptTokensTime":1781383405013,"now":1781383407016} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31893,"n_decoded":107}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9031,"promptTokensTotal":97559,"promptTokensSeconds":1101.94},"lastPromptTokensTotal":97559,"lastPromptTokensTime":1781383407016,"now":1781383409019} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31893,"n_decoded":107}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9031,"promptTokensTotal":97559,"promptTokensSeconds":1101.94},"lastPromptTokensTotal":97559,"lastPromptTokensTime":1781383409019,"now":1781383411028} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31893,"n_decoded":107}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9031,"promptTokensTotal":97559,"promptTokensSeconds":1101.94},"lastPromptTokensTotal":97559,"lastPromptTokensTime":1781383411029,"now":1781383413038} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31893,"n_decoded":107}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9031,"promptTokensTotal":97559,"promptTokensSeconds":1101.94},"lastPromptTokensTotal":97559,"lastPromptTokensTime":1781383413038,"now":1781383415040} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31893,"n_decoded":107}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9031,"promptTokensTotal":97559,"promptTokensSeconds":1101.94},"lastPromptTokensTotal":97559,"lastPromptTokensTime":1781383415041,"now":1781383417039} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31893,"n_decoded":107}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9031,"promptTokensTotal":97559,"promptTokensSeconds":1101.94},"lastPromptTokensTotal":97559,"lastPromptTokensTime":1781383417039,"now":1781383419039} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31893,"n_decoded":107}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9031,"promptTokensTotal":97559,"promptTokensSeconds":1101.94},"lastPromptTokensTotal":97559,"lastPromptTokensTime":1781383419039,"now":1781383421051} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31893,"n_decoded":107}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9031,"promptTokensTotal":97559,"promptTokensSeconds":1101.94},"lastPromptTokensTotal":97559,"lastPromptTokensTime":1781383421051,"now":1781383423053} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31893,"n_decoded":107}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9031,"promptTokensTotal":97559,"promptTokensSeconds":1101.94},"lastPromptTokensTotal":97559,"lastPromptTokensTime":1781383423053,"now":1781383425058} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31893,"n_decoded":107}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9031,"promptTokensTotal":97559,"promptTokensSeconds":1101.94},"lastPromptTokensTotal":97559,"lastPromptTokensTime":1781383425058,"now":1781383427073} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31893,"n_decoded":107}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9031,"promptTokensTotal":97559,"promptTokensSeconds":1101.94},"lastPromptTokensTotal":97559,"lastPromptTokensTime":1781383427073,"now":1781383429085} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31893,"n_decoded":107}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9031,"promptTokensTotal":97559,"promptTokensSeconds":1101.94},"lastPromptTokensTotal":97559,"lastPromptTokensTime":1781383429085,"now":1781383431099} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31893,"n_decoded":107}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9031,"promptTokensTotal":97559,"promptTokensSeconds":1101.94},"lastPromptTokensTotal":97559,"lastPromptTokensTime":1781383431099,"now":1781383433110} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31893,"n_decoded":107}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9031,"promptTokensTotal":97559,"promptTokensSeconds":1101.94},"lastPromptTokensTotal":97559,"lastPromptTokensTime":1781383433110,"now":1781383435194} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":true,"next_token":{"has_next_token":true,"has_new_line":false,"n_remain":31999,"n_decoded":1}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9031,"promptTokensTotal":97632,"promptTokensSeconds":1094.26},"lastPromptTokensTotal":97559,"lastPromptTokensTime":1781383435194,"now":1781383437159} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":true,"next_token":{"has_next_token":true,"has_new_line":true,"n_remain":31952,"n_decoded":48}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} +{"metrics":{"predictedTps":24.9031,"promptTokensTotal":97632,"promptTokensSeconds":1094.26},"lastPromptTokensTotal":97632,"lastPromptTokensTime":1781383437159,"now":1781383439126} +{"baseUrl":"http://localhost:8080","model":"qwen3635b3-ud-q6_k","slotsType":"object","slotsCtor":"Array","slotsKeys":["0","1","2","3"],"slotsLen":4,"slotsProto":"[object Array]","slot0":"{\"id\":0,\"n_ctx\":160000,\"speculative\":false,\"is_processing\":false}","isProcessing":[{"id":0,"is_processing":false},{"id":1,"is_processing":true,"next_token":{"has_next_token":true,"has_new_line":true,"n_remain":31903,"n_decoded":97}},{"id":2,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":30096,"n_decoded":1904}},{"id":3,"is_processing":false,"next_token":{"has_next_token":false,"has_new_line":false,"n_remain":31948,"n_decoded":52}}]} diff --git a/tui.tsx b/tui.tsx index 1ca6877..47f7649 100644 --- a/tui.tsx +++ b/tui.tsx @@ -10,7 +10,7 @@ type StreamSample = { const STREAM_WINDOW_MS = 5_000 const LIVE_STALE_MS = 1_500 const SINGLE_SAMPLE_MS = 1_000 -const SLOT_POLL_MS = 500 +const SLOT_POLL_MS = 2000 type MessageTiming = { sessionID: string requestStartAt: number @@ -33,6 +33,10 @@ type TrackerState = { sessionAverageByID: Record lastPrefillRate: number lastGeneratedTps: number + isPrefilling: boolean + prefillSlotId: number | null + prefillCapturedTokens: number | null + prefillStartAt: number } function estimateStreamTokens(delta: string) { @@ -154,7 +158,7 @@ async function fetchMetrics(baseUrl: string, model?: string): Promise = {} for (const line of text.split("\n")) { - const match = line.match(/^(llamacpp:[\w_]+)\s+(\d+(?:\.\d+)?)/) + const match = line.match(/^(llamacpp:[\w_]+)\s+(\d+(?:\.\d+)?(?:e[+-]?\d+)?)/) if (match) { result[match[1]] = parseFloat(match[2]) } @@ -188,6 +192,27 @@ function formatTtft(value: number) { return `${value.toFixed(1)}s` } +function formatPps(value: number) { + if (!Number.isFinite(value) || value <= 0) return undefined + return `${Math.round(value)}` +} + +function formatTps(value: number) { + if (!Number.isFinite(value) || value <= 0) return undefined + return `${Math.round(value)}` +} + +function formatAvg(value: number) { + if (!Number.isFinite(value) || value <= 0) return undefined + if (value >= 100) return `${Math.round(value)}` + return `${value.toFixed(1)}` +} + +function pad4(s: string | undefined) { + if (!s) return "----" + return s.padStart(4, " ") +} + function activeDurationMs(samples: StreamSample[], tailAt?: number) { if (samples.length === 0) return 0 if (samples.length === 1) { @@ -214,20 +239,6 @@ function SessionPromptRight(props: { version: () => number clock: () => number }) { - const sessionAverage = createMemo(() => { - props.version() - const totals = props.tracker.sessionAverageByID[props.sessionID] - if (!totals || totals.totalTokens <= 0 || totals.totalDurationMs <= 0) return undefined - return formatRate(totals.totalTokens / (totals.totalDurationMs / 1000), "AVG") - }) - - const sessionTtft = createMemo(() => { - props.version() - const totals = props.tracker.sessionAverageByID[props.sessionID] - if (!totals || totals.messageCount <= 0 || totals.totalTtftMs < 0) return undefined - return formatTtft(totals.totalTtftMs / totals.messageCount / 1000) - }) - const liveTps = createMemo(() => { props.version() props.clock() @@ -243,32 +254,29 @@ function SessionPromptRight(props: { const total = relevant.reduce((sum, sample) => sum + sample.tokens, 0) const durationSeconds = activeDurationMs(relevant, now) / 1000 if (durationSeconds <= 0) return undefined - return formatRate(total / durationSeconds, "AVG") + return total / durationSeconds }) const prefillRate = createMemo(() => { props.version() + if (!props.tracker.isPrefilling) return undefined const rate = props.tracker.lastPrefillRate if (!Number.isFinite(rate) || rate <= 0) return undefined - return formatRate(rate, "P") + return rate }) const generatedTps = createMemo(() => { props.version() const rate = props.tracker.lastGeneratedTps if (!Number.isFinite(rate) || rate <= 0) return undefined - return formatRate(rate, "AVG") + return rate }) const text = createMemo(() => { - const live = liveTps() ?? "-" - const avg = sessionAverage() ?? "-" - const ttft = sessionTtft() ?? "-" - const p = prefillRate() - const g = generatedTps() - const parts = [`TPS ${live}`, `AVG ${avg}`, `TTFT ${ttft}`] - if (p) parts.push(`P ${p}`) - if (g) parts.push(`G ${g}`) + const pp = prefillRate() + const tps = liveTps() + const avg = generatedTps() + const parts = [`PP ${pad4(pp ? formatPps(pp) : undefined)}`, `TPS ${pad4(tps ? formatTps(tps) : undefined)}`, `AVG ${pad4(avg ? formatAvg(avg) : undefined)}`] return parts.join(" | ") }) @@ -283,6 +291,10 @@ const tui: TuiPlugin = async (api) => { sessionAverageByID: {}, lastPrefillRate: 0, lastGeneratedTps: 0, + isPrefilling: false, + prefillSlotId: null, + prefillCapturedTokens: null, + prefillStartAt: 0, } const [version, setVersion] = createSignal(0) const [clock, setClock] = createSignal(Date.now()) @@ -347,23 +359,61 @@ const tui: TuiPlugin = async (api) => { } } + let anyPrefilling = false + for (const baseUrl of llamaServerUrls) { const metrics = await fetchMetrics(baseUrl, model) if (!metrics) continue - const promptTps = metrics["llamacpp:prompt_tokens_seconds"] const predictedTps = metrics["llamacpp:predicted_tokens_seconds"] - - if (Number.isFinite(promptTps) && promptTps > 0) { - tracker.lastPrefillRate = promptTps - } if (Number.isFinite(predictedTps) && predictedTps > 0) { tracker.lastGeneratedTps = predictedTps } - if (tracker.lastPrefillRate > 0 || tracker.lastGeneratedTps > 0) { - bump() + const slots = await fetchSlots(baseUrl, model) + const slotList = Array.isArray(slots) ? slots : slots ? Object.values(slots) : [] + + let prefillSlot: any = null + for (const slot of slotList) { + if (slot?.is_processing && slot.next_token?.[0]?.n_remain === -1) { + prefillSlot = slot + break + } } + + if (prefillSlot) { + anyPrefilling = true + const slotId = prefillSlot.id + const now = Date.now() + const nt = prefillSlot.n_prompt_tokens ?? 0 + + if (tracker.prefillSlotId === slotId) { + if (tracker.prefillCapturedTokens !== null) { + const dt = (now - tracker.prefillStartAt) / 1000 + const delta = nt - tracker.prefillCapturedTokens + if (dt > 0 && delta > 0) { + tracker.lastPrefillRate = delta / dt + } + } + } else { + tracker.prefillSlotId = slotId + tracker.prefillCapturedTokens = nt + tracker.prefillStartAt = now + tracker.lastPrefillRate = 0 + } + } else { + if (tracker.isPrefilling) { + tracker.prefillSlotId = null + tracker.prefillCapturedTokens = null + tracker.prefillStartAt = 0 + } + } + + tracker.isPrefilling = anyPrefilling + } + + if (tracker.lastPrefillRate > 0 || tracker.lastGeneratedTps > 0) { + bump() } }