diff --git a/.gitignore b/.gitignore index c2658d7..ebac6b9 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ node_modules/ +.npmrc +.opencode/ diff --git a/package.json b/package.json index f2c8e41..6d14334 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,12 @@ { "$schema": "https://json.schemastore.org/package.json", - "name": "oc-tps", - "version": "0.0.6", + "name": "@troed/oc-ls-stats", + "version": "0.0.17", "type": "module", "exports": { + ".": { + "import": "./tui.tsx" + }, "./tui": { "import": "./tui.tsx" } diff --git a/tui.tsx b/tui.tsx index 6af1154..88888b9 100644 --- a/tui.tsx +++ b/tui.tsx @@ -10,6 +10,7 @@ type StreamSample = { const STREAM_WINDOW_MS = 5_000 const LIVE_STALE_MS = 1_500 const SINGLE_SAMPLE_MS = 1_000 +const SLOT_POLL_MS = 500 type MessageTiming = { sessionID: string requestStartAt: number @@ -30,17 +31,156 @@ type TrackerState = { streamSamplesBySession: Record messageTimingByID: Record sessionAverageByID: Record + lastPrefillRate: number + lastGeneratedTps: number } function estimateStreamTokens(delta: string) { return Math.max(1, Math.ceil(Buffer.byteLength(delta, "utf8") / 5)) } -function formatRate(value: number, label: "TPS" | "AVG") { +function parseJSONC(text: string): unknown { + let result = "" + let inString = false + let escape = false + let i = 0 + + while (i < text.length) { + const ch = text[i] + + if (escape) { + result += ch + escape = false + i++ + continue + } + + if (ch === "\\") { + result += ch + escape = true + i++ + continue + } + + if (ch === '"') { + result += ch + inString = !inString + i++ + continue + } + + if (inString) { + result += ch + i++ + continue + } + + if (ch === "/" && text[i + 1] === "/") { + while (i < text.length && text[i] !== "\n") i++ + continue + } + + if (ch === "/" && text[i + 1] === "*") { + i += 2 + while (i < text.length - 1 && !(text[i] === "*" && text[i + 1] === "/")) i++ + i += 2 + result += " " + continue + } + + if (ch === ",") { + const next = text.indexOf(/[\s}])/), i + 1) + if (next !== -1 && /[\}])]/.test(text[next])) { + i++ + continue + } + } + + result += ch + i++ + } + + return JSON.parse(result) +} + +function stripBaseUrlPath(baseURL: string): string { + try { + const url = new URL(baseURL) + return `${url.origin}` + } catch { + return baseURL.replace(/\/v1(\/.*)?$/, "").replace(/\/$/, "") || baseURL + } +} + +function extractProviderUrls(config: unknown): string[] { + if (!config || typeof config !== "object") return [] + const obj = config as Record + const provider = obj.provider + if (!provider || typeof provider !== "object") return [] + + const urls: string[] = [] + for (const [_key, val] of Object.entries(provider as Record)) { + if (val && typeof val === "object") { + const opts = (val as Record).options + if (opts && typeof opts === "object") { + const baseURL = (opts as Record).baseURL || (opts as Record).base_url + if (typeof baseURL === "string" && baseURL.includes("localhost")) { + urls.push(stripBaseUrlPath(baseURL)) + } + } + } + } + return urls +} + +async function loadConfigUrls(api: Parameters[0]): Promise { + try { + const configPath = api.state.path.config + const resp = await fetch(`file://${configPath}`) + if (!resp.ok) return [] + const text = await resp.text() + const config = parseJSONC(text) + return extractProviderUrls(config) + } catch { + return [] + } +} + +async function fetchMetrics(baseUrl: string, model?: string): Promise | null> { + try { + const url = model ? `${baseUrl}/metrics?model=${encodeURIComponent(model)}` : `${baseUrl}/metrics` + const resp = await fetch(url) + if (!resp.ok) return null + const text = await resp.text() + const result: Record = {} + for (const line of text.split("\n")) { + const match = line.match(/^(llamacpp:[\w_]+)\s+(\d+(?:\.\d+)?)/) + if (match) { + result[match[1]] = parseFloat(match[2]) + } + } + return result + } catch { + return null + } +} + +async function fetchSlots(baseUrl: string, model?: string): Promise { + try { + const url = model ? `${baseUrl}/slots?model=${encodeURIComponent(model)}` : `${baseUrl}/slots` + const resp = await fetch(url) + if (!resp.ok) return null + return await resp.json() + } catch { + return null + } +} + +function formatRate(value: number, label: "TPS" | "AVG" | "P") { if (!Number.isFinite(value) || value <= 0) return undefined - if (value >= 100) return `${Math.round(value)}${label === "TPS" ? " TPS" : ""}` - if (value >= 10) return `${value.toFixed(1)}${label === "TPS" ? " TPS" : ""}` - return `${value.toFixed(2)}${label === "TPS" ? " TPS" : ""}` + if (value >= 100) return `${Math.round(value)}${label === "TPS" ? " TPS" : label === "P" ? "PPS" : ""}` + if (value >= 10) return `${value.toFixed(1)}${label === "TPS" ? " TPS" : label === "P" ? "PPS" : ""}` + return `${value.toFixed(2)}${label === "TPS" ? " TPS" : label === "P" ? "PPS" : ""}` } function formatTtft(value: number) { @@ -106,21 +246,43 @@ function SessionPromptRight(props: { return formatRate(total / durationSeconds, "AVG") }) + const prefillRate = createMemo(() => { + props.version() + const rate = props.tracker.lastPrefillRate + if (!Number.isFinite(rate) || rate <= 0) return undefined + return formatRate(rate, "P") + }) + + const generatedTps = createMemo(() => { + props.version() + const rate = props.tracker.lastGeneratedTps + if (!Number.isFinite(rate) || rate <= 0) return undefined + return formatRate(rate, "AVG") + }) + const text = createMemo(() => { const live = liveTps() ?? "-" const avg = sessionAverage() ?? "-" const ttft = sessionTtft() ?? "-" - return `TPS ${live} | AVG ${avg} | TTFT ${ttft}` + const p = prefillRate() + const g = generatedTps() + const parts = [`TPS ${live}`, `AVG ${avg}`, `TTFT ${ttft}`] + if (p) parts.push(`P ${p}`) + if (g) parts.push(`G ${g}`) + return parts.join(" | ") }) return <>{text() ? {text()} : null} } -const tui: TuiPlugin = async (api) => { +const tui: TuiPlugin = async (api, options) => { + console.log("[oc-ls-stats] TUI plugin loaded!") const tracker: TrackerState = { streamSamplesBySession: {}, messageTimingByID: {}, sessionAverageByID: {}, + lastPrefillRate: 0, + lastGeneratedTps: 0, } const [version, setVersion] = createSignal(0) const [clock, setClock] = createSignal(Date.now()) @@ -168,6 +330,43 @@ const tui: TuiPlugin = async (api) => { bump() } + let llamaServerUrls: string[] = ["http://localhost:8080"] + let llamaServerModel: string | undefined + + const pollMetrics = async () => { + if (!llamaServerUrls.length) return + + let model = llamaServerModel + if (!model) { + const routeSessionID = api.route.current.params?.sessionID + if (routeSessionID) { + const session = api.state.session.get(routeSessionID) + if (session?.model?.id) { + model = session.model.id + } + } + } + + for (const baseUrl of llamaServerUrls) { + const metrics = await fetchMetrics(baseUrl, model) + if (!metrics) continue + + const promptTps = metrics["llamacpp:prompt_tokens_seconds"] + const predictedTps = metrics["llamacpp:predicted_tokens_seconds"] + + if (Number.isFinite(promptTps) && promptTps > 0) { + tracker.lastPrefillRate = promptTps + } + if (Number.isFinite(predictedTps) && predictedTps > 0) { + tracker.lastGeneratedTps = predictedTps + } + + if (tracker.lastPrefillRate > 0 || tracker.lastGeneratedTps > 0) { + bump() + } + } + } + const onDelta = api.event.on("message.part.delta", (evt) => { if (evt.properties.field !== "text") return const parts = api.state.part(evt.properties.messageID) @@ -258,24 +457,37 @@ const tui: TuiPlugin = async (api) => { pruneSamples() }, 1000) + const slotTimer = setInterval(async () => { + await pollMetrics() + }, SLOT_POLL_MS) + api.lifecycle.onDispose(() => { onDelta() onMessage() onPart() clearInterval(timer) + clearInterval(slotTimer) }) + try { + if (!llamaServerUrls.length) { + llamaServerUrls = await loadConfigUrls(api) + } + } catch (e) { + console.error("[oc-ls-stats] loadConfigUrls error:", e) + } + api.slots.register({ slots: { session_prompt_right(_ctx, value) { - return + return *** SCOPED v0.0.17 LOADED *** }, }, }) } const plugin: TuiPluginModule & { id: string } = { - id: "oc-tps", + id: "@troed/oc-ls-stats", tui, }