From e6f4dadc62edfa8ef3b9c1a5ea94cb9595912ba1 Mon Sep 17 00:00:00 2001 From: John Smith Date: Thu, 24 Sep 2026 01:55:21 -0400 Subject: [PATCH] feat(ai): show provider response telemetry in the chat --- AUTONOMOUS_STATE.json | 8 ++++- .../AiRequestChat/CanPayForAiRequest.spec.js | 32 ++++++++++++++++++ .../AiRequestChat/ChatMessages.js | 21 ++++++++++++ .../src/AiGeneration/AiRequestChat/Utils.js | 33 +++++++++++++++++++ 4 files changed, 93 insertions(+), 1 deletion(-) diff --git a/AUTONOMOUS_STATE.json b/AUTONOMOUS_STATE.json index 7ed40dbef..65b5daab6 100644 --- a/AUTONOMOUS_STATE.json +++ b/AUTONOMOUS_STATE.json @@ -7,7 +7,7 @@ "asset-curator", "qa-playtester" ], - "last_cycle": 192, + "last_cycle": 193, "completed_targets": [ { "cycle": 75, @@ -1126,6 +1126,12 @@ "target": "Make provider telemetry inspectable per request (Tier 2 audit data layer)", "metric_delta": "The x-omniroute-* telemetry was console-only; the objective wants full request/response inspectability. Added parseProviderTelemetry (numbers, drops non-numeric, null when nothing), refactored formatProviderTelemetry onto it, a per-request registry localAiRequestProviderTelemetry with customGetAiRequestProviderTelemetry(id), an onProviderTelemetry callback threaded through sendChatCompletion AND streamChatCompletion, and storage on all three turn paths (sub-agent attributes to the parent). Cleared by the test reset. +1 test (parse shape, non-numeric drop, live storage via a create with telemetry headers) red pre-fix, mutation-checked (removing the callback yields null). 1630/1630 across 81 suites, eslint+prettier clean, divergence clean.", "commit": "feat(ai): expose per-request provider telemetry" + }, + { + "cycle": 193, + "target": "Surface the per-request provider telemetry in the chat (Tier 2 consumer)", + "metric_delta": "The cycle-192 telemetry registry had no consumer. Added formatProviderTelemetryNote (pure, in AiRequestChat/Utils.js) and a small 'Provider response: · · ~in/out tokens · cache' line in ChatMessages' status block (fork-modified), reading customGetAiRequestProviderTelemetry. +3 tests (full format, missing token half, null cases) mutation-checked (dropping the cache part fails the full-format test). Note is only shown when the endpoint reports x-omniroute-* headers, so it stays silent for Ollama (which sends none) - verified live that Ollama emits none; the format itself is pinned by tests. 1633/1633 across 81 suites, eslint+prettier clean, production build exit 0, dev watcher compiled, divergence clean.", + "commit": "feat(ai): show provider response telemetry in the chat" } ], "backlog": [ diff --git a/newIDE/app/src/AiGeneration/AiRequestChat/CanPayForAiRequest.spec.js b/newIDE/app/src/AiGeneration/AiRequestChat/CanPayForAiRequest.spec.js index 4813f8cbb..0dec25d6b 100644 --- a/newIDE/app/src/AiGeneration/AiRequestChat/CanPayForAiRequest.spec.js +++ b/newIDE/app/src/AiGeneration/AiRequestChat/CanPayForAiRequest.spec.js @@ -14,6 +14,7 @@ import { shouldShowLocalColdStartHint, useLocalColdStartHint, hasModelResponded, + formatProviderTelemetryNote, COLD_START_HINT_DELAY_MS, } from './Utils'; import { @@ -528,6 +529,37 @@ describe('useLocalColdStartHint', () => { }); }); +describe('formatProviderTelemetryNote', () => { + it('formats model, latency, tokens and cache', () => { + expect( + formatProviderTelemetryNote({ + model: 'deepseek-v4.1-flash', + latencyMs: 3361.4, + tokensIn: 8856, + tokensOut: 90, + cache: 'MISS', + }) + ).toBe( + 'deepseek-v4.1-flash · 3361 ms · ~8856 in / 90 out tokens · cache MISS' + ); + }); + + it('shows what is present and a ? for a missing half of the token pair', () => { + expect(formatProviderTelemetryNote({ tokensOut: 12 })).toBe( + '~? in / 12 out tokens' + ); + expect(formatProviderTelemetryNote({ latencyMs: 20 })).toBe('20 ms'); + }); + + it('is null when there is nothing to show', () => { + expect(formatProviderTelemetryNote(null)).toBe(null); + expect(formatProviderTelemetryNote(undefined)).toBe(null); + expect(formatProviderTelemetryNote({})).toBe(null); + expect(formatProviderTelemetryNote({ latencyMs: NaN })).toBe(null); + expect(formatProviderTelemetryNote({ model: '' })).toBe(null); + }); +}); + describe('hasModelResponded', () => { it('is false before the model produces anything', () => { expect(hasModelResponded(null)).toBe(false); diff --git a/newIDE/app/src/AiGeneration/AiRequestChat/ChatMessages.js b/newIDE/app/src/AiGeneration/AiRequestChat/ChatMessages.js index 84afe4301..51d91c19f 100644 --- a/newIDE/app/src/AiGeneration/AiRequestChat/ChatMessages.js +++ b/newIDE/app/src/AiGeneration/AiRequestChat/ChatMessages.js @@ -53,6 +53,7 @@ import { getAvailableCredits, useLocalColdStartHint, hasModelResponded, + formatProviderTelemetryNote, } from './Utils'; import Text from '../../UI/Text'; import { ColumnStackLayout, LineStackLayout } from '../../UI/Layout'; @@ -69,6 +70,7 @@ import { customGetAiRequestSystemCompacted, customGetAiRequestTokenTotal, customGetAiRequestPartialContent, + customGetAiRequestProviderTelemetry, } from '../../AI/CustomAIClient'; /** @@ -1387,6 +1389,25 @@ export const ChatMessages: React.ComponentType = React.memo( ) : null} + {!isWorking && + formatProviderTelemetryNote( + customGetAiRequestProviderTelemetry(aiRequest.id) + ) ? ( + + + Provider response:{' '} + {formatProviderTelemetryNote( + customGetAiRequestProviderTelemetry(aiRequest.id) + )} + + + ) : null} + {aiRequest.status === 'error' ? ( { + if (!telemetry || typeof telemetry !== 'object') return null; + const parts = []; + if (typeof telemetry.model === 'string' && telemetry.model) { + parts.push(telemetry.model); + } + if ( + typeof telemetry.latencyMs === 'number' && + Number.isFinite(telemetry.latencyMs) + ) { + parts.push(`${Math.round(telemetry.latencyMs)} ms`); + } + const hasIn = typeof telemetry.tokensIn === 'number'; + const hasOut = typeof telemetry.tokensOut === 'number'; + if (hasIn || hasOut) { + parts.push( + `~${hasIn ? telemetry.tokensIn : '?'} in / ${ + hasOut ? telemetry.tokensOut : '?' + } out tokens` + ); + } + if (typeof telemetry.cache === 'string' && telemetry.cache) { + parts.push(`cache ${telemetry.cache}`); + } + return parts.length > 0 ? parts.join(' · ') : null; +}; + export const hasModelResponded = (output: ?Array): boolean => !!(output || []).some( message =>