feat(ai): show provider response telemetry in the chat

This commit is contained in:
John Smith
2026-09-24 01:55:21 -04:00
parent caff144931
commit e6f4dadc62
4 changed files with 93 additions and 1 deletions
+7 -1
View File
@@ -7,7 +7,7 @@
"asset-curator",
"qa-playtester"
],
"last_cycle": 192,
"last_cycle": 193,
"completed_targets": [
{
"cycle": 75,
@@ -1126,6 +1126,12 @@
"target": "Make provider telemetry inspectable per request (Tier 2 audit data layer)",
"metric_delta": "The x-omniroute-* telemetry was console-only; the objective wants full request/response inspectability. Added parseProviderTelemetry (numbers, drops non-numeric, null when nothing), refactored formatProviderTelemetry onto it, a per-request registry localAiRequestProviderTelemetry with customGetAiRequestProviderTelemetry(id), an onProviderTelemetry callback threaded through sendChatCompletion AND streamChatCompletion, and storage on all three turn paths (sub-agent attributes to the parent). Cleared by the test reset. +1 test (parse shape, non-numeric drop, live storage via a create with telemetry headers) red pre-fix, mutation-checked (removing the callback yields null). 1630/1630 across 81 suites, eslint+prettier clean, divergence clean.",
"commit": "feat(ai): expose per-request provider telemetry"
},
{
"cycle": 193,
"target": "Surface the per-request provider telemetry in the chat (Tier 2 consumer)",
"metric_delta": "The cycle-192 telemetry registry had no consumer. Added formatProviderTelemetryNote (pure, in AiRequestChat/Utils.js) and a small 'Provider response: <model> · <ms> · ~in/out tokens · cache' line in ChatMessages' status block (fork-modified), reading customGetAiRequestProviderTelemetry. +3 tests (full format, missing token half, null cases) mutation-checked (dropping the cache part fails the full-format test). Note is only shown when the endpoint reports x-omniroute-* headers, so it stays silent for Ollama (which sends none) - verified live that Ollama emits none; the format itself is pinned by tests. 1633/1633 across 81 suites, eslint+prettier clean, production build exit 0, dev watcher compiled, divergence clean.",
"commit": "feat(ai): show provider response telemetry in the chat"
}
],
"backlog": [
@@ -14,6 +14,7 @@ import {
shouldShowLocalColdStartHint,
useLocalColdStartHint,
hasModelResponded,
formatProviderTelemetryNote,
COLD_START_HINT_DELAY_MS,
} from './Utils';
import {
@@ -528,6 +529,37 @@ describe('useLocalColdStartHint', () => {
});
});
describe('formatProviderTelemetryNote', () => {
it('formats model, latency, tokens and cache', () => {
expect(
formatProviderTelemetryNote({
model: 'deepseek-v4.1-flash',
latencyMs: 3361.4,
tokensIn: 8856,
tokensOut: 90,
cache: 'MISS',
})
).toBe(
'deepseek-v4.1-flash · 3361 ms · ~8856 in / 90 out tokens · cache MISS'
);
});
it('shows what is present and a ? for a missing half of the token pair', () => {
expect(formatProviderTelemetryNote({ tokensOut: 12 })).toBe(
'~? in / 12 out tokens'
);
expect(formatProviderTelemetryNote({ latencyMs: 20 })).toBe('20 ms');
});
it('is null when there is nothing to show', () => {
expect(formatProviderTelemetryNote(null)).toBe(null);
expect(formatProviderTelemetryNote(undefined)).toBe(null);
expect(formatProviderTelemetryNote({})).toBe(null);
expect(formatProviderTelemetryNote({ latencyMs: NaN })).toBe(null);
expect(formatProviderTelemetryNote({ model: '' })).toBe(null);
});
});
describe('hasModelResponded', () => {
it('is false before the model produces anything', () => {
expect(hasModelResponded(null)).toBe(false);
@@ -53,6 +53,7 @@ import {
getAvailableCredits,
useLocalColdStartHint,
hasModelResponded,
formatProviderTelemetryNote,
} from './Utils';
import Text from '../../UI/Text';
import { ColumnStackLayout, LineStackLayout } from '../../UI/Layout';
@@ -69,6 +70,7 @@ import {
customGetAiRequestSystemCompacted,
customGetAiRequestTokenTotal,
customGetAiRequestPartialContent,
customGetAiRequestProviderTelemetry,
} from '../../AI/CustomAIClient';
/**
@@ -1387,6 +1389,25 @@ export const ChatMessages: React.ComponentType<Props> = React.memo<Props>(
</Line>
) : null}
{!isWorking &&
formatProviderTelemetryNote(
customGetAiRequestProviderTelemetry(aiRequest.id)
) ? (
<Line justifyContent="flex-start">
<Text
noMargin
displayInlineAsSpan
size="body-small"
color="secondary"
>
<Trans>Provider response:</Trans>{' '}
{formatProviderTelemetryNote(
customGetAiRequestProviderTelemetry(aiRequest.id)
)}
</Text>
</Line>
) : null}
{aiRequest.status === 'error' ? (
<Line justifyContent="flex-start">
<AiRequestErrorRow
@@ -211,6 +211,39 @@ export const shouldShowSendAgainLabel = ({
* turn — later slow turns are context-size slowness, not a model loading into
* memory, and narrating every one as a cold start is noise.
*/
/**
* A short, human-readable note for the provider telemetry a routing proxy
* reported for the last turn (`x-omniroute-*`), or null when there is nothing
* to show. Kept pure so the chat's rendering stays a one-liner and the format
* is unit-testable.
*/
export const formatProviderTelemetryNote = (telemetry: ?Object): ?string => {
if (!telemetry || typeof telemetry !== 'object') return null;
const parts = [];
if (typeof telemetry.model === 'string' && telemetry.model) {
parts.push(telemetry.model);
}
if (
typeof telemetry.latencyMs === 'number' &&
Number.isFinite(telemetry.latencyMs)
) {
parts.push(`${Math.round(telemetry.latencyMs)} ms`);
}
const hasIn = typeof telemetry.tokensIn === 'number';
const hasOut = typeof telemetry.tokensOut === 'number';
if (hasIn || hasOut) {
parts.push(
`~${hasIn ? telemetry.tokensIn : '?'} in / ${
hasOut ? telemetry.tokensOut : '?'
} out tokens`
);
}
if (typeof telemetry.cache === 'string' && telemetry.cache) {
parts.push(`cache ${telemetry.cache}`);
}
return parts.length > 0 ? parts.join(' · ') : null;
};
export const hasModelResponded = (output: ?Array<Object>): boolean =>
!!(output || []).some(
message =>