feat(ai-builder): Report the eval harness's own model usage per case (no-changelog) (#40718)

Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>

packages/@n8n/api-types/src/index.ts

@@ -710,6 +710,7 @@ export type {
 	InstanceAiEvalMockHints,
 	InstanceAiEvalMockedCredential,
 	InstanceAiEvalRewrittenCredential,
+	InstanceAiEvalLlmUsage,
 	InstanceAiEvalExecutionResult,
 	InstanceAiEvalAgentToolCallRecord,
 	InstanceAiEvalAgentModelTurnRecord,

packages/@n8n/api-types/src/schemas/instance-ai.schema.ts

@@ -2876,6 +2876,19 @@ export interface InstanceAiEvalRewrittenCredential {
 	field: string;
 }
 
+/** Token usage of the eval's own model calls (mocks, judges, simulated user), per agent and model. */
+export interface InstanceAiEvalLlmUsage {
+	/** Eval agent name, e.g. `eval-mock-responder`. */
+	agent: string;
+	model: string;
+	calls: number;
+	/** Prompt tokens billed at the full input price: cache reads and writes are not included. */
+	uncachedInputTokens: number;
+	cacheReadTokens: number;
+	cacheWriteTokens: number;
+	outputTokens: number;
+}
+
 export interface InstanceAiEvalExecutionResult {
 	executionId: string;
 	success: boolean;
@@ -2884,6 +2897,8 @@ export interface InstanceAiEvalExecutionResult {
 	hints: InstanceAiEvalMockHints;
 	mockedCredentials: InstanceAiEvalMockedCredential[];
 	rewrittenCredentials?: InstanceAiEvalRewrittenCredential[];
+	/** Model usage of the mocks, pin data and hints for this run. */
+	llmUsage?: InstanceAiEvalLlmUsage[];
 }
 
 export class InstanceAiEvalExecutionRequest extends Z.class({
@@ -2994,6 +3009,8 @@ export interface InstanceAiEvalAgentExecutionResult {
 	seed: InstanceAiEvalAgentScenarioSeed;
 	skippedFeatures: InstanceAiEvalAgentSkippedFeature[];
 	mockedCredentials: InstanceAiEvalMockedCredential[];
+	/** Model usage of the seed and the mocks for this run. The agent's own model call is in `usage`. */
+	llmUsage?: InstanceAiEvalLlmUsage[];
 }
 
 export class InstanceAiEvalAgentExecutionRequest extends Z.class({

packages/@n8n/instance-ai/evaluations/README.md

@@ -208,6 +208,9 @@ Every run produces:
 
 - **Console** — live progress, per-scenario pass/fail with `[failure_category]` tag, and a grouped summary.
 - **`eval-results.json`** — structured results in `--output-dir` (or cwd). Consumed by the CI PR comment.
+  `harnessUsagePerRun` gives the tokens of the eval's own model calls for each iteration of a case:
+  judges, simulated user, mocks, pin data and hints. Each entry is one agent on one model.
+  `uncachedInputTokens` excludes cache reads and writes. `null` means the iteration was not measured.
 - **`.data/workflow-eval-report.html`** — self-contained debugging view with a green/red stage review for prompt, planner, builder, and verifier behavior, generalized prompt-improvement suggestions for failures, per-node execution traces, intercepted requests, mock responses, Phase 1 hints, verifier reasoning, and the per-built-workflow check rubric (see below).
 - **LangSmith experiment** — only when `LANGSMITH_API_KEY` is set. See the caveat in [Environment variables](#environment-variables).
 

packages/@n8n/instance-ai/evaluations/__tests__/case-pipeline.test.ts

@@ -1,3 +1,6 @@
+import type { InstanceAiEvalLlmUsage } from '@n8n/api-types';
+
+import { EvalUsageMeter, recordEvalUsage } from '../../src/utils/eval-usage';
 import type { CliArgs } from '../cli/args';
 import type { BuildResult } from '../harness/build-workflow';
 import { cleanupBuild } from '../harness/cleanup';
@@ -113,16 +116,57 @@ function makeDeps(
 		buildExpectationsByKey: new Map(),
 		agentContextByKey: new Map(),
 		runDebugByThreadId: new Map(),
+		harnessUsageByKey: new Map(),
 		...overrides,
 	};
 }
 
+function usageOf(agent: string, calls: number): InstanceAiEvalLlmUsage {
+	return {
+		agent,
+		model: 'anthropic/claude-sonnet-4-6',
+		calls,
+		uncachedInputTokens: 100 * calls,
+		cacheReadTokens: 0,
+		cacheWriteTokens: 0,
+		outputTokens: 10 * calls,
+	};
+}
+
 beforeEach(() => {
 	vi.mocked(cleanupBuild).mockClear();
 	vi.mocked(cleanupBuild).mockResolvedValue(true);
 });
 
 describe('createCasePipeline', () => {
+	it('counts the model usage of each row and of its build under the case and iteration', async () => {
+		const lane = makeLane();
+		// The scenario run reports the server's mock usage, the build the simulated user's.
+		vi.mocked(lane.tracedExecute).mockImplementation(async () => {
+			recordEvalUsage([usageOf('eval-mock-responder', 2)]);
+			return await Promise.resolve({ success: true, score: 1, reasoning: 'works' } as never);
+		});
+		const orchestrator = makeOrchestrator({ build: okBuild(), lane, buildDurationMs: 42 });
+		vi.mocked(orchestrator.getOrBuild).mockImplementation(async () => {
+			recordEvalUsage([usageOf('eval-user-proxy', 1)]);
+			return await Promise.resolve({ build: okBuild(), lane, buildDurationMs: 42 });
+		});
+		const harnessUsageByKey = new Map<string, EvalUsageMeter>();
+		const pipeline = createCasePipeline(makeDeps(orchestrator, { harnessUsageByKey }));
+
+		await Promise.all([
+			pipeline.runRow({ ...rowInputs('happy-path'), _iteration: 0 }),
+			pipeline.runRow({ ...rowInputs('happy-path'), _iteration: 1 }),
+		]);
+
+		for (const key of ['0:case-a', '1:case-a']) {
+			expect(harnessUsageByKey.get(key)?.entries()).toEqual([
+				usageOf('eval-user-proxy', 1),
+				usageOf('eval-mock-responder', 2),
+			]);
+		}
+	});
+
 	it('runs a workflow scenario row and eagerly cleans up after the last row', async () => {
 		const lane = makeLane();
 		vi.mocked(lane.tracedExecute).mockResolvedValue({

packages/@n8n/instance-ai/evaluations/__tests__/eval-results-dispatcher-contract.test.ts

@@ -1,4 +1,4 @@
-import type { InstanceAiEvalExecutionResult } from '@n8n/api-types';
+import type { InstanceAiEvalExecutionResult, InstanceAiEvalLlmUsage } from '@n8n/api-types';
 import { mkdtempSync, readFileSync } from 'fs';
 import { jsonParse } from 'n8n-workflow';
 import { tmpdir } from 'os';
@@ -152,6 +152,7 @@ interface DispatcherView {
 		}> | null>;
 		buildCostUsdPerRun?: Array<number | null>;
 		buildTurnsPerRun?: Array<number | null>;
+		harnessUsagePerRun: Array<InstanceAiEvalLlmUsage[] | null>;
 		transcriptPerRun: Array<TranscriptTurn[] | null>;
 		buildErrorPerRun: Array<string | null>;
 		threadIds: Array<string | null>;
@@ -391,6 +392,38 @@ describe('eval-results.json — dispatcher contract', () => {
 		);
 	});
 
+	it('serializes the harness model usage of each iteration, null when it was not measured', () => {
+		const judgeUsage: InstanceAiEvalLlmUsage = {
+			agent: 'eval-checklist-verifier',
+			model: 'anthropic/claude-sonnet-4-6',
+			calls: 3,
+			uncachedInputTokens: 41_000,
+			cacheReadTokens: 12_000,
+			cacheWriteTokens: 4_000,
+			outputTokens: 2_500,
+		};
+		const evaluation = aggregateResults(
+			[[{ ...iteration1(), harnessUsage: [judgeUsage] }], [iteration2()]],
+			2,
+		);
+		const dir = mkdtempSync(join(tmpdir(), 'eval-results-contract-'));
+		const { jsonPath } = writeEvalResults(
+			evaluation,
+			1234,
+			dir,
+			'exp-dispatcher-contract',
+			undefined,
+			undefined,
+			new Map([[testCase, 'daily-digest']]),
+			undefined,
+			undefined,
+		);
+		const report = jsonParse<DispatcherView>(readFileSync(jsonPath, 'utf8'));
+
+		// LangTracer prices these per model. A null run is "not measured", never "free".
+		expect(report.testCases[0].harnessUsagePerRun).toEqual([[judgeUsage], null]);
+	});
+
 	it('serializes per-iteration `claude` build spend when a run recorded it', () => {
 		const evaluation = aggregateResults(
 			[[{ ...iteration1(), buildCostUsd: 0.31, buildTurns: 5 }], [iteration2()]],

packages/@n8n/instance-ai/evaluations/__tests__/fixtures/eval-results.partial-abort.json

@@ -41,7 +41,8 @@
 			],
 			"transcriptPerRun": [null],
 			"buildErrorPerRun": [null],
-			"buildTimeoutPerRun": [null]
+			"buildTimeoutPerRun": [null],
+			"harnessUsagePerRun": [null]
 		},
 		{
 			"name": "build me something",
@@ -76,7 +77,8 @@
 			],
 			"transcriptPerRun": [null],
 			"buildErrorPerRun": ["TimeoutError: The operation was aborted due to timeout"],
-			"buildTimeoutPerRun": [null]
+			"buildTimeoutPerRun": [null],
+			"harnessUsagePerRun": [null]
 		}
 	]
 }

packages/@n8n/instance-ai/evaluations/__tests__/n8n-client-eval-usage.test.ts

@@ -0,0 +1,53 @@
+import type { InstanceAiEvalLlmUsage } from '@n8n/api-types';
+import { afterEach, vi } from 'vitest';
+
+import { EvalUsageMeter } from '../../src/utils/eval-usage';
+import { N8nClient } from '../clients/n8n-client';
+
+const MOCK_RESPONDER_USAGE: InstanceAiEvalLlmUsage = {
+	agent: 'eval-mock-responder',
+	model: 'anthropic/claude-sonnet-4-6',
+	calls: 4,
+	uncachedInputTokens: 2400,
+	cacheReadTokens: 0,
+	cacheWriteTokens: 0,
+	outputTokens: 600,
+};
+
+function mockFetch(data: unknown) {
+	vi.stubGlobal(
+		'fetch',
+		vi.fn(() => ({
+			ok: true,
+			status: 200,
+			headers: { get: () => null },
+			json: () => ({ data }),
+			text: () => '',
+		})),
+	);
+}
+
+afterEach(() => vi.unstubAllGlobals());
+
+describe('N8nClient eval runs', () => {
+	it('adds the model usage a workflow scenario run reports to the current meter', async () => {
+		mockFetch({ executionId: 'exec-1', success: true, llmUsage: [MOCK_RESPONDER_USAGE] });
+		const meter = new EvalUsageMeter();
+
+		await meter.run(async () => await new N8nClient('http://n8n.test').executeWithLlmMock('wf-1'));
+
+		expect(meter.entries()).toEqual([MOCK_RESPONDER_USAGE]);
+	});
+
+	it('adds the model usage an agent scenario run reports to the current meter', async () => {
+		mockFetch({ runId: 'run-1', success: true, llmUsage: [MOCK_RESPONDER_USAGE] });
+		const meter = new EvalUsageMeter();
+
+		await meter.run(
+			async () =>
+				await new N8nClient('http://n8n.test').executeAgentWithLlmMock('agent-1', 'project-1'),
+		);
+
+		expect(meter.entries()).toEqual([MOCK_RESPONDER_USAGE]);
+	});
+});

packages/@n8n/instance-ai/evaluations/__tests__/reshape.test.ts

@@ -386,6 +386,44 @@ describe('reshapeLangSmithRuns', () => {
 		expect(result[0][0]?.runDebug).toHaveLength(1);
 		expect(result[0][0]?.runDebug?.[0]?.runId).toBe('run-1');
 	});
+
+	it('attaches each iteration its own harness model usage by iteration:fileSlug', () => {
+		const cases = [withFile('airtable', [scenario('s1')])];
+		const rows = [0, 1].map((iteration) =>
+			row(
+				{ testCaseFile: 'airtable', scenarioName: 's1', _iteration: iteration },
+				{ buildSuccess: true, passed: true, score: 1, reasoning: 'ok' },
+			),
+		);
+		const usage = (calls: number) => [
+			{
+				agent: 'eval-checklist-verifier',
+				model: 'anthropic/claude-sonnet-4-6',
+				calls,
+				uncachedInputTokens: 1000 * calls,
+				cacheReadTokens: 0,
+				cacheWriteTokens: 0,
+				outputTokens: 100 * calls,
+			},
+		];
+		const harnessUsageByKey = new Map([
+			['0:airtable', usage(1)],
+			['1:airtable', usage(2)],
+		]);
+
+		const result = reshapeLangSmithRuns(
+			rows,
+			cases,
+			2,
+			new Map(),
+			new Map(),
+			undefined,
+			new Map(),
+			harnessUsageByKey,
+		);
+
+		expect(result.map((iteration) => iteration[0]?.harnessUsage)).toEqual([usage(1), usage(2)]);
+	});
 });
 
 describe('sentinelOutcomeFromVerdicts', () => {

packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts

@@ -32,6 +32,8 @@ import type { ExecutionStatus } from 'n8n-workflow';
 import { Agent, setGlobalDispatcher } from 'undici';
 import { z } from 'zod';
 
+import { recordEvalUsage } from '../../src/utils/eval-usage';
+
 // Disable undici's 300s timeouts — mocked eval runs take minutes; the per-request
 // AbortSignal is the real bound. This is process-global: only ever imported by the
 // eval CLI harness — never import into the n8n server or shared runtime code.
@@ -1350,6 +1352,7 @@ export class N8nClient {
 			body,
 			timeoutMs: serverBudgetMs + CLIENT_ABORT_MARGIN_MS,
 		})) as { data: InstanceAiEvalExecutionResult };
+		recordEvalUsage(result.data.llmUsage);
 		return result.data;
 	}
 
@@ -1377,6 +1380,7 @@ export class N8nClient {
 				timeoutMs: serverBudgetMs + CLIENT_ABORT_MARGIN_MS,
 			},
 		)) as { data: InstanceAiEvalAgentExecutionResult };
+		recordEvalUsage(result.data.llmUsage);
 		return result.data;
 	}
 

packages/@n8n/instance-ai/evaluations/run/case-pipeline.ts

@@ -14,6 +14,7 @@ import type { InstanceAiRunDebugResponse } from '@n8n/api-types';
 
 import type { BuildOrchestrator } from './build-orchestrator';
 import { sentinelOutcomeFromVerdicts, type TargetOutput } from './reshape';
+import { EvalUsageMeter } from '../../src/utils/eval-usage';
 import type { CliArgs } from '../cli/args';
 import {
 	draftAgentVerdict,
@@ -57,6 +58,8 @@ export interface CasePipelineDeps {
 	buildExpectationsByKey: Map<string, Promise<BuildExpectationResult[]>>;
 	agentContextByKey: Map<string, Promise<AgentScenarioContext>>;
 	runDebugByThreadId: Map<string, Promise<InstanceAiRunDebugResponse[]>>;
+	/** Usage of the eval's own model calls, per `iteration:fileSlug`. Filled here. */
+	harnessUsageByKey: Map<string, EvalUsageMeter>;
 }
 
 export interface CasePipeline {
@@ -85,9 +88,19 @@ export function createCasePipeline(deps: CasePipelineDeps): CasePipeline {
 		buildExpectationsByKey,
 		agentContextByKey,
 		runDebugByThreadId,
+		harnessUsageByKey,
 	} = deps;
 	const { getOrBuild, buildCache } = orchestrator;
 
+	function usageMeterFor(key: string): EvalUsageMeter {
+		let meter = harnessUsageByKey.get(key);
+		if (!meter) {
+			meter = new EvalUsageMeter();
+			harnessUsageByKey.set(key, meter);
+		}
+		return meter;
+	}
+
 	// Rows remaining per `iteration:fileSlug` build. When the last row of a
 	// build finishes, its backend artifacts (workflow, data tables, thread — and
 	// with the thread its sandbox) are deleted right away instead of at the end
@@ -147,7 +160,10 @@ export function createCasePipeline(deps: CasePipelineDeps): CasePipeline {
 	const runRow = async (inputs: ScenarioRowInputs): Promise<TargetOutput> => {
 		const iteration = inputs._iteration ?? 0;
 		try {
-			return await runScenarioRow(inputs, iteration);
+			// Only rows start builds, so a build and its judges count on the meter of its own case.
+			return await usageMeterFor(`${String(iteration)}:${inputs.testCaseFile}`).run(
+				async () => await runScenarioRow(inputs, iteration),
+			);
 		} catch (error: unknown) {
 			// runScenarioRow guards scenario execution internally, but a build-phase
 			// throw (getOrBuild) or a budget abort must not reject up to the driver

packages/@n8n/instance-ai/evaluations/run/direct-driver.ts

@@ -92,6 +92,7 @@ export async function runDirect(config: DirectRunConfig): Promise<{
 			sideBand.buildExpectations,
 			lanes[0]?.baseUrl,
 			sideBand.runDebug,
+			sideBand.harnessUsage,
 		);
 		for (const iterationResults of allRunResults) {
 			partialResults?.push(iterationResults);

packages/@n8n/instance-ai/evaluations/run/eval-session.ts

@@ -9,6 +9,7 @@
 // ---------------------------------------------------------------------------
 
 import type {
+	InstanceAiEvalLlmUsage,
 	InstanceAiEvalThreadMemoryResponse,
 	InstanceAiRunDebugResponse,
 } from '@n8n/api-types';
@@ -25,6 +26,7 @@ import {
 import { createCasePipeline, type CasePipeline } from './case-pipeline';
 import { LaneAllocator } from './lane-allocator';
 import { provisionBuildProject } from './lane-users';
+import type { EvalUsageMeter } from '../../src/utils/eval-usage';
 import type { CliArgs } from '../cli/args';
 import { N8nClient } from '../clients/n8n-client';
 import type { WorkflowTestCaseWithFile } from '../data/workflows';
@@ -80,6 +82,8 @@ export interface ResolvedSideBand {
 	 *  builds (no threadId) still attach their outcome-expectation verdicts. */
 	buildExpectations: Map<string, BuildExpectationResult[]>;
 	runDebug: Map<string, InstanceAiRunDebugResponse[]>;
+	/** Keyed by the build-cache key (`iteration:fileSlug`). */
+	harnessUsage: Map<string, InstanceAiEvalLlmUsage[]>;
 }
 
 export interface EvalSession {
@@ -262,6 +266,7 @@ export function createEvalSession(config: EvalSessionConfig): EvalSession {
 	// Agent config + skills, fetched once per build and shared by every
 	// scenario row of the case (the agent analog of the cached workflow JSON).
 	const agentContextByKey = new Map<string, Promise<AgentScenarioContext>>();
+	const harnessUsageByKey = new Map<string, EvalUsageMeter>();
 
 	const orchestrator = createBuildOrchestrator({
 		args,
@@ -288,6 +293,7 @@ export function createEvalSession(config: EvalSessionConfig): EvalSession {
 		buildExpectationsByKey,
 		agentContextByKey,
 		runDebugByThreadId,
+		harnessUsageByKey,
 	});
 
 	const resolveSideBand = async (): Promise<ResolvedSideBand> => {
@@ -299,7 +305,10 @@ export function createEvalSession(config: EvalSessionConfig): EvalSession {
 		for (const [threadId, runDebugPromise] of runDebugByThreadId) {
 			runDebug.set(threadId, await runDebugPromise);
 		}
-		return { transcriptByThreadId, buildExpectations, runDebug };
+		const harnessUsage = new Map(
+			[...harnessUsageByKey].map(([key, meter]) => [key, meter.entries()]),
+		);
+		return { transcriptByThreadId, buildExpectations, runDebug, harnessUsage };
 	};
 
 	const drainBuilds = async (): Promise<void> => {

packages/@n8n/instance-ai/evaluations/run/langsmith-driver.ts

@@ -253,6 +253,7 @@ export async function runWithLangSmith(config: RunConfig): Promise<{
 			sideBand.buildExpectations,
 			lanes[0]?.baseUrl,
 			sideBand.runDebug,
+			sideBand.harnessUsage,
 		);
 		const evaluation = aggregateResults(allRunResults, args.iterations);
 

packages/@n8n/instance-ai/evaluations/run/persist.ts

@@ -524,6 +524,10 @@ export function writeEvalResults(
 						buildTurnsPerRun: tc.runs.map((run) => run.buildTurns ?? null),
 					}
 				: {}),
+			// Tokens of the eval's own model calls (judges, simulated user, mocks) per
+			// iteration, by agent and model. Null when the iteration was not measured,
+			// e.g. one recovered from the row journal after a crash.
+			harnessUsagePerRun: tc.runs.map((run) => run.harnessUsage ?? null),
 			scenarios: tc.executionScenarios.map((sa) => ({
 				name: sa.scenario.name,
 				passCount: sa.passCount,

packages/@n8n/instance-ai/evaluations/run/reshape.ts

@@ -10,6 +10,7 @@
 import type {
 	InstanceAiEvalAgentExecutionResult,
 	InstanceAiEvalExecutionResult,
+	InstanceAiEvalLlmUsage,
 	InstanceAiRunDebugResponse,
 } from '@n8n/api-types';
 import { isRecord } from '@n8n/utils/is-record';
@@ -262,6 +263,8 @@ export function reshapeLangSmithRuns(
 	buildExpectationsByKey: Map<string, BuildExpectationResult[]>,
 	n8nBaseUrl: string | undefined,
 	runDebugByThreadId: Map<string, InstanceAiRunDebugResponse[]> = new Map(),
+	/** Keyed by the build-cache key (`iteration:fileSlug`). */
+	harnessUsageByKey: Map<string, InstanceAiEvalLlmUsage[]> = new Map(),
 ): WorkflowTestCaseResult[][] {
 	// Index runs by (iteration, testCaseFile, scenarioName) using the `_iteration`
 	// we injected in expandExamplesForIterations. Falls back to 0 for single-run.
@@ -365,7 +368,8 @@ export function reshapeLangSmithRuns(
 			}
 
 			const transcript = threadId ? transcriptByThreadId.get(threadId) : undefined;
-			const buildExpectationResults = buildExpectationsByKey.get(`${String(iter)}:${fileSlug}`);
+			const buildKey = `${String(iter)}:${fileSlug}`;
+			const buildExpectationResults = buildExpectationsByKey.get(buildKey);
 			runResults.push({
 				testCase,
 				fileSlug,
@@ -387,6 +391,7 @@ export function reshapeLangSmithRuns(
 				buildTurns,
 				n8nBaseUrl,
 				runDebug: threadId ? runDebugByThreadId.get(threadId) : undefined,
+				harnessUsage: harnessUsageByKey.get(buildKey),
 			});
 		}
 		allRunResults.push(runResults);

packages/@n8n/instance-ai/evaluations/types.ts

@@ -6,6 +6,7 @@ import type {
 	AgentSkill,
 	InstanceAiEvalAgentExecutionResult,
 	InstanceAiEvalExecutionResult,
+	InstanceAiEvalLlmUsage,
 	InstanceAiEvalSeedDataTable,
 	InstanceAiRunDebugResponse,
 	InstanceAiPromptConfiguration,
@@ -387,6 +388,8 @@ export interface WorkflowTestCaseResult {
 	n8nBaseUrl?: string;
 	/** Per-run LLM step debug captured from the instance-ai debug API after build. */
 	runDebug?: InstanceAiRunDebugResponse[];
+	/** Usage of the eval's own model calls in this iteration: judges, simulated user, mocks. */
+	harnessUsage?: InstanceAiEvalLlmUsage[];
 }
 
 // ---------------------------------------------------------------------------

packages/@n8n/instance-ai/src/index.ts

@@ -35,6 +35,7 @@ import type * as LangsmithTracingMod from './tracing/langsmith-tracing';
 import type * as TraceReplayMod from './tracing/trace-replay';
 import type * as AgentTreeMod from './utils/agent-tree';
 import type * as EvalAgentsMod from './utils/eval-agents';
+import type * as EvalUsageMod from './utils/eval-usage';
 import type * as StreamHelpersMod from './utils/stream-helpers';
 import type * as WorkflowLoopMod from './workflow-loop';
 import type * as WorkflowLoopRuntimeMod from './workflow-loop/runtime';
@@ -147,6 +148,7 @@ const loadMaterializeRuntimeSkills = lazyModule(
 	() => require('./skills/materialize-runtime-skills') as typeof MaterializeRuntimeSkillsMod,
 );
 const loadEvalAgents = lazyModule(() => require('./utils/eval-agents') as typeof EvalAgentsMod);
+const loadEvalUsage = lazyModule(() => require('./utils/eval-usage') as typeof EvalUsageMod);
 const loadAgentTree = lazyModule(() => require('./utils/agent-tree') as typeof AgentTreeMod);
 const loadBuilderTemplatesService = lazyModule(
 	() => require('./workspace/builder-templates-service') as typeof BuilderTemplatesServiceMod,
@@ -450,6 +452,9 @@ export const createEvalAgent: typeof EvalAgentsMod.createEvalAgent = lazyFunctio
 export const extractText: typeof EvalAgentsMod.extractText = lazyFunction(
 	() => loadEvalAgents().extractText,
 );
+export const meterEvalUsage: typeof EvalUsageMod.meterEvalUsage = lazyFunction(
+	() => loadEvalUsage().meterEvalUsage,
+);
 defineLazyExport('PURE_REPLAY_TOOLS', () => loadTraceReplay().PURE_REPLAY_TOOLS);
 defineLazyExport(
 	'SUB_AGENT_RESOURCE_PREFIX',

packages/@n8n/instance-ai/src/utils/__tests__/eval-agents.test.ts

@@ -4,6 +4,7 @@ type MockAgentInstance = {
 	model: Mock;
 	instructions: Mock;
 	thinking: Mock;
+	configuration: Mock;
 };
 
 const mockAgentInstances: MockAgentInstance[] = [];
@@ -13,6 +14,7 @@ vi.mock('@n8n/agents', () => ({
 		this.model = vi.fn().mockReturnThis();
 		this.instructions = vi.fn().mockReturnThis();
 		this.thinking = vi.fn().mockReturnThis();
+		this.configuration = vi.fn().mockReturnThis();
 		mockAgentInstances.push(this);
 	}),
 	Tool: vi.fn(),

packages/@n8n/instance-ai/src/utils/__tests__/eval-usage.test.ts

@@ -0,0 +1,138 @@
+import type { InstanceAiEvalLlmUsage } from '@n8n/api-types';
+import { convertArrayToReadableStream, MockLanguageModelV3 } from 'ai/test';
+
+import { createEvalAgent } from '../eval-agents';
+import { EvalUsageMeter, recordEvalUsage } from '../eval-usage';
+
+const ORIGINAL_ENV = { ...process.env };
+
+/** A model that answers "ok" to every call and reports these token counts. */
+function modelReporting(tokens: {
+	noCache: number;
+	cacheRead: number;
+	cacheWrite: number;
+	output: number;
+}): MockLanguageModelV3 {
+	const finishReason = { unified: 'stop', raw: 'end_turn' } as const;
+	const usage = {
+		inputTokens: {
+			total: tokens.noCache + tokens.cacheRead + tokens.cacheWrite,
+			noCache: tokens.noCache,
+			cacheRead: tokens.cacheRead,
+			cacheWrite: tokens.cacheWrite,
+		},
+		outputTokens: { total: tokens.output, text: tokens.output, reasoning: 0 },
+	};
+	return new MockLanguageModelV3({
+		provider: 'mock',
+		modelId: 'judge',
+		doGenerate: { content: [{ type: 'text', text: 'ok' }], finishReason, usage, warnings: [] },
+		doStream: async () =>
+			await Promise.resolve({
+				stream: convertArrayToReadableStream([
+					{ type: 'stream-start', warnings: [] },
+					{ type: 'text-start', id: 'text-1' },
+					{ type: 'text-delta', id: 'text-1', delta: 'ok' },
+					{ type: 'text-end', id: 'text-1' },
+					{ type: 'finish', finishReason, usage },
+				]),
+			}),
+	});
+}
+
+/** No eval key in the environment, so the agent runs on the mock model. */
+function judgeOn(model: MockLanguageModelV3, name = 'eval-checklist-verifier') {
+	return createEvalAgent(name, { instructions: 'Judge the run.', fallbackModelConfig: model });
+}
+
+describe('eval usage meter', () => {
+	beforeEach(() => {
+		process.env = { ...ORIGINAL_ENV };
+		for (const key of [
+			'N8N_INSTANCE_AI_MODEL',
+			'N8N_INSTANCE_AI_EVAL_MODEL',
+			'N8N_INSTANCE_AI_MODEL_API_KEY',
+			'N8N_INSTANCE_AI_MODEL_URL',
+			'EVAL_MODAL_LLM_HEADERS',
+			'N8N_AI_ANTHROPIC_KEY',
+			'ANTHROPIC_API_KEY',
+		]) {
+			delete process.env[key];
+		}
+	});
+
+	afterAll(() => {
+		process.env = ORIGINAL_ENV;
+	});
+
+	it('counts every call an eval agent makes inside the meter, by agent and model', async () => {
+		const judge = judgeOn(
+			modelReporting({ noCache: 100, cacheRead: 1000, cacheWrite: 200, output: 50 }),
+		);
+		const meter = new EvalUsageMeter();
+
+		await meter.run(async () => {
+			await judge.generate('First scenario');
+			await judge.generate('Second scenario');
+		});
+
+		expect(meter.entries()).toEqual([
+			{
+				agent: 'eval-checklist-verifier',
+				model: 'mock/judge',
+				calls: 2,
+				uncachedInputTokens: 200,
+				cacheReadTokens: 2000,
+				cacheWriteTokens: 400,
+				outputTokens: 100,
+			},
+		]);
+	});
+
+	it('keeps the calls of concurrent meters apart', async () => {
+		const judge = judgeOn(modelReporting({ noCache: 10, cacheRead: 0, cacheWrite: 0, output: 1 }));
+		const first = new EvalUsageMeter();
+		const second = new EvalUsageMeter();
+
+		await Promise.all([
+			first.run(async () => await judge.generate('Case one')),
+			second.run(async () => {
+				await judge.generate('Case two');
+				await judge.generate('Case two again');
+			}),
+		]);
+
+		expect(first.entries().map((entry) => entry.calls)).toEqual([1]);
+		expect(second.entries().map((entry) => entry.calls)).toEqual([2]);
+	});
+
+	it('adds usage that an eval endpoint reported to the current meter', async () => {
+		const reported: InstanceAiEvalLlmUsage = {
+			agent: 'eval-mock-responder',
+			model: 'anthropic/claude-sonnet-4-6',
+			calls: 3,
+			uncachedInputTokens: 10,
+			cacheReadTokens: 20,
+			cacheWriteTokens: 30,
+			outputTokens: 5,
+		};
+		const meter = new EvalUsageMeter();
+
+		await meter.run(async () => {
+			recordEvalUsage([reported]);
+			recordEvalUsage([reported]);
+			await Promise.resolve();
+		});
+
+		expect(meter.entries()).toEqual([
+			{
+				...reported,
+				calls: 6,
+				uncachedInputTokens: 20,
+				cacheReadTokens: 40,
+				cacheWriteTokens: 60,
+				outputTokens: 10,
+			},
+		]);
+	});
+});

packages/@n8n/instance-ai/src/utils/eval-agents.ts

@@ -3,6 +3,7 @@
 import { Agent, Tool, type GenerateResult, type ModelConfig } from '@n8n/agents';
 import { getProviderPrefix, splitModelId } from '@n8n/ai-utilities/agent-config';
 
+import { evalUsageGuardrail } from './eval-usage';
 import { parseModelHeadersJson } from './parse-model-headers';
 import { applyAgentThinking } from '../agent/apply-agent-thinking';
 
@@ -174,6 +175,8 @@ export function createEvalAgent(
 	}
 
 	applyAgentThinking(agent, model);
+	// `configuration()` replaces the defaults it was given before: put any other default option in this call.
+	agent.configuration({ guardrails: { hooks: [evalUsageGuardrail(name)] } });
 
 	return agent;
 }

packages/@n8n/instance-ai/src/utils/eval-usage.ts

@@ -0,0 +1,77 @@
+/** Adds up the token usage of eval agent calls, so an eval run can report what its own model calls cost. */
+
+import type { ModelGuardrail, TokenUsage } from '@n8n/agents';
+import type { InstanceAiEvalLlmUsage } from '@n8n/api-types';
+import { AsyncLocalStorage } from 'node:async_hooks';
+
+const currentMeter = new AsyncLocalStorage<EvalUsageMeter>();
+
+export class EvalUsageMeter {
+	private readonly usage = new Map<string, InstanceAiEvalLlmUsage>();
+
+	/** Every eval agent call made inside `fn`, at any depth, adds to this meter. */
+	async run<T>(fn: () => Promise<T>): Promise<T> {
+		return await currentMeter.run(this, fn);
+	}
+
+	add(entry: InstanceAiEvalLlmUsage): void {
+		const key = JSON.stringify([entry.agent, entry.model]);
+		const total = this.usage.get(key);
+		if (!total) {
+			this.usage.set(key, { ...entry });
+			return;
+		}
+		total.calls += entry.calls;
+		total.uncachedInputTokens += entry.uncachedInputTokens;
+		total.cacheReadTokens += entry.cacheReadTokens;
+		total.cacheWriteTokens += entry.cacheWriteTokens;
+		total.outputTokens += entry.outputTokens;
+	}
+
+	entries(): InstanceAiEvalLlmUsage[] {
+		return [...this.usage.values()].map((entry) => ({ ...entry }));
+	}
+}
+
+/** Adds usage reported by an eval endpoint to the current meter. Does nothing outside a meter. */
+export function recordEvalUsage(entries: InstanceAiEvalLlmUsage[] | undefined): void {
+	const meter = currentMeter.getStore();
+	if (!meter || !entries) return;
+	for (const entry of entries) meter.add(entry);
+}
+
+/** Runs `fn` on a new meter and returns the usage it counted. */
+export async function meterEvalUsage<T>(
+	fn: () => Promise<T>,
+): Promise<{ result: T; usage: InstanceAiEvalLlmUsage[] }> {
+	const meter = new EvalUsageMeter();
+	const result = await meter.run(fn);
+	return { result, usage: meter.entries() };
+}
+
+function toEvalLlmUsage(agent: string, model: string, usage: TokenUsage): InstanceAiEvalLlmUsage {
+	const cacheReadTokens = usage.inputTokenDetails?.cacheRead ?? 0;
+	const cacheWriteTokens = usage.inputTokenDetails?.cacheWrite ?? 0;
+	return {
+		agent,
+		model,
+		calls: 1,
+		// A zero `noCache` is left out of the details, so derive it from the total.
+		uncachedInputTokens:
+			usage.inputTokenDetails?.noCache ??
+			Math.max(usage.promptTokens - cacheReadTokens - cacheWriteTokens, 0),
+		cacheReadTokens,
+		cacheWriteTokens,
+		outputTokens: usage.completionTokens,
+	};
+}
+
+/** Runs after each model call of the agent named `agent` and adds its usage to the current meter. */
+export function evalUsageGuardrail(agent: string): ModelGuardrail {
+	return {
+		async after(ctx, usage) {
+			const meter = currentMeter.getStore();
+			if (meter && usage) meter.add(toEvalLlmUsage(agent, ctx.model, usage));
+		},
+	};
+}

packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts

@@ -25,6 +25,10 @@ vi.mock('@n8n/instance-ai', () => {
 			...(prior ?? {}),
 			boundTargets: prior?.boundTargets,
 		})),
+		meterEvalUsage: vi.fn(async (fn: () => Promise<unknown>) => ({
+			result: await fn(),
+			usage: [],
+		})),
 	};
 });
 
@@ -40,7 +44,10 @@ import type {
 	AiPreferenceDto,
 	InstanceAiPreferenceCardEvent,
 	InstanceAiAdminSettingsUpdateRequest,
+	InstanceAiEvalAgentExecutionResult,
 	InstanceAiEvalCredentialAllowlistRequest,
+	InstanceAiEvalExecutionResult,
+	InstanceAiEvalLlmUsage,
 	InstanceAiEvalRestoreThreadRequest,
 	InstanceAiEvalThreadMemoryResponse,
 	InstanceAiSendMessageRequest,
@@ -59,7 +66,11 @@ import type {
 } from '@n8n/api-types';
 import type { ModuleRegistry } from '@n8n/backend-common';
 import type { GlobalConfig } from '@n8n/config';
-import { buildAgentTreeFromEvents, seedAgentBuilderTargetMetadata } from '@n8n/instance-ai';
+import {
+	buildAgentTreeFromEvents,
+	meterEvalUsage,
+	seedAgentBuilderTargetMetadata,
+} from '@n8n/instance-ai';
 import {
 	InstanceAiPersistPendingAgentRequest,
 	InstanceAiThreadTabsRequestDto,
@@ -116,6 +127,16 @@ function scopeOf(handlerName: string): { scope: Scope; globalOnly: boolean } | u
 	return route.accessScope;
 }
 
+const MOCK_RESPONDER_USAGE: InstanceAiEvalLlmUsage = {
+	agent: 'eval-mock-responder',
+	model: 'anthropic/claude-sonnet-4-6',
+	calls: 2,
+	uncachedInputTokens: 1200,
+	cacheReadTokens: 0,
+	cacheWriteTokens: 0,
+	outputTokens: 300,
+};
+
 describe('InstanceAiController', () => {
 	const instanceAiService = mock<InstanceAiService>();
 	const gatewayService = mock<InstanceAiGatewayService>();
@@ -142,6 +163,8 @@ describe('InstanceAiController', () => {
 	const projectService = mock<ProjectService>();
 	const instanceAiErrorReporter = mock<InstanceAiErrorReporterService>();
 
+	const evalExecutionService = mock<EvalExecutionService>();
+	const evalAgentExecutionService = mock<EvalAgentExecutionService>();
 	const evalCredentialAllowlists = new EvalThreadCredentialAllowlistService();
 	const evalThreadRestore = mock<EvalThreadRestoreService>();
 	const preferenceCardService = mock<InstanceAiPreferenceCardService>();
@@ -157,8 +180,8 @@ describe('InstanceAiController', () => {
 		pendingAgentService,
 		settingsService,
 		modelCatalogService,
-		mock<EvalExecutionService>(),
-		mock<EvalAgentExecutionService>(),
+		evalExecutionService,
+		evalAgentExecutionService,
 		evalCredentialAllowlists,
 		evalThreadRestore,
 		eventBus,
@@ -1010,6 +1033,41 @@ describe('InstanceAiController', () => {
 		it('should require instanceAi:eval scope', () => {
 			expect(scopeOf('executeWithLlmMock')).toEqual({ scope: 'instanceAi:eval', globalOnly: true });
 		});
+
+		it('should return the model usage of the run with its result', async () => {
+			const runResult = { executionId: 'exec-1', success: true } as InstanceAiEvalExecutionResult;
+			evalExecutionService.executeWithLlmMock.mockResolvedValue(runResult);
+			vi.mocked(meterEvalUsage).mockImplementationOnce(async (fn) => ({
+				result: await fn(),
+				usage: [MOCK_RESPONDER_USAGE],
+			}));
+
+			const result = await controller.executeWithLlmMock(req, res, 'wf-1', {});
+
+			expect(result).toEqual({ ...runResult, llmUsage: [MOCK_RESPONDER_USAGE] });
+			expect(evalExecutionService.executeWithLlmMock).toHaveBeenCalledWith('wf-1', req.user, {});
+		});
+	});
+
+	describe('executeAgentWithLlmMock', () => {
+		it('should return the model usage of the run with its result', async () => {
+			const runResult = { runId: 'run-1', success: true } as InstanceAiEvalAgentExecutionResult;
+			evalAgentExecutionService.executeWithLlmMock.mockResolvedValue(runResult);
+			vi.mocked(meterEvalUsage).mockImplementationOnce(async (fn) => ({
+				result: await fn(),
+				usage: [MOCK_RESPONDER_USAGE],
+			}));
+			const payload = { projectId: 'project-1' };
+
+			const result = await controller.executeAgentWithLlmMock(req, res, 'agent-1', payload);
+
+			expect(result).toEqual({ ...runResult, llmUsage: [MOCK_RESPONDER_USAGE] });
+			expect(evalAgentExecutionService.executeWithLlmMock).toHaveBeenCalledWith(
+				'agent-1',
+				req.user,
+				payload,
+			);
+		});
 	});
 
 	describe('setThreadCredentialAllowlist', () => {

packages/cli/src/modules/instance-ai/instance-ai.controller.ts

@@ -58,6 +58,7 @@ import { hasGlobalScope } from '@n8n/permissions';
 import {
 	buildAgentTreeFromEvents,
 	clearedAgentBuilderTargetMetadata,
+	meterEvalUsage,
 	seedAgentBuilderTargetMetadata,
 } from '@n8n/instance-ai';
 import {
@@ -1160,7 +1161,11 @@ export class InstanceAiController {
 		@Param('workflowId') workflowId: string,
 		@Body payload: InstanceAiEvalExecutionRequest,
 	) {
-		return await this.evalExecutionService.executeWithLlmMock(workflowId, req.user, payload);
+		// The mocks run on the eval model; the caller adds their usage to the case's cost.
+		const { result, usage } = await meterEvalUsage(
+			async () => await this.evalExecutionService.executeWithLlmMock(workflowId, req.user, payload),
+		);
+		return { ...result, llmUsage: usage };
 	}
 
 	// Runs for minutes; same client timeout handling as the workflow variant.
@@ -1172,7 +1177,11 @@ export class InstanceAiController {
 		@Param('agentId') agentId: string,
 		@Body payload: InstanceAiEvalAgentExecutionRequest,
 	) {
-		return await this.evalAgentExecutionService.executeWithLlmMock(agentId, req.user, payload);
+		const { result, usage } = await meterEvalUsage(
+			async () =>
+				await this.evalAgentExecutionService.executeWithLlmMock(agentId, req.user, payload),
+		);
+		return { ...result, llmUsage: usage };
 	}
 
 	/**