diff --git a/packages/@n8n/api-types/src/index.ts b/packages/@n8n/api-types/src/index.ts index 683d7f2a58e..91942ffa547 100644 --- a/packages/@n8n/api-types/src/index.ts +++ b/packages/@n8n/api-types/src/index.ts @@ -710,6 +710,7 @@ export type { InstanceAiEvalMockHints, InstanceAiEvalMockedCredential, InstanceAiEvalRewrittenCredential, + InstanceAiEvalLlmUsage, InstanceAiEvalExecutionResult, InstanceAiEvalAgentToolCallRecord, InstanceAiEvalAgentModelTurnRecord, diff --git a/packages/@n8n/api-types/src/schemas/instance-ai.schema.ts b/packages/@n8n/api-types/src/schemas/instance-ai.schema.ts index 8bb439a1782..221b56482ba 100644 --- a/packages/@n8n/api-types/src/schemas/instance-ai.schema.ts +++ b/packages/@n8n/api-types/src/schemas/instance-ai.schema.ts @@ -2876,6 +2876,19 @@ export interface InstanceAiEvalRewrittenCredential { field: string; } +/** Token usage of the eval's own model calls (mocks, judges, simulated user), per agent and model. */ +export interface InstanceAiEvalLlmUsage { + /** Eval agent name, e.g. `eval-mock-responder`. */ + agent: string; + model: string; + calls: number; + /** Prompt tokens billed at the full input price: cache reads and writes are not included. */ + uncachedInputTokens: number; + cacheReadTokens: number; + cacheWriteTokens: number; + outputTokens: number; +} + export interface InstanceAiEvalExecutionResult { executionId: string; success: boolean; @@ -2884,6 +2897,8 @@ export interface InstanceAiEvalExecutionResult { hints: InstanceAiEvalMockHints; mockedCredentials: InstanceAiEvalMockedCredential[]; rewrittenCredentials?: InstanceAiEvalRewrittenCredential[]; + /** Model usage of the mocks, pin data and hints for this run. */ + llmUsage?: InstanceAiEvalLlmUsage[]; } export class InstanceAiEvalExecutionRequest extends Z.class({ @@ -2994,6 +3009,8 @@ export interface InstanceAiEvalAgentExecutionResult { seed: InstanceAiEvalAgentScenarioSeed; skippedFeatures: InstanceAiEvalAgentSkippedFeature[]; mockedCredentials: InstanceAiEvalMockedCredential[]; + /** Model usage of the seed and the mocks for this run. The agent's own model call is in `usage`. */ + llmUsage?: InstanceAiEvalLlmUsage[]; } export class InstanceAiEvalAgentExecutionRequest extends Z.class({ diff --git a/packages/@n8n/instance-ai/evaluations/README.md b/packages/@n8n/instance-ai/evaluations/README.md index f3c651bbb90..2912a71f5ec 100644 --- a/packages/@n8n/instance-ai/evaluations/README.md +++ b/packages/@n8n/instance-ai/evaluations/README.md @@ -208,6 +208,9 @@ Every run produces: - **Console** — live progress, per-scenario pass/fail with `[failure_category]` tag, and a grouped summary. - **`eval-results.json`** — structured results in `--output-dir` (or cwd). Consumed by the CI PR comment. + `harnessUsagePerRun` gives the tokens of the eval's own model calls for each iteration of a case: + judges, simulated user, mocks, pin data and hints. Each entry is one agent on one model. + `uncachedInputTokens` excludes cache reads and writes. `null` means the iteration was not measured. - **`.data/workflow-eval-report.html`** — self-contained debugging view with a green/red stage review for prompt, planner, builder, and verifier behavior, generalized prompt-improvement suggestions for failures, per-node execution traces, intercepted requests, mock responses, Phase 1 hints, verifier reasoning, and the per-built-workflow check rubric (see below). - **LangSmith experiment** — only when `LANGSMITH_API_KEY` is set. See the caveat in [Environment variables](#environment-variables). diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/case-pipeline.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/case-pipeline.test.ts index 9fa14a180c5..44664ecf613 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/case-pipeline.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/case-pipeline.test.ts @@ -1,3 +1,6 @@ +import type { InstanceAiEvalLlmUsage } from '@n8n/api-types'; + +import { EvalUsageMeter, recordEvalUsage } from '../../src/utils/eval-usage'; import type { CliArgs } from '../cli/args'; import type { BuildResult } from '../harness/build-workflow'; import { cleanupBuild } from '../harness/cleanup'; @@ -113,16 +116,57 @@ function makeDeps( buildExpectationsByKey: new Map(), agentContextByKey: new Map(), runDebugByThreadId: new Map(), + harnessUsageByKey: new Map(), ...overrides, }; } +function usageOf(agent: string, calls: number): InstanceAiEvalLlmUsage { + return { + agent, + model: 'anthropic/claude-sonnet-4-6', + calls, + uncachedInputTokens: 100 * calls, + cacheReadTokens: 0, + cacheWriteTokens: 0, + outputTokens: 10 * calls, + }; +} + beforeEach(() => { vi.mocked(cleanupBuild).mockClear(); vi.mocked(cleanupBuild).mockResolvedValue(true); }); describe('createCasePipeline', () => { + it('counts the model usage of each row and of its build under the case and iteration', async () => { + const lane = makeLane(); + // The scenario run reports the server's mock usage, the build the simulated user's. + vi.mocked(lane.tracedExecute).mockImplementation(async () => { + recordEvalUsage([usageOf('eval-mock-responder', 2)]); + return await Promise.resolve({ success: true, score: 1, reasoning: 'works' } as never); + }); + const orchestrator = makeOrchestrator({ build: okBuild(), lane, buildDurationMs: 42 }); + vi.mocked(orchestrator.getOrBuild).mockImplementation(async () => { + recordEvalUsage([usageOf('eval-user-proxy', 1)]); + return await Promise.resolve({ build: okBuild(), lane, buildDurationMs: 42 }); + }); + const harnessUsageByKey = new Map(); + const pipeline = createCasePipeline(makeDeps(orchestrator, { harnessUsageByKey })); + + await Promise.all([ + pipeline.runRow({ ...rowInputs('happy-path'), _iteration: 0 }), + pipeline.runRow({ ...rowInputs('happy-path'), _iteration: 1 }), + ]); + + for (const key of ['0:case-a', '1:case-a']) { + expect(harnessUsageByKey.get(key)?.entries()).toEqual([ + usageOf('eval-user-proxy', 1), + usageOf('eval-mock-responder', 2), + ]); + } + }); + it('runs a workflow scenario row and eagerly cleans up after the last row', async () => { const lane = makeLane(); vi.mocked(lane.tracedExecute).mockResolvedValue({ diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/eval-results-dispatcher-contract.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/eval-results-dispatcher-contract.test.ts index 53b772c41d7..e7eb351156e 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/eval-results-dispatcher-contract.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/eval-results-dispatcher-contract.test.ts @@ -1,4 +1,4 @@ -import type { InstanceAiEvalExecutionResult } from '@n8n/api-types'; +import type { InstanceAiEvalExecutionResult, InstanceAiEvalLlmUsage } from '@n8n/api-types'; import { mkdtempSync, readFileSync } from 'fs'; import { jsonParse } from 'n8n-workflow'; import { tmpdir } from 'os'; @@ -152,6 +152,7 @@ interface DispatcherView { }> | null>; buildCostUsdPerRun?: Array; buildTurnsPerRun?: Array; + harnessUsagePerRun: Array; transcriptPerRun: Array; buildErrorPerRun: Array; threadIds: Array; @@ -391,6 +392,38 @@ describe('eval-results.json — dispatcher contract', () => { ); }); + it('serializes the harness model usage of each iteration, null when it was not measured', () => { + const judgeUsage: InstanceAiEvalLlmUsage = { + agent: 'eval-checklist-verifier', + model: 'anthropic/claude-sonnet-4-6', + calls: 3, + uncachedInputTokens: 41_000, + cacheReadTokens: 12_000, + cacheWriteTokens: 4_000, + outputTokens: 2_500, + }; + const evaluation = aggregateResults( + [[{ ...iteration1(), harnessUsage: [judgeUsage] }], [iteration2()]], + 2, + ); + const dir = mkdtempSync(join(tmpdir(), 'eval-results-contract-')); + const { jsonPath } = writeEvalResults( + evaluation, + 1234, + dir, + 'exp-dispatcher-contract', + undefined, + undefined, + new Map([[testCase, 'daily-digest']]), + undefined, + undefined, + ); + const report = jsonParse(readFileSync(jsonPath, 'utf8')); + + // LangTracer prices these per model. A null run is "not measured", never "free". + expect(report.testCases[0].harnessUsagePerRun).toEqual([[judgeUsage], null]); + }); + it('serializes per-iteration `claude` build spend when a run recorded it', () => { const evaluation = aggregateResults( [[{ ...iteration1(), buildCostUsd: 0.31, buildTurns: 5 }], [iteration2()]], diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/fixtures/eval-results.partial-abort.json b/packages/@n8n/instance-ai/evaluations/__tests__/fixtures/eval-results.partial-abort.json index b220a45f13d..8e64beb9cb3 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/fixtures/eval-results.partial-abort.json +++ b/packages/@n8n/instance-ai/evaluations/__tests__/fixtures/eval-results.partial-abort.json @@ -41,7 +41,8 @@ ], "transcriptPerRun": [null], "buildErrorPerRun": [null], - "buildTimeoutPerRun": [null] + "buildTimeoutPerRun": [null], + "harnessUsagePerRun": [null] }, { "name": "build me something", @@ -76,7 +77,8 @@ ], "transcriptPerRun": [null], "buildErrorPerRun": ["TimeoutError: The operation was aborted due to timeout"], - "buildTimeoutPerRun": [null] + "buildTimeoutPerRun": [null], + "harnessUsagePerRun": [null] } ] } diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/n8n-client-eval-usage.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/n8n-client-eval-usage.test.ts new file mode 100644 index 00000000000..2d95f14fbba --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/__tests__/n8n-client-eval-usage.test.ts @@ -0,0 +1,53 @@ +import type { InstanceAiEvalLlmUsage } from '@n8n/api-types'; +import { afterEach, vi } from 'vitest'; + +import { EvalUsageMeter } from '../../src/utils/eval-usage'; +import { N8nClient } from '../clients/n8n-client'; + +const MOCK_RESPONDER_USAGE: InstanceAiEvalLlmUsage = { + agent: 'eval-mock-responder', + model: 'anthropic/claude-sonnet-4-6', + calls: 4, + uncachedInputTokens: 2400, + cacheReadTokens: 0, + cacheWriteTokens: 0, + outputTokens: 600, +}; + +function mockFetch(data: unknown) { + vi.stubGlobal( + 'fetch', + vi.fn(() => ({ + ok: true, + status: 200, + headers: { get: () => null }, + json: () => ({ data }), + text: () => '', + })), + ); +} + +afterEach(() => vi.unstubAllGlobals()); + +describe('N8nClient eval runs', () => { + it('adds the model usage a workflow scenario run reports to the current meter', async () => { + mockFetch({ executionId: 'exec-1', success: true, llmUsage: [MOCK_RESPONDER_USAGE] }); + const meter = new EvalUsageMeter(); + + await meter.run(async () => await new N8nClient('http://n8n.test').executeWithLlmMock('wf-1')); + + expect(meter.entries()).toEqual([MOCK_RESPONDER_USAGE]); + }); + + it('adds the model usage an agent scenario run reports to the current meter', async () => { + mockFetch({ runId: 'run-1', success: true, llmUsage: [MOCK_RESPONDER_USAGE] }); + const meter = new EvalUsageMeter(); + + await meter.run( + async () => + await new N8nClient('http://n8n.test').executeAgentWithLlmMock('agent-1', 'project-1'), + ); + + expect(meter.entries()).toEqual([MOCK_RESPONDER_USAGE]); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/reshape.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/reshape.test.ts index 8b8db09182d..2f89167b450 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/reshape.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/reshape.test.ts @@ -386,6 +386,44 @@ describe('reshapeLangSmithRuns', () => { expect(result[0][0]?.runDebug).toHaveLength(1); expect(result[0][0]?.runDebug?.[0]?.runId).toBe('run-1'); }); + + it('attaches each iteration its own harness model usage by iteration:fileSlug', () => { + const cases = [withFile('airtable', [scenario('s1')])]; + const rows = [0, 1].map((iteration) => + row( + { testCaseFile: 'airtable', scenarioName: 's1', _iteration: iteration }, + { buildSuccess: true, passed: true, score: 1, reasoning: 'ok' }, + ), + ); + const usage = (calls: number) => [ + { + agent: 'eval-checklist-verifier', + model: 'anthropic/claude-sonnet-4-6', + calls, + uncachedInputTokens: 1000 * calls, + cacheReadTokens: 0, + cacheWriteTokens: 0, + outputTokens: 100 * calls, + }, + ]; + const harnessUsageByKey = new Map([ + ['0:airtable', usage(1)], + ['1:airtable', usage(2)], + ]); + + const result = reshapeLangSmithRuns( + rows, + cases, + 2, + new Map(), + new Map(), + undefined, + new Map(), + harnessUsageByKey, + ); + + expect(result.map((iteration) => iteration[0]?.harnessUsage)).toEqual([usage(1), usage(2)]); + }); }); describe('sentinelOutcomeFromVerdicts', () => { diff --git a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts index 44310eea582..baf0f714b31 100644 --- a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts +++ b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts @@ -32,6 +32,8 @@ import type { ExecutionStatus } from 'n8n-workflow'; import { Agent, setGlobalDispatcher } from 'undici'; import { z } from 'zod'; +import { recordEvalUsage } from '../../src/utils/eval-usage'; + // Disable undici's 300s timeouts — mocked eval runs take minutes; the per-request // AbortSignal is the real bound. This is process-global: only ever imported by the // eval CLI harness — never import into the n8n server or shared runtime code. @@ -1350,6 +1352,7 @@ export class N8nClient { body, timeoutMs: serverBudgetMs + CLIENT_ABORT_MARGIN_MS, })) as { data: InstanceAiEvalExecutionResult }; + recordEvalUsage(result.data.llmUsage); return result.data; } @@ -1377,6 +1380,7 @@ export class N8nClient { timeoutMs: serverBudgetMs + CLIENT_ABORT_MARGIN_MS, }, )) as { data: InstanceAiEvalAgentExecutionResult }; + recordEvalUsage(result.data.llmUsage); return result.data; } diff --git a/packages/@n8n/instance-ai/evaluations/run/case-pipeline.ts b/packages/@n8n/instance-ai/evaluations/run/case-pipeline.ts index f663419bacd..5e28e8b4d5d 100644 --- a/packages/@n8n/instance-ai/evaluations/run/case-pipeline.ts +++ b/packages/@n8n/instance-ai/evaluations/run/case-pipeline.ts @@ -14,6 +14,7 @@ import type { InstanceAiRunDebugResponse } from '@n8n/api-types'; import type { BuildOrchestrator } from './build-orchestrator'; import { sentinelOutcomeFromVerdicts, type TargetOutput } from './reshape'; +import { EvalUsageMeter } from '../../src/utils/eval-usage'; import type { CliArgs } from '../cli/args'; import { draftAgentVerdict, @@ -57,6 +58,8 @@ export interface CasePipelineDeps { buildExpectationsByKey: Map>; agentContextByKey: Map>; runDebugByThreadId: Map>; + /** Usage of the eval's own model calls, per `iteration:fileSlug`. Filled here. */ + harnessUsageByKey: Map; } export interface CasePipeline { @@ -85,9 +88,19 @@ export function createCasePipeline(deps: CasePipelineDeps): CasePipeline { buildExpectationsByKey, agentContextByKey, runDebugByThreadId, + harnessUsageByKey, } = deps; const { getOrBuild, buildCache } = orchestrator; + function usageMeterFor(key: string): EvalUsageMeter { + let meter = harnessUsageByKey.get(key); + if (!meter) { + meter = new EvalUsageMeter(); + harnessUsageByKey.set(key, meter); + } + return meter; + } + // Rows remaining per `iteration:fileSlug` build. When the last row of a // build finishes, its backend artifacts (workflow, data tables, thread — and // with the thread its sandbox) are deleted right away instead of at the end @@ -147,7 +160,10 @@ export function createCasePipeline(deps: CasePipelineDeps): CasePipeline { const runRow = async (inputs: ScenarioRowInputs): Promise => { const iteration = inputs._iteration ?? 0; try { - return await runScenarioRow(inputs, iteration); + // Only rows start builds, so a build and its judges count on the meter of its own case. + return await usageMeterFor(`${String(iteration)}:${inputs.testCaseFile}`).run( + async () => await runScenarioRow(inputs, iteration), + ); } catch (error: unknown) { // runScenarioRow guards scenario execution internally, but a build-phase // throw (getOrBuild) or a budget abort must not reject up to the driver diff --git a/packages/@n8n/instance-ai/evaluations/run/direct-driver.ts b/packages/@n8n/instance-ai/evaluations/run/direct-driver.ts index 49a4cf7994b..2c1d6a685b2 100644 --- a/packages/@n8n/instance-ai/evaluations/run/direct-driver.ts +++ b/packages/@n8n/instance-ai/evaluations/run/direct-driver.ts @@ -92,6 +92,7 @@ export async function runDirect(config: DirectRunConfig): Promise<{ sideBand.buildExpectations, lanes[0]?.baseUrl, sideBand.runDebug, + sideBand.harnessUsage, ); for (const iterationResults of allRunResults) { partialResults?.push(iterationResults); diff --git a/packages/@n8n/instance-ai/evaluations/run/eval-session.ts b/packages/@n8n/instance-ai/evaluations/run/eval-session.ts index cbd79bdacca..540972319fb 100644 --- a/packages/@n8n/instance-ai/evaluations/run/eval-session.ts +++ b/packages/@n8n/instance-ai/evaluations/run/eval-session.ts @@ -9,6 +9,7 @@ // --------------------------------------------------------------------------- import type { + InstanceAiEvalLlmUsage, InstanceAiEvalThreadMemoryResponse, InstanceAiRunDebugResponse, } from '@n8n/api-types'; @@ -25,6 +26,7 @@ import { import { createCasePipeline, type CasePipeline } from './case-pipeline'; import { LaneAllocator } from './lane-allocator'; import { provisionBuildProject } from './lane-users'; +import type { EvalUsageMeter } from '../../src/utils/eval-usage'; import type { CliArgs } from '../cli/args'; import { N8nClient } from '../clients/n8n-client'; import type { WorkflowTestCaseWithFile } from '../data/workflows'; @@ -80,6 +82,8 @@ export interface ResolvedSideBand { * builds (no threadId) still attach their outcome-expectation verdicts. */ buildExpectations: Map; runDebug: Map; + /** Keyed by the build-cache key (`iteration:fileSlug`). */ + harnessUsage: Map; } export interface EvalSession { @@ -262,6 +266,7 @@ export function createEvalSession(config: EvalSessionConfig): EvalSession { // Agent config + skills, fetched once per build and shared by every // scenario row of the case (the agent analog of the cached workflow JSON). const agentContextByKey = new Map>(); + const harnessUsageByKey = new Map(); const orchestrator = createBuildOrchestrator({ args, @@ -288,6 +293,7 @@ export function createEvalSession(config: EvalSessionConfig): EvalSession { buildExpectationsByKey, agentContextByKey, runDebugByThreadId, + harnessUsageByKey, }); const resolveSideBand = async (): Promise => { @@ -299,7 +305,10 @@ export function createEvalSession(config: EvalSessionConfig): EvalSession { for (const [threadId, runDebugPromise] of runDebugByThreadId) { runDebug.set(threadId, await runDebugPromise); } - return { transcriptByThreadId, buildExpectations, runDebug }; + const harnessUsage = new Map( + [...harnessUsageByKey].map(([key, meter]) => [key, meter.entries()]), + ); + return { transcriptByThreadId, buildExpectations, runDebug, harnessUsage }; }; const drainBuilds = async (): Promise => { diff --git a/packages/@n8n/instance-ai/evaluations/run/langsmith-driver.ts b/packages/@n8n/instance-ai/evaluations/run/langsmith-driver.ts index e4b50dbb71e..34ade3e32ea 100644 --- a/packages/@n8n/instance-ai/evaluations/run/langsmith-driver.ts +++ b/packages/@n8n/instance-ai/evaluations/run/langsmith-driver.ts @@ -253,6 +253,7 @@ export async function runWithLangSmith(config: RunConfig): Promise<{ sideBand.buildExpectations, lanes[0]?.baseUrl, sideBand.runDebug, + sideBand.harnessUsage, ); const evaluation = aggregateResults(allRunResults, args.iterations); diff --git a/packages/@n8n/instance-ai/evaluations/run/persist.ts b/packages/@n8n/instance-ai/evaluations/run/persist.ts index a85fb3c56f5..5baba715cb6 100644 --- a/packages/@n8n/instance-ai/evaluations/run/persist.ts +++ b/packages/@n8n/instance-ai/evaluations/run/persist.ts @@ -524,6 +524,10 @@ export function writeEvalResults( buildTurnsPerRun: tc.runs.map((run) => run.buildTurns ?? null), } : {}), + // Tokens of the eval's own model calls (judges, simulated user, mocks) per + // iteration, by agent and model. Null when the iteration was not measured, + // e.g. one recovered from the row journal after a crash. + harnessUsagePerRun: tc.runs.map((run) => run.harnessUsage ?? null), scenarios: tc.executionScenarios.map((sa) => ({ name: sa.scenario.name, passCount: sa.passCount, diff --git a/packages/@n8n/instance-ai/evaluations/run/reshape.ts b/packages/@n8n/instance-ai/evaluations/run/reshape.ts index fe11444ec96..7e23e52577d 100644 --- a/packages/@n8n/instance-ai/evaluations/run/reshape.ts +++ b/packages/@n8n/instance-ai/evaluations/run/reshape.ts @@ -10,6 +10,7 @@ import type { InstanceAiEvalAgentExecutionResult, InstanceAiEvalExecutionResult, + InstanceAiEvalLlmUsage, InstanceAiRunDebugResponse, } from '@n8n/api-types'; import { isRecord } from '@n8n/utils/is-record'; @@ -262,6 +263,8 @@ export function reshapeLangSmithRuns( buildExpectationsByKey: Map, n8nBaseUrl: string | undefined, runDebugByThreadId: Map = new Map(), + /** Keyed by the build-cache key (`iteration:fileSlug`). */ + harnessUsageByKey: Map = new Map(), ): WorkflowTestCaseResult[][] { // Index runs by (iteration, testCaseFile, scenarioName) using the `_iteration` // we injected in expandExamplesForIterations. Falls back to 0 for single-run. @@ -365,7 +368,8 @@ export function reshapeLangSmithRuns( } const transcript = threadId ? transcriptByThreadId.get(threadId) : undefined; - const buildExpectationResults = buildExpectationsByKey.get(`${String(iter)}:${fileSlug}`); + const buildKey = `${String(iter)}:${fileSlug}`; + const buildExpectationResults = buildExpectationsByKey.get(buildKey); runResults.push({ testCase, fileSlug, @@ -387,6 +391,7 @@ export function reshapeLangSmithRuns( buildTurns, n8nBaseUrl, runDebug: threadId ? runDebugByThreadId.get(threadId) : undefined, + harnessUsage: harnessUsageByKey.get(buildKey), }); } allRunResults.push(runResults); diff --git a/packages/@n8n/instance-ai/evaluations/types.ts b/packages/@n8n/instance-ai/evaluations/types.ts index 440f6effdbe..7d1b3707f75 100644 --- a/packages/@n8n/instance-ai/evaluations/types.ts +++ b/packages/@n8n/instance-ai/evaluations/types.ts @@ -6,6 +6,7 @@ import type { AgentSkill, InstanceAiEvalAgentExecutionResult, InstanceAiEvalExecutionResult, + InstanceAiEvalLlmUsage, InstanceAiEvalSeedDataTable, InstanceAiRunDebugResponse, InstanceAiPromptConfiguration, @@ -387,6 +388,8 @@ export interface WorkflowTestCaseResult { n8nBaseUrl?: string; /** Per-run LLM step debug captured from the instance-ai debug API after build. */ runDebug?: InstanceAiRunDebugResponse[]; + /** Usage of the eval's own model calls in this iteration: judges, simulated user, mocks. */ + harnessUsage?: InstanceAiEvalLlmUsage[]; } // --------------------------------------------------------------------------- diff --git a/packages/@n8n/instance-ai/src/index.ts b/packages/@n8n/instance-ai/src/index.ts index 73e158033ad..5a4a17de891 100644 --- a/packages/@n8n/instance-ai/src/index.ts +++ b/packages/@n8n/instance-ai/src/index.ts @@ -35,6 +35,7 @@ import type * as LangsmithTracingMod from './tracing/langsmith-tracing'; import type * as TraceReplayMod from './tracing/trace-replay'; import type * as AgentTreeMod from './utils/agent-tree'; import type * as EvalAgentsMod from './utils/eval-agents'; +import type * as EvalUsageMod from './utils/eval-usage'; import type * as StreamHelpersMod from './utils/stream-helpers'; import type * as WorkflowLoopMod from './workflow-loop'; import type * as WorkflowLoopRuntimeMod from './workflow-loop/runtime'; @@ -147,6 +148,7 @@ const loadMaterializeRuntimeSkills = lazyModule( () => require('./skills/materialize-runtime-skills') as typeof MaterializeRuntimeSkillsMod, ); const loadEvalAgents = lazyModule(() => require('./utils/eval-agents') as typeof EvalAgentsMod); +const loadEvalUsage = lazyModule(() => require('./utils/eval-usage') as typeof EvalUsageMod); const loadAgentTree = lazyModule(() => require('./utils/agent-tree') as typeof AgentTreeMod); const loadBuilderTemplatesService = lazyModule( () => require('./workspace/builder-templates-service') as typeof BuilderTemplatesServiceMod, @@ -450,6 +452,9 @@ export const createEvalAgent: typeof EvalAgentsMod.createEvalAgent = lazyFunctio export const extractText: typeof EvalAgentsMod.extractText = lazyFunction( () => loadEvalAgents().extractText, ); +export const meterEvalUsage: typeof EvalUsageMod.meterEvalUsage = lazyFunction( + () => loadEvalUsage().meterEvalUsage, +); defineLazyExport('PURE_REPLAY_TOOLS', () => loadTraceReplay().PURE_REPLAY_TOOLS); defineLazyExport( 'SUB_AGENT_RESOURCE_PREFIX', diff --git a/packages/@n8n/instance-ai/src/utils/__tests__/eval-agents.test.ts b/packages/@n8n/instance-ai/src/utils/__tests__/eval-agents.test.ts index cbab43e6388..0f8f1747f2c 100644 --- a/packages/@n8n/instance-ai/src/utils/__tests__/eval-agents.test.ts +++ b/packages/@n8n/instance-ai/src/utils/__tests__/eval-agents.test.ts @@ -4,6 +4,7 @@ type MockAgentInstance = { model: Mock; instructions: Mock; thinking: Mock; + configuration: Mock; }; const mockAgentInstances: MockAgentInstance[] = []; @@ -13,6 +14,7 @@ vi.mock('@n8n/agents', () => ({ this.model = vi.fn().mockReturnThis(); this.instructions = vi.fn().mockReturnThis(); this.thinking = vi.fn().mockReturnThis(); + this.configuration = vi.fn().mockReturnThis(); mockAgentInstances.push(this); }), Tool: vi.fn(), diff --git a/packages/@n8n/instance-ai/src/utils/__tests__/eval-usage.test.ts b/packages/@n8n/instance-ai/src/utils/__tests__/eval-usage.test.ts new file mode 100644 index 00000000000..470d5a43f67 --- /dev/null +++ b/packages/@n8n/instance-ai/src/utils/__tests__/eval-usage.test.ts @@ -0,0 +1,138 @@ +import type { InstanceAiEvalLlmUsage } from '@n8n/api-types'; +import { convertArrayToReadableStream, MockLanguageModelV3 } from 'ai/test'; + +import { createEvalAgent } from '../eval-agents'; +import { EvalUsageMeter, recordEvalUsage } from '../eval-usage'; + +const ORIGINAL_ENV = { ...process.env }; + +/** A model that answers "ok" to every call and reports these token counts. */ +function modelReporting(tokens: { + noCache: number; + cacheRead: number; + cacheWrite: number; + output: number; +}): MockLanguageModelV3 { + const finishReason = { unified: 'stop', raw: 'end_turn' } as const; + const usage = { + inputTokens: { + total: tokens.noCache + tokens.cacheRead + tokens.cacheWrite, + noCache: tokens.noCache, + cacheRead: tokens.cacheRead, + cacheWrite: tokens.cacheWrite, + }, + outputTokens: { total: tokens.output, text: tokens.output, reasoning: 0 }, + }; + return new MockLanguageModelV3({ + provider: 'mock', + modelId: 'judge', + doGenerate: { content: [{ type: 'text', text: 'ok' }], finishReason, usage, warnings: [] }, + doStream: async () => + await Promise.resolve({ + stream: convertArrayToReadableStream([ + { type: 'stream-start', warnings: [] }, + { type: 'text-start', id: 'text-1' }, + { type: 'text-delta', id: 'text-1', delta: 'ok' }, + { type: 'text-end', id: 'text-1' }, + { type: 'finish', finishReason, usage }, + ]), + }), + }); +} + +/** No eval key in the environment, so the agent runs on the mock model. */ +function judgeOn(model: MockLanguageModelV3, name = 'eval-checklist-verifier') { + return createEvalAgent(name, { instructions: 'Judge the run.', fallbackModelConfig: model }); +} + +describe('eval usage meter', () => { + beforeEach(() => { + process.env = { ...ORIGINAL_ENV }; + for (const key of [ + 'N8N_INSTANCE_AI_MODEL', + 'N8N_INSTANCE_AI_EVAL_MODEL', + 'N8N_INSTANCE_AI_MODEL_API_KEY', + 'N8N_INSTANCE_AI_MODEL_URL', + 'EVAL_MODAL_LLM_HEADERS', + 'N8N_AI_ANTHROPIC_KEY', + 'ANTHROPIC_API_KEY', + ]) { + delete process.env[key]; + } + }); + + afterAll(() => { + process.env = ORIGINAL_ENV; + }); + + it('counts every call an eval agent makes inside the meter, by agent and model', async () => { + const judge = judgeOn( + modelReporting({ noCache: 100, cacheRead: 1000, cacheWrite: 200, output: 50 }), + ); + const meter = new EvalUsageMeter(); + + await meter.run(async () => { + await judge.generate('First scenario'); + await judge.generate('Second scenario'); + }); + + expect(meter.entries()).toEqual([ + { + agent: 'eval-checklist-verifier', + model: 'mock/judge', + calls: 2, + uncachedInputTokens: 200, + cacheReadTokens: 2000, + cacheWriteTokens: 400, + outputTokens: 100, + }, + ]); + }); + + it('keeps the calls of concurrent meters apart', async () => { + const judge = judgeOn(modelReporting({ noCache: 10, cacheRead: 0, cacheWrite: 0, output: 1 })); + const first = new EvalUsageMeter(); + const second = new EvalUsageMeter(); + + await Promise.all([ + first.run(async () => await judge.generate('Case one')), + second.run(async () => { + await judge.generate('Case two'); + await judge.generate('Case two again'); + }), + ]); + + expect(first.entries().map((entry) => entry.calls)).toEqual([1]); + expect(second.entries().map((entry) => entry.calls)).toEqual([2]); + }); + + it('adds usage that an eval endpoint reported to the current meter', async () => { + const reported: InstanceAiEvalLlmUsage = { + agent: 'eval-mock-responder', + model: 'anthropic/claude-sonnet-4-6', + calls: 3, + uncachedInputTokens: 10, + cacheReadTokens: 20, + cacheWriteTokens: 30, + outputTokens: 5, + }; + const meter = new EvalUsageMeter(); + + await meter.run(async () => { + recordEvalUsage([reported]); + recordEvalUsage([reported]); + await Promise.resolve(); + }); + + expect(meter.entries()).toEqual([ + { + ...reported, + calls: 6, + uncachedInputTokens: 20, + cacheReadTokens: 40, + cacheWriteTokens: 60, + outputTokens: 10, + }, + ]); + }); +}); diff --git a/packages/@n8n/instance-ai/src/utils/eval-agents.ts b/packages/@n8n/instance-ai/src/utils/eval-agents.ts index a5dcac4d7de..fca381b840b 100644 --- a/packages/@n8n/instance-ai/src/utils/eval-agents.ts +++ b/packages/@n8n/instance-ai/src/utils/eval-agents.ts @@ -3,6 +3,7 @@ import { Agent, Tool, type GenerateResult, type ModelConfig } from '@n8n/agents'; import { getProviderPrefix, splitModelId } from '@n8n/ai-utilities/agent-config'; +import { evalUsageGuardrail } from './eval-usage'; import { parseModelHeadersJson } from './parse-model-headers'; import { applyAgentThinking } from '../agent/apply-agent-thinking'; @@ -174,6 +175,8 @@ export function createEvalAgent( } applyAgentThinking(agent, model); + // `configuration()` replaces the defaults it was given before: put any other default option in this call. + agent.configuration({ guardrails: { hooks: [evalUsageGuardrail(name)] } }); return agent; } diff --git a/packages/@n8n/instance-ai/src/utils/eval-usage.ts b/packages/@n8n/instance-ai/src/utils/eval-usage.ts new file mode 100644 index 00000000000..f07300d15cb --- /dev/null +++ b/packages/@n8n/instance-ai/src/utils/eval-usage.ts @@ -0,0 +1,77 @@ +/** Adds up the token usage of eval agent calls, so an eval run can report what its own model calls cost. */ + +import type { ModelGuardrail, TokenUsage } from '@n8n/agents'; +import type { InstanceAiEvalLlmUsage } from '@n8n/api-types'; +import { AsyncLocalStorage } from 'node:async_hooks'; + +const currentMeter = new AsyncLocalStorage(); + +export class EvalUsageMeter { + private readonly usage = new Map(); + + /** Every eval agent call made inside `fn`, at any depth, adds to this meter. */ + async run(fn: () => Promise): Promise { + return await currentMeter.run(this, fn); + } + + add(entry: InstanceAiEvalLlmUsage): void { + const key = JSON.stringify([entry.agent, entry.model]); + const total = this.usage.get(key); + if (!total) { + this.usage.set(key, { ...entry }); + return; + } + total.calls += entry.calls; + total.uncachedInputTokens += entry.uncachedInputTokens; + total.cacheReadTokens += entry.cacheReadTokens; + total.cacheWriteTokens += entry.cacheWriteTokens; + total.outputTokens += entry.outputTokens; + } + + entries(): InstanceAiEvalLlmUsage[] { + return [...this.usage.values()].map((entry) => ({ ...entry })); + } +} + +/** Adds usage reported by an eval endpoint to the current meter. Does nothing outside a meter. */ +export function recordEvalUsage(entries: InstanceAiEvalLlmUsage[] | undefined): void { + const meter = currentMeter.getStore(); + if (!meter || !entries) return; + for (const entry of entries) meter.add(entry); +} + +/** Runs `fn` on a new meter and returns the usage it counted. */ +export async function meterEvalUsage( + fn: () => Promise, +): Promise<{ result: T; usage: InstanceAiEvalLlmUsage[] }> { + const meter = new EvalUsageMeter(); + const result = await meter.run(fn); + return { result, usage: meter.entries() }; +} + +function toEvalLlmUsage(agent: string, model: string, usage: TokenUsage): InstanceAiEvalLlmUsage { + const cacheReadTokens = usage.inputTokenDetails?.cacheRead ?? 0; + const cacheWriteTokens = usage.inputTokenDetails?.cacheWrite ?? 0; + return { + agent, + model, + calls: 1, + // A zero `noCache` is left out of the details, so derive it from the total. + uncachedInputTokens: + usage.inputTokenDetails?.noCache ?? + Math.max(usage.promptTokens - cacheReadTokens - cacheWriteTokens, 0), + cacheReadTokens, + cacheWriteTokens, + outputTokens: usage.completionTokens, + }; +} + +/** Runs after each model call of the agent named `agent` and adds its usage to the current meter. */ +export function evalUsageGuardrail(agent: string): ModelGuardrail { + return { + async after(ctx, usage) { + const meter = currentMeter.getStore(); + if (meter && usage) meter.add(toEvalLlmUsage(agent, ctx.model, usage)); + }, + }; +} diff --git a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts index daee39fb29c..b874220ddb4 100644 --- a/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts +++ b/packages/cli/src/modules/instance-ai/__tests__/instance-ai.controller.test.ts @@ -25,6 +25,10 @@ vi.mock('@n8n/instance-ai', () => { ...(prior ?? {}), boundTargets: prior?.boundTargets, })), + meterEvalUsage: vi.fn(async (fn: () => Promise) => ({ + result: await fn(), + usage: [], + })), }; }); @@ -40,7 +44,10 @@ import type { AiPreferenceDto, InstanceAiPreferenceCardEvent, InstanceAiAdminSettingsUpdateRequest, + InstanceAiEvalAgentExecutionResult, InstanceAiEvalCredentialAllowlistRequest, + InstanceAiEvalExecutionResult, + InstanceAiEvalLlmUsage, InstanceAiEvalRestoreThreadRequest, InstanceAiEvalThreadMemoryResponse, InstanceAiSendMessageRequest, @@ -59,7 +66,11 @@ import type { } from '@n8n/api-types'; import type { ModuleRegistry } from '@n8n/backend-common'; import type { GlobalConfig } from '@n8n/config'; -import { buildAgentTreeFromEvents, seedAgentBuilderTargetMetadata } from '@n8n/instance-ai'; +import { + buildAgentTreeFromEvents, + meterEvalUsage, + seedAgentBuilderTargetMetadata, +} from '@n8n/instance-ai'; import { InstanceAiPersistPendingAgentRequest, InstanceAiThreadTabsRequestDto, @@ -116,6 +127,16 @@ function scopeOf(handlerName: string): { scope: Scope; globalOnly: boolean } | u return route.accessScope; } +const MOCK_RESPONDER_USAGE: InstanceAiEvalLlmUsage = { + agent: 'eval-mock-responder', + model: 'anthropic/claude-sonnet-4-6', + calls: 2, + uncachedInputTokens: 1200, + cacheReadTokens: 0, + cacheWriteTokens: 0, + outputTokens: 300, +}; + describe('InstanceAiController', () => { const instanceAiService = mock(); const gatewayService = mock(); @@ -142,6 +163,8 @@ describe('InstanceAiController', () => { const projectService = mock(); const instanceAiErrorReporter = mock(); + const evalExecutionService = mock(); + const evalAgentExecutionService = mock(); const evalCredentialAllowlists = new EvalThreadCredentialAllowlistService(); const evalThreadRestore = mock(); const preferenceCardService = mock(); @@ -157,8 +180,8 @@ describe('InstanceAiController', () => { pendingAgentService, settingsService, modelCatalogService, - mock(), - mock(), + evalExecutionService, + evalAgentExecutionService, evalCredentialAllowlists, evalThreadRestore, eventBus, @@ -1010,6 +1033,41 @@ describe('InstanceAiController', () => { it('should require instanceAi:eval scope', () => { expect(scopeOf('executeWithLlmMock')).toEqual({ scope: 'instanceAi:eval', globalOnly: true }); }); + + it('should return the model usage of the run with its result', async () => { + const runResult = { executionId: 'exec-1', success: true } as InstanceAiEvalExecutionResult; + evalExecutionService.executeWithLlmMock.mockResolvedValue(runResult); + vi.mocked(meterEvalUsage).mockImplementationOnce(async (fn) => ({ + result: await fn(), + usage: [MOCK_RESPONDER_USAGE], + })); + + const result = await controller.executeWithLlmMock(req, res, 'wf-1', {}); + + expect(result).toEqual({ ...runResult, llmUsage: [MOCK_RESPONDER_USAGE] }); + expect(evalExecutionService.executeWithLlmMock).toHaveBeenCalledWith('wf-1', req.user, {}); + }); + }); + + describe('executeAgentWithLlmMock', () => { + it('should return the model usage of the run with its result', async () => { + const runResult = { runId: 'run-1', success: true } as InstanceAiEvalAgentExecutionResult; + evalAgentExecutionService.executeWithLlmMock.mockResolvedValue(runResult); + vi.mocked(meterEvalUsage).mockImplementationOnce(async (fn) => ({ + result: await fn(), + usage: [MOCK_RESPONDER_USAGE], + })); + const payload = { projectId: 'project-1' }; + + const result = await controller.executeAgentWithLlmMock(req, res, 'agent-1', payload); + + expect(result).toEqual({ ...runResult, llmUsage: [MOCK_RESPONDER_USAGE] }); + expect(evalAgentExecutionService.executeWithLlmMock).toHaveBeenCalledWith( + 'agent-1', + req.user, + payload, + ); + }); }); describe('setThreadCredentialAllowlist', () => { diff --git a/packages/cli/src/modules/instance-ai/instance-ai.controller.ts b/packages/cli/src/modules/instance-ai/instance-ai.controller.ts index c729fbb0ebf..a6abe0cbaad 100644 --- a/packages/cli/src/modules/instance-ai/instance-ai.controller.ts +++ b/packages/cli/src/modules/instance-ai/instance-ai.controller.ts @@ -58,6 +58,7 @@ import { hasGlobalScope } from '@n8n/permissions'; import { buildAgentTreeFromEvents, clearedAgentBuilderTargetMetadata, + meterEvalUsage, seedAgentBuilderTargetMetadata, } from '@n8n/instance-ai'; import { @@ -1160,7 +1161,11 @@ export class InstanceAiController { @Param('workflowId') workflowId: string, @Body payload: InstanceAiEvalExecutionRequest, ) { - return await this.evalExecutionService.executeWithLlmMock(workflowId, req.user, payload); + // The mocks run on the eval model; the caller adds their usage to the case's cost. + const { result, usage } = await meterEvalUsage( + async () => await this.evalExecutionService.executeWithLlmMock(workflowId, req.user, payload), + ); + return { ...result, llmUsage: usage }; } // Runs for minutes; same client timeout handling as the workflow variant. @@ -1172,7 +1177,11 @@ export class InstanceAiController { @Param('agentId') agentId: string, @Body payload: InstanceAiEvalAgentExecutionRequest, ) { - return await this.evalAgentExecutionService.executeWithLlmMock(agentId, req.user, payload); + const { result, usage } = await meterEvalUsage( + async () => + await this.evalAgentExecutionService.executeWithLlmMock(agentId, req.user, payload), + ); + return { ...result, llmUsage: usage }; } /**