test(core): Add a routing mode to the Assistant discovery evals (no-changelog) (#40288)

packages/@n8n/instance-ai/evaluations/discovery/cli.ts

@@ -6,16 +6,26 @@
 //   pnpm eval:discovery                                # run all scenarios, 3 trials each
 //   pnpm eval:discovery --filter slack-oauth --verbose
 //   pnpm eval:discovery --trials 5
+//   pnpm eval:discovery --cases-dir <dir>              # routing mode
 //
 // Loads scenarios from evaluations/data/discovery/, runs each scenario × N
 // trials via the in-process runner, reports per-scenario pass-rates, exits
 // non-zero on any scenario below threshold, or on any scenario with zero passes
 // when --fail-on-zero-pass is set.
+//
+// Routing mode (`--cases-dir`) loads `route-*.json` cases from that folder, in
+// the format LangTracer exports. A judge ends each trial as soon as the
+// Assistant picks a route. The CLI prints a pass rate per bucket.
+// It always exits 0: it measures routing and does not gate a merge.
 // ---------------------------------------------------------------------------
 
-import { runDiscoveryScenario, type DiscoveryRunResult } from './runner';
+import { runDiscoveryScenario, runOrchestratorTurn, type DiscoveryRunResult } from './runner';
 import type { DiscoveryTestCase } from './types';
+import { isAgentFeatureEnabled } from '../../src/utils/agent-feature-enabled';
 import { loadDiscoveryTestCasesWithFiles } from '../data/discovery';
+import { loadRoutingCases, ROUTING_BUCKETS, type RoutingBucket } from '../routing/cases';
+import { casePasses, createRouteWatcher, routeLabel, trialPasses } from '../routing/grade';
+import { judgeRoute } from '../routing/judge';
 
 // ---------------------------------------------------------------------------
 // Args
@@ -33,6 +43,8 @@ interface CliArgs {
 	concurrency: number;
 	nodesJsonPath?: string;
 	failOnZeroPass: boolean;
+	/** Folder of routing cases. Set switches the CLI to routing mode. */
+	casesDir?: string;
 }
 
 const DEFAULT_MODEL = process.env.N8N_INSTANCE_AI_EVAL_MODEL ?? 'anthropic/claude-sonnet-4-6';
@@ -125,6 +137,14 @@ function parseArgs(argv: string[]): CliArgs {
 			case '--fail-on-zero-pass':
 				args.failOnZeroPass = true;
 				break;
+			case '--cases-dir':
+				args.casesDir = argv[++i];
+				// Without a directory the CLI would fall back to the full discovery suite.
+				if (!args.casesDir) {
+					console.error('--cases-dir needs a directory of routing cases.');
+					process.exit(1);
+				}
+				break;
 			default:
 				break;
 		}
@@ -133,6 +153,16 @@ function parseArgs(argv: string[]): CliArgs {
 	return args;
 }
 
+/** Runs `args.trials` trials, at most `args.concurrency` at a time. */
+async function runTrials<T>(args: CliArgs, runTrial: () => Promise<T>): Promise<T[]> {
+	const results: T[] = [];
+	for (let i = 0; i < args.trials; i += args.concurrency) {
+		const batchSize = Math.min(args.concurrency, args.trials - i);
+		results.push(...(await Promise.all(Array.from({ length: batchSize }, runTrial))));
+	}
+	return results;
+}
+
 // ---------------------------------------------------------------------------
 // Local mode
 // ---------------------------------------------------------------------------
@@ -162,25 +192,17 @@ async function runLocalMode(args: CliArgs): Promise<void> {
 	// — keeps the load profile predictable and per-scenario reports atomic.
 	for (const { testCase, fileSlug } of cases) {
 		process.stdout.write(`▸ ${fileSlug} ... `);
-		const trialResults: DiscoveryRunResult[] = [];
-
-		for (let i = 0; i < args.trials; i += args.concurrency) {
-			const batchSize = Math.min(args.concurrency, args.trials - i);
-			const batch = await Promise.all(
-				Array.from(
-					{ length: batchSize },
-					async () =>
-						await runDiscoveryScenario({
-							scenario: testCase,
-							modelId: args.modelId,
-							maxSteps: args.maxSteps,
-							timeoutMs: args.timeoutMs,
-							...(args.nodesJsonPath ? { nodesJsonPath: args.nodesJsonPath } : {}),
-						}),
-				),
-			);
-			trialResults.push(...batch);
-		}
+		const trialResults = await runTrials(
+			args,
+			async () =>
+				await runDiscoveryScenario({
+					scenario: testCase,
+					modelId: args.modelId,
+					maxSteps: args.maxSteps,
+					timeoutMs: args.timeoutMs,
+					...(args.nodesJsonPath ? { nodesJsonPath: args.nodesJsonPath } : {}),
+				}),
+		);
 
 		const passCount = trialResults.filter((r) => r.check.pass).length;
 		const passRate = passCount / trialResults.length;
@@ -256,6 +278,87 @@ function printSummary(aggregates: ScenarioAggregate[], args: CliArgs): void {
 	}
 }
 
+// ---------------------------------------------------------------------------
+// Routing mode
+// ---------------------------------------------------------------------------
+
+function percent(passed: number, total: number): string {
+	return `${String(passed)}/${String(total)} (${total > 0 ? ((passed / total) * 100).toFixed(0) : '0'}%)`;
+}
+
+async function runRoutingMode(args: CliArgs, casesDir: string): Promise<void> {
+	const { cases, needsSetup } = loadRoutingCases(casesDir, args.filter);
+	if (needsSetup.length > 0) {
+		console.log(
+			`Skipping ${String(needsSetup.length)} case(s) that need setup (earlier messages, an open workflow or Agent, or accounts): ${needsSetup.join(', ')}`,
+		);
+	}
+	if (cases.length === 0) {
+		console.log(`No routing cases found in ${casesDir}.`);
+		return;
+	}
+	if (!isAgentFeatureEnabled()) {
+		console.warn(
+			'Warning: the Agents module is off, so the Assistant cannot build an Agent and agent cases fail. Set N8N_ENABLED_MODULES=agents,instance-ai.',
+		);
+	}
+
+	console.log(
+		`Running ${String(cases.length)} routing case(s) × ${String(args.trials)} trial(s) (model: ${args.modelId}, concurrency: ${String(args.concurrency)}).\n`,
+	);
+
+	const results: Array<{ bucket: RoutingBucket; passed: boolean }> = [];
+	let runErrors = 0;
+	let judgeErrors = 0;
+
+	for (const routingCase of cases) {
+		process.stdout.write(`▸ ${routingCase.id} ... `);
+		const trials = await runTrials(args, async () => {
+			const watcher = createRouteWatcher(judgeRoute);
+			const turn = await runOrchestratorTurn({
+				scenario: routingCase,
+				modelId: args.modelId,
+				maxSteps: args.maxSteps,
+				timeoutMs: args.timeoutMs,
+				stopBeforeTool: watcher.beforeToolCall,
+				...(args.nodesJsonPath ? { nodesJsonPath: args.nodesJsonPath } : {}),
+			});
+			const resolution = await watcher.resolve(turn);
+			return { turn, resolution, passed: trialPasses(routingCase, resolution) };
+		});
+
+		const passed = casePasses(trials.filter((t) => t.passed).length, trials.length);
+		console.log(
+			`${passed ? '✓' : '✗'} ${trials.map((t) => routeLabel(t.resolution)).join(', ')} (accepts ${routingCase.accepts.join(' | ')})`,
+		);
+		for (const [idx, { turn, resolution, passed: trialPassed }] of trials.entries()) {
+			if (turn.runError) runErrors++;
+			if (resolution.judgeError) judgeErrors++;
+			if (!args.verbose) continue;
+			console.log(
+				`  ${trialPassed ? '✓' : '✗'} trial ${String(idx + 1)} (${(turn.durationMs / 1000).toFixed(1)}s, ${turn.streamStatus}) — ${routeLabel(resolution)} from ${resolution.evidence}`,
+			);
+			if (resolution.judgeReason) console.log(`     judge: ${resolution.judgeReason}`);
+			if (resolution.judgeError) console.log(`     judge error: ${resolution.judgeError}`);
+			if (turn.runError) console.log(`     run error: ${turn.runError}`);
+		}
+
+		results.push({ bucket: routingCase.bucket, passed });
+	}
+
+	console.log('\n=== Routing summary (a case passes when 2/3 of its trials pass) ===');
+	const summaryLine = (name: string, rows: typeof results) =>
+		console.log(`  ${name.padEnd(8)} ${percent(rows.filter((r) => r.passed).length, rows.length)}`);
+	for (const name of ROUTING_BUCKETS) {
+		const rows = results.filter((r) => r.bucket === name);
+		if (rows.length > 0) summaryLine(name, rows);
+	}
+	summaryLine('total', results);
+	if (runErrors > 0 || judgeErrors > 0) {
+		console.log(`Errored trials: ${String(runErrors)} run, ${String(judgeErrors)} judge.`);
+	}
+}
+
 // ---------------------------------------------------------------------------
 // Main
 // ---------------------------------------------------------------------------
@@ -270,7 +373,11 @@ async function main(): Promise<void> {
 		process.exit(1);
 	}
 
-	await runLocalMode(args);
+	if (args.casesDir) {
+		await runRoutingMode(args, args.casesDir);
+	} else {
+		await runLocalMode(args);
+	}
 }
 
 main()

packages/@n8n/instance-ai/evaluations/discovery/confirmation-policy.ts

@@ -27,7 +27,9 @@ export type ApprovalResponder = (
 	payload: Record<string, unknown>,
 ) => Record<string, unknown> | undefined;
 
-export function buildConfirmationPolicy(scenario: DiscoveryTestCase): ConfirmationPolicy {
+export function buildConfirmationPolicy(
+	scenario: Pick<DiscoveryTestCase, 'confirmations'>,
+): ConfirmationPolicy {
 	const policy: ConfirmationPolicy = new Map();
 	for (const [toolName, answer] of Object.entries(scenario.confirmations ?? {})) {
 		policy.set(toolName, typeof answer === 'string' ? { decision: answer } : answer);

packages/@n8n/instance-ai/evaluations/discovery/expected-tools-invoked.ts

@@ -242,6 +242,8 @@ function invalidTrialReason(trial: DiscoveryTrialFacts): string | undefined {
 		case 'errored':
 		case 'suspended':
 			return `Run did not complete (${trial.runError ? `${trial.streamStatus}: ${trial.runError}` : trial.streamStatus}).`;
+		case 'stopped-on-route':
+			return undefined;
 		case 'completed':
 			return trial.unmatchedConfirmations.length > 0
 				? `Scenario declared confirmation answers for [${trial.unmatchedConfirmations.join(', ')}] that no suspension asked for, so those decisions never ran.`

packages/@n8n/instance-ai/evaluations/discovery/runner.ts

@@ -14,9 +14,15 @@
 // timeout bounds the trial, not the run: at the budget the trial stops and fails,
 // and the abandoned stream is left to unwind on its own. Scenarios or --max-steps
 // can additionally opt into an iteration cap.
+//
+// Routing mode (`stopBeforeTool`) checks each orchestrator tool call before it
+// runs, and ends the run when the check says so (see ../routing/grade.ts).
+// Routing cases have no tool expectations, so they call `runOrchestratorTurn`
+// and skip the check.
 // ---------------------------------------------------------------------------
 
 import type { InstanceAiEvent, TaskList } from '@n8n/api-types';
+import { isRecord } from '@n8n/utils/is-record';
 import { nanoid } from 'nanoid';
 
 import {
@@ -37,7 +43,14 @@ import {
 	stubMcpServerConfigs,
 	type StubMcpRegistry,
 } from './stub-mcp-registry';
-import type { DiscoveryCheckResult, DiscoveryStreamStatus, DiscoveryTestCase } from './types';
+import {
+	ORCHESTRATOR_AGENT_ID,
+	type DiscoveryCheckResult,
+	type DiscoveryScenario,
+	type DiscoveryStreamStatus,
+	type DiscoveryTestCase,
+	type PendingToolCall,
+} from './types';
 import { createInstanceAgent } from '../../src/agent/instance-agent';
 import type { InstanceAiEventBus } from '../../src/event-bus';
 import type { Logger } from '../../src/logger';
@@ -93,6 +106,50 @@ export interface DiscoveryRunResult {
 export async function runDiscoveryScenario(
 	options: DiscoveryRunOptions,
 ): Promise<DiscoveryRunResult> {
+	const turn = await runOrchestratorTurn(options);
+	const outcome = extractOutcomeFromEvents(turn.events);
+	const check = evaluateDiscoveryTrial(options.scenario, outcome, {
+		streamStatus: turn.streamStatus,
+		timeoutMs: turn.timeoutMs,
+		...(turn.runError ? { runError: turn.runError } : {}),
+		unmatchedConfirmations: turn.unmatchedConfirmations,
+	});
+
+	return {
+		scenario: options.scenario,
+		check,
+		events: turn.events,
+		outcome,
+		durationMs: turn.durationMs,
+		streamStatus: turn.streamStatus,
+		...(turn.runError ? { runError: turn.runError } : {}),
+	};
+}
+
+export interface OrchestratorTurnOptions extends Omit<DiscoveryRunOptions, 'scenario'> {
+	scenario: DiscoveryScenario;
+	/**
+	 * Runs before each orchestrator tool call, and the call waits for it.
+	 * `true` ends the run there, before the call runs.
+	 */
+	stopBeforeTool?: (call: PendingToolCall, events: readonly InstanceAiEvent[]) => Promise<boolean>;
+}
+
+export interface OrchestratorTurnResult {
+	events: CapturedEvent[];
+	/** The same events with their schema types. */
+	instanceEvents: InstanceAiEvent[];
+	durationMs: number;
+	streamStatus: DiscoveryStreamStatus;
+	runError?: string;
+	timeoutMs: number;
+	unmatchedConfirmations: string[];
+}
+
+/** Runs one orchestrator turn and returns what it published, with no check. */
+export async function runOrchestratorTurn(
+	options: OrchestratorTurnOptions,
+): Promise<OrchestratorTurnResult> {
 	const started = Date.now();
 	// Unset by default: the orchestrator legitimately explores past any small fixed cap
 	// (data-table-workflow needs >8 iterations). Unset still lands on the SDK's own
@@ -100,10 +157,12 @@ export async function runDiscoveryScenario(
 	const maxSteps = options.scenario?.maxSteps ?? options.maxSteps;
 	const timeoutMs = options.scenario?.timeoutMs ?? options.timeoutMs ?? 60_000;
 	const nodesJsonPath = options.nodesJsonPath ?? defaultNodesJsonPath();
+	const { stopBeforeTool } = options;
 
 	const events: CapturedEvent[] = [];
+	const instanceEvents: InstanceAiEvent[] = [];
 
-	let streamStatus: DiscoveryRunResult['streamStatus'] = 'completed';
+	let streamStatus: DiscoveryStreamStatus = 'completed';
 	let runError: string | undefined;
 
 	const confirmationPolicy = buildConfirmationPolicy(options.scenario);
@@ -118,6 +177,14 @@ export async function runDiscoveryScenario(
 			resolve('timed-out');
 		}, timeoutMs);
 	});
+	let stopRun = () => {};
+	const routeStopped = new Promise<'stopped-on-route'>((resolve) => {
+		stopRun = () => {
+			// Resolve first, so the race ends before the abort rejects the run.
+			resolve('stopped-on-route');
+			abortController.abort();
+		};
+	});
 
 	try {
 		const services = await createStubServices({ nodesJsonPath });
@@ -141,6 +208,7 @@ export async function runDiscoveryScenario(
 
 		const eventBus = wrapEventBusWithObserver(createInMemoryEventBus(), (event) => {
 			events.push(toCapturedEvent(event));
+			instanceEvents.push(event);
 		});
 
 		// `OrchestrationContext` is required for the orchestrator to receive tools like
@@ -173,6 +241,22 @@ export async function runDiscoveryScenario(
 				providerOptions: {
 					anthropic: { cacheControl: { type: 'ephemeral' as const } },
 				},
+				...(stopBeforeTool
+					? {
+							guardrails: {
+								hooks: [
+									{
+										beforeTool: async ({ toolCallId, toolName, input }) => {
+											const call = { toolCallId, toolName, args: isRecord(input) ? input : {} };
+											if (!(await stopBeforeTool(call, instanceEvents))) return undefined;
+											stopRun();
+											return { action: 'stop' as const, code: 'route-picked' };
+										},
+									},
+								],
+							},
+						}
+					: {}),
 			}),
 		);
 
@@ -182,7 +266,7 @@ export async function runDiscoveryScenario(
 			context: {
 				threadId,
 				runId,
-				agentId: 'n8n-instance-agent',
+				agentId: ORCHESTRATOR_AGENT_ID,
 				eventBus,
 				signal: abortController.signal,
 				logger: silentLogger(),
@@ -197,9 +281,12 @@ export async function runDiscoveryScenario(
 			},
 		});
 		void run.catch(() => {});
-		const result = await Promise.race([run, budgetExpired]);
+		const result = await Promise.race([run, budgetExpired, routeStopped]);
 
-		streamStatus = resolveStreamStatus(result, abortController.signal.aborted);
+		streamStatus =
+			result === 'stopped-on-route'
+				? result
+				: resolveStreamStatus(result, abortController.signal.aborted);
 	} catch (error) {
 		runError = error instanceof Error ? error.message : String(error);
 		streamStatus = abortController.signal.aborted ? 'timed-out' : 'errored';
@@ -208,24 +295,15 @@ export async function runDiscoveryScenario(
 		await mcpManager?.disconnect();
 	}
 
-	const observedEvents = [...events];
-	const outcome = extractOutcomeFromEvents(observedEvents);
-	const check = evaluateDiscoveryTrial(options.scenario, outcome, {
-		streamStatus,
-		timeoutMs,
-		...(runError ? { runError } : {}),
-		unmatchedConfirmations: unmatchedConfirmations(confirmationPolicy, suspensions.values()),
-	});
-
+	// An abandoned stream can still publish, so copy what the run saw at its end.
 	return {
-		scenario: options.scenario,
-		check,
-		// An abandoned stream can still publish, so hand back what the verdict saw.
 		events: [...events],
-		outcome,
+		instanceEvents: [...instanceEvents],
 		durationMs: Date.now() - started,
 		streamStatus,
 		...(runError ? { runError } : {}),
+		timeoutMs,
+		unmatchedConfirmations: unmatchedConfirmations(confirmationPolicy, suspensions.values()),
 	};
 }
 
@@ -235,7 +313,7 @@ export async function runDiscoveryScenario(
 
 function applyInstanceState(
 	base: InstanceAiContext,
-	scenario: DiscoveryTestCase,
+	scenario: DiscoveryScenario,
 	mcpRegistry: StubMcpRegistry | undefined,
 ): InstanceAiContext {
 	const state = scenario.instanceState;
@@ -329,7 +407,7 @@ function createStubOrchestrationContext(
 		threadId: opts.threadId,
 		runId: opts.runId,
 		userId: opts.context.userId,
-		orchestratorAgentId: 'n8n-instance-agent',
+		orchestratorAgentId: ORCHESTRATOR_AGENT_ID,
 		modelId: opts.modelId,
 		eventBus: opts.eventBus,
 		logger: silentLogger(),

packages/@n8n/instance-ai/evaluations/discovery/types.ts

@@ -94,12 +94,30 @@ export interface DiscoveryTestCase {
 	timeoutMs?: number;
 }
 
+/** What the runner reads from a case. Routing cases carry no tool expectations. */
+export type DiscoveryScenario = Pick<
+	DiscoveryTestCase,
+	'userMessage' | 'instanceState' | 'confirmations' | 'maxSteps' | 'timeoutMs'
+>;
+
 export type DiscoveryStreamStatus =
 	| 'completed'
 	| 'errored'
 	| 'timed-out'
 	| 'suspended'
-	| 'step-exhausted';
+	| 'step-exhausted'
+	/** The runner ended the run before an orchestrator tool call, because the route was picked. */
+	| 'stopped-on-route';
+
+/** Only the orchestrator's calls pick a route; sub-agents act on its choice. */
+export const ORCHESTRATOR_AGENT_ID = 'n8n-instance-agent';
+
+/** An orchestrator tool call that is about to run. */
+export interface PendingToolCall {
+	toolCallId: string;
+	toolName: string;
+	args: Record<string, unknown>;
+}
 
 export interface DiscoveryTrialFacts {
 	streamStatus: DiscoveryStreamStatus;

packages/@n8n/instance-ai/evaluations/routing/__tests__/cases.test.ts

@@ -0,0 +1,144 @@
+import { mkdtempSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import { loadRoutingCases } from '../cases';
+
+// List files in reverse name order, so the tests fail if the loader stops sorting.
+vi.mock('fs', async (importOriginal) => {
+	const fs = await importOriginal<typeof import('fs')>();
+	return { ...fs, readdirSync: (dir: string) => fs.readdirSync(dir).sort().reverse() };
+});
+
+/** A case as LangTracer's `export_suite` writes it. */
+function exportedCase(tags: string[] = [], overrides: Record<string, unknown> = {}) {
+	return {
+		complexity: 'simple',
+		tags: ['routing', 'bucket:debug', 'accepts:debug', 'accepts:clarify', 'lang:eng', ...tags],
+		executionScenarios: [],
+		conversation: [{ role: 'user', text: 'It failed again.' }],
+		description: 'A failed run with no other context.',
+		processExpectations: ['Routes to one of: debug, clarify'],
+		...overrides,
+	};
+}
+
+function caseDir(files: Record<string, unknown>): string {
+	const dir = mkdtempSync(join(tmpdir(), 'routing-cases-'));
+	for (const [name, body] of Object.entries(files)) {
+		writeFileSync(join(dir, name), typeof body === 'string' ? body : JSON.stringify(body));
+	}
+	return dir;
+}
+
+describe('loadRoutingCases', () => {
+	it('loads the route-*.json files in name order, drops export-only keys, and reads the tags with the bucket accepted', () => {
+		const dir = caseDir({
+			'route-debug-two.json': exportedCase([], { id: 42, name: 'Two', suiteId: 64 }),
+			'route-clarify-open.json': exportedCase([], {
+				tags: ['routing', 'bucket:clarify', 'accepts:clarify:open'],
+			}),
+			'route-prod-agent-one.json': exportedCase([], {
+				tags: ['routing', 'bucket:agent', 'accepts:clarify:agent'],
+				conversation: [{ role: 'user', text: ['Answer our support inbox.', 'Use our FAQ.'] }],
+			}),
+			'results.json': '{}',
+			'route-notes.md': 'not a case',
+		});
+
+		const { cases, needsSetup } = loadRoutingCases(dir);
+
+		expect(cases).toEqual([
+			{
+				id: 'route-clarify-open',
+				bucket: 'clarify',
+				accepts: ['clarify:open'],
+				userMessage: 'It failed again.',
+			},
+			{
+				id: 'route-debug-two',
+				bucket: 'debug',
+				accepts: ['debug', 'clarify'],
+				userMessage: 'It failed again.',
+			},
+			{
+				id: 'route-prod-agent-one',
+				bucket: 'agent',
+				accepts: ['agent', 'clarify:agent'],
+				userMessage: 'Answer our support inbox.\nUse our FAQ.',
+			},
+		]);
+		expect(needsSetup).toEqual([]);
+	});
+
+	it('keeps the files whose name contains one of the filter tokens', () => {
+		const dir = caseDir({
+			'route-agent-one.json': exportedCase(),
+			'route-debug-one.json': exportedCase(),
+			'route-answer-one.json': exportedCase(),
+		});
+
+		expect(loadRoutingCases(dir, 'AGENT, debug').cases.map((c) => c.id)).toEqual([
+			'route-agent-one',
+			'route-debug-one',
+		]);
+	});
+
+	it('returns the cases that need setup without running them', () => {
+		const dir = caseDir({
+			'route-debug-ok.json': exportedCase(),
+			'route-debug-no-accounts.json': exportedCase([], { credentials: [] }),
+			'route-debug-seeded.json': exportedCase([], {
+				seed: { mode: 'inline', messages: [{ role: 'user', text: 'Build me a daily report.' }] },
+			}),
+			'route-debug-with-account.json': exportedCase([], { credentials: [{ type: 'slackApi' }] }),
+			'route-debug-in-browser.json': exportedCase([], { credentialFixture: 'local' }),
+		});
+
+		const { cases, needsSetup } = loadRoutingCases(dir);
+
+		expect(cases.map((c) => c.id)).toEqual(['route-debug-no-accounts', 'route-debug-ok']);
+		expect(needsSetup).toEqual([
+			'route-debug-in-browser',
+			'route-debug-seeded',
+			'route-debug-with-account',
+		]);
+	});
+
+	it('reports every invalid file in one error', () => {
+		const tags = (...extra: string[]) => ({ tags: ['routing', ...extra] });
+		const dir = caseDir({
+			'route-debug-ok.json': exportedCase(),
+			'route-debug-broken.json': '{',
+			'route-debug-assistant-turn.json': exportedCase([], {
+				conversation: [{ role: 'assistant', text: 'It failed again.' }],
+			}),
+			'route-debug-no-routing-tag.json': exportedCase([], {
+				tags: ['bucket:debug', 'accepts:debug'],
+			}),
+			'route-debug-two-buckets.json': exportedCase(
+				[],
+				tags('bucket:debug', 'bucket:agent', 'accepts:debug'),
+			),
+			'route-clarify-no-accepts.json': exportedCase([], tags('bucket:clarify')),
+			'route-debug-bad-token.json': exportedCase(
+				[],
+				tags('bucket:debug', 'accepts:clarify:workflow'),
+			),
+			'route-debug-two-turns.json': exportedCase([], {
+				conversation: [
+					{ role: 'user', text: 'It failed.' },
+					{ role: 'user', text: 'Again.' },
+				],
+			}),
+		});
+
+		expect(() => loadRoutingCases(dir)).toThrow(
+			expect.objectContaining({
+				message: expect.stringMatching(
+					/route-clarify-no-accepts\.json: accepts: a bucket:clarify case needs an accepts:<token> tag[\s\S]*route-debug-assistant-turn\.json: needs exactly one user message[\s\S]*route-debug-bad-token\.json: accepts\.0[\s\S]*route-debug-broken\.json[\s\S]*route-debug-no-routing-tag\.json: has no "routing" tag[\s\S]*route-debug-two-buckets\.json: bucket: needs exactly one bucket:<route> tag[\s\S]*route-debug-two-turns\.json: needs exactly one user message/,
+				),
+			}),
+		);
+	});
+});

packages/@n8n/instance-ai/evaluations/routing/__tests__/grade.test.ts

@@ -0,0 +1,238 @@
+import type { InstanceAiEvent } from '@n8n/api-types';
+
+import { ORCHESTRATOR_AGENT_ID } from '../../discovery/types';
+import type { RoutingCase } from '../cases';
+import {
+	casePasses,
+	createRouteWatcher,
+	type RouteResolution,
+	traceSteps,
+	trialPasses,
+} from '../grade';
+import type { JudgeInput, JudgeVerdict } from '../judge';
+
+const base = { runId: 'run-1', agentId: ORCHESTRATOR_AGENT_ID };
+
+function textEvent(value: string, agentId = ORCHESTRATOR_AGENT_ID): InstanceAiEvent {
+	return { type: 'text-delta', ...base, agentId, payload: { text: value } };
+}
+
+function callEvent(
+	toolName: string,
+	args: Record<string, unknown> = {},
+	agentId = ORCHESTRATOR_AGENT_ID,
+): InstanceAiEvent {
+	return { type: 'tool-call', ...base, agentId, payload: { toolCallId: toolName, toolName, args } };
+}
+
+function pending(toolName: string, args: Record<string, unknown> = {}) {
+	return { toolCallId: toolName, toolName, args };
+}
+
+function routingCase(
+	bucket: RoutingCase['bucket'],
+	accepts: RoutingCase['accepts'] = [],
+): RoutingCase {
+	return { id: `route-${bucket}-x`, bucket, userMessage: 'Do the thing.', accepts };
+}
+
+const go: JudgeVerdict = {
+	decision: 'continue',
+	route: 'none',
+	steer: 'none',
+	reason: 'Explores.',
+};
+
+function stop(route: JudgeVerdict['route'], steer: JudgeVerdict['steer'] = 'none'): JudgeVerdict {
+	return { decision: 'stop', route, steer, reason: `Picks ${route}.` };
+}
+
+/** A judge that returns the verdicts in order, then the last one again. */
+function judgeReturning(...verdicts: Array<JudgeVerdict | Error>) {
+	let index = 0;
+	return vi.fn(async (_input: JudgeInput) => {
+		const verdict = verdicts[Math.min(index++, verdicts.length - 1)];
+		if (verdict instanceof Error) throw verdict;
+		return await Promise.resolve(verdict);
+	});
+}
+
+const clarify = (steer: RouteResolution['steer']): RouteResolution => ({
+	route: 'clarify',
+	steer,
+	evidence: 'ask-user call',
+});
+
+describe('traceSteps', () => {
+	it('keeps the orchestrator text and calls in order and drops sub-agent events', () => {
+		const steps = traceSteps([
+			textEvent('Let me '),
+			textEvent('check. '),
+			callEvent('search-nodes', { query: 'slack' }),
+			textEvent('sub-agent text', 'builder-1'),
+			callEvent('write_config', {}, 'builder-1'),
+			textEvent('Found it.'),
+		]);
+
+		expect(steps).toEqual([
+			{ kind: 'text', text: 'Let me check.' },
+			{ kind: 'call', toolName: 'search-nodes', args: { query: 'slack' } },
+			{ kind: 'text', text: 'Found it.' },
+		]);
+	});
+
+	it('adds the pending call last when it has no event yet', () => {
+		const steps = traceSteps([textEvent('A question first.')], pending('ask-user'));
+
+		expect(steps).toEqual([
+			{ kind: 'text', text: 'A question first.' },
+			{ kind: 'call', toolName: 'ask-user', args: {} },
+		]);
+	});
+
+	it('does not add the pending call twice when its event is already in', () => {
+		const steps = traceSteps([callEvent('ask-user')], pending('ask-user'));
+
+		expect(steps).toEqual([{ kind: 'call', toolName: 'ask-user', args: {} }]);
+	});
+});
+
+describe('createRouteWatcher', () => {
+	it('stops before the call that the judge marks as the route', async () => {
+		const judge = judgeReturning(go, stop('workflow', 'workflow'));
+		const watcher = createRouteWatcher(judge);
+
+		expect(await watcher.beforeToolCall(pending('search-nodes'), [])).toBe(false);
+		expect(
+			await watcher.beforeToolCall(pending('build-workflow'), [callEvent('search-nodes')]),
+		).toBe(true);
+		const resolution = await watcher.resolve({
+			instanceEvents: [],
+			streamStatus: 'stopped-on-route',
+		});
+
+		expect(resolution).toEqual({
+			route: 'workflow',
+			steer: 'workflow',
+			evidence: 'build-workflow call',
+			judgeReason: 'Picks workflow.',
+		});
+		expect(judge).toHaveBeenCalledTimes(2);
+		expect(judge).toHaveBeenLastCalledWith({
+			steps: [
+				{ kind: 'call', toolName: 'search-nodes', args: {} },
+				{ kind: 'call', toolName: 'build-workflow', args: {} },
+			],
+		});
+	});
+
+	it('lets the run go on when the judge stops without a route', async () => {
+		const watcher = createRouteWatcher(judgeReturning(stop('none')));
+
+		expect(await watcher.beforeToolCall(pending('search-nodes'), [])).toBe(false);
+	});
+
+	it('skips the check before a read-only action, and stops it once the route is picked', async () => {
+		const judge = judgeReturning(stop('one-off'));
+		const watcher = createRouteWatcher(judge);
+
+		expect(await watcher.beforeToolCall(pending('load_skill'), [])).toBe(false);
+		expect(await watcher.beforeToolCall(pending('nodes', { action: 'search' }), [])).toBe(false);
+		expect(judge).not.toHaveBeenCalled();
+		expect(await watcher.beforeToolCall(pending('nodes', { action: 'execute' }), [])).toBe(true);
+		expect(await watcher.beforeToolCall(pending('load_skill'), [])).toBe(true);
+		expect(judge).toHaveBeenCalledTimes(1);
+	});
+
+	it('stops a parallel call without a second judge call once the route is picked', async () => {
+		const judge = judgeReturning(stop('multi'));
+		const watcher = createRouteWatcher(judge);
+
+		const results = await Promise.all([
+			watcher.beforeToolCall(pending('create-tasks'), []),
+			watcher.beforeToolCall(pending('search-nodes'), []),
+		]);
+
+		expect(results).toEqual([true, true]);
+		expect(judge).toHaveBeenCalledTimes(1);
+	});
+
+	it('asks the judge for the route of the full turn when no call stopped the run', async () => {
+		const judge = judgeReturning(go, stop('answer'));
+		const watcher = createRouteWatcher(judge);
+		await watcher.beforeToolCall(pending('n8n-docs'), []);
+
+		const resolution = await watcher.resolve({
+			instanceEvents: [callEvent('n8n-docs'), textEvent('Webhooks start workflows.')],
+			streamStatus: 'completed',
+		});
+
+		expect(resolution).toMatchObject({ route: 'answer', evidence: 'end of turn (completed)' });
+		expect(judge).toHaveBeenLastCalledWith({
+			steps: [
+				{ kind: 'call', toolName: 'n8n-docs', args: {} },
+				{ kind: 'text', text: 'Webhooks start workflows.' },
+			],
+			endStatus: 'completed',
+		});
+	});
+
+	it('lets the run go on when a check fails and reports the error', async () => {
+		const judge = judgeReturning(new Error('Routing judge failed: 529'), stop('debug'));
+		const watcher = createRouteWatcher(judge);
+
+		expect(await watcher.beforeToolCall(pending('executions'), [])).toBe(false);
+		expect(await watcher.beforeToolCall(pending('executions'), [])).toBe(true);
+		const resolution = await watcher.resolve({
+			instanceEvents: [],
+			streamStatus: 'stopped-on-route',
+		});
+
+		expect(resolution).toMatchObject({ route: 'debug', judgeError: 'Routing judge failed: 529' });
+	});
+
+	it('returns none with the error when the end-of-turn judge fails', async () => {
+		const watcher = createRouteWatcher(judgeReturning(new Error('Routing judge failed: 529')));
+
+		const resolution = await watcher.resolve({ instanceEvents: [], streamStatus: 'timed-out' });
+
+		expect(resolution).toEqual({
+			route: 'none',
+			evidence: 'end of turn (timed-out), judge failed',
+			judgeError: 'Routing judge failed: 529',
+		});
+	});
+});
+
+describe('trialPasses', () => {
+	it.each<[RoutingCase['accepts'], RouteResolution, boolean]>([
+		[['workflow'], { route: 'workflow', evidence: 'build-workflow' }, true],
+		[['workflow'], { route: 'one-off', evidence: 'build-workflow' }, false],
+		[['clarify'], clarify('workflow'), true],
+		[['clarify:agent'], clarify('both'), true],
+		[['clarify:agent'], clarify('none'), false],
+		[['clarify:open'], clarify('none'), true],
+		[['clarify:open'], clarify('workflow'), false],
+		[['answer:agent'], { route: 'answer', steer: 'both', evidence: 'end of turn' }, true],
+		[['answer:open'], { route: 'answer', steer: 'workflow', evidence: 'end of turn' }, false],
+		[['answer:open'], clarify('none'), false],
+	])('accepts %j with %j: %s', (accepts, resolution, expected) => {
+		expect(trialPasses(routingCase('clarify', accepts), resolution)).toBe(expected);
+	});
+
+	it('fails a question toward the bucket artifact when the case accepts no question', () => {
+		expect(trialPasses(routingCase('workflow', ['workflow']), clarify('workflow'))).toBe(false);
+	});
+});
+
+describe('casePasses', () => {
+	it.each([
+		[2, 3, true],
+		[1, 3, false],
+		[4, 6, true],
+		[3, 6, false],
+		[0, 0, false],
+	])('%i of %i trials: %s', (passed, total, expected) => {
+		expect(casePasses(passed, total)).toBe(expected);
+	});
+});

packages/@n8n/instance-ai/evaluations/routing/cases.ts

@@ -0,0 +1,139 @@
+// ---------------------------------------------------------------------------
+// Loader for routing eval cases: one `route-<slug>.json` file per case, in the
+// main eval case format that LangTracer exports. The slug is the case id.
+//
+// The routing labels are tags: `routing`, one `bucket:<route>`, and one
+// `accepts:<token>` for each other accepted route. A judge picks the route of
+// each trial (see grade.ts).
+// ---------------------------------------------------------------------------
+
+import { readFileSync } from 'fs';
+import { basename, resolve } from 'path';
+import { z } from 'zod';
+
+import { EvalTestCaseSchema } from '../harness/schema';
+import { normalizeExportedCase } from '../langtracer/normalize';
+import { getJsonFiles } from '../utils/get-json-files';
+
+export const ROUTING_BUCKETS = [
+	'agent',
+	'workflow',
+	'one-off',
+	'debug',
+	'answer',
+	'clarify',
+	'multi',
+	'decline',
+] as const;
+export type RoutingBucket = (typeof ROUTING_BUCKETS)[number];
+
+export const ROUTING_ACCEPT_TOKENS = [
+	...ROUTING_BUCKETS,
+	'clarify:agent',
+	'clarify:open',
+	'answer:agent',
+	'answer:open',
+] as const;
+export type AcceptToken = (typeof ROUTING_ACCEPT_TOKENS)[number];
+
+const routingTagsSchema = z
+	.object({
+		/** The main expected route. Reports group cases by it. */
+		bucket: z
+			.array(z.enum(ROUTING_BUCKETS))
+			.length(1, 'needs exactly one bucket:<route> tag')
+			.transform((buckets) => buckets[0]),
+		/** A trial passes when its route matches one of these tokens. */
+		accepts: z.array(z.enum(ROUTING_ACCEPT_TOKENS)),
+	})
+	// The bucket always passes, so case files need not repeat it. A clarify case
+	// keeps only its own tokens: most accept only `clarify:open`, and plain
+	// `clarify` would also pass a question that only asks about a workflow.
+	.transform(({ bucket, accepts }) => ({
+		bucket,
+		accepts: bucket === 'clarify' || accepts.includes(bucket) ? accepts : [bucket, ...accepts],
+	}))
+	.refine(({ accepts }) => accepts.length > 0, {
+		message: 'a bucket:clarify case needs an accepts:<token> tag',
+		path: ['accepts'],
+	});
+
+export type RoutingCase = z.infer<typeof routingTagsSchema> & { id: string; userMessage: string };
+
+type ParsedFile = { kind: 'case'; routingCase: RoutingCase } | { kind: 'needs-setup'; id: string };
+
+function tagValues(tags: string[], prefix: string): string[] {
+	return tags.filter((tag) => tag.startsWith(prefix)).map((tag) => tag.slice(prefix.length));
+}
+
+function formatIssues(error: z.ZodError): string {
+	return error.issues
+		.map((issue) => `${issue.path.join('.') || '(root)'}: ${issue.message}`)
+		.join('; ');
+}
+
+function parseRoutingCaseFile(filePath: string): ParsedFile {
+	const id = basename(filePath, '.json');
+	let raw: unknown;
+	try {
+		raw = JSON.parse(readFileSync(filePath, 'utf-8'));
+	} catch (error) {
+		throw new Error(`${filePath}: ${error instanceof Error ? error.message : String(error)}`);
+	}
+	// Strips export-only keys (id, name, suiteId, ...), as the LangTracer pull does.
+	const parsed = EvalTestCaseSchema.safeParse(normalizeExportedCase(raw));
+	if (!parsed.success) throw new Error(`${filePath}: ${formatIssues(parsed.error)}`);
+
+	const { tags, conversation = [], seed, credentials, credentialFixture } = parsed.data;
+	if (!tags.includes('routing')) throw new Error(`${filePath}: has no "routing" tag`);
+	// ponytail: the runner cannot create earlier messages, an open workflow or Agent, or accounts yet.
+	if (seed || credentials?.length || credentialFixture) {
+		return { kind: 'needs-setup', id };
+	}
+	if (conversation.length !== 1 || conversation[0].role !== 'user') {
+		throw new Error(`${filePath}: needs exactly one user message`);
+	}
+
+	const labels = routingTagsSchema.safeParse({
+		bucket: tagValues(tags, 'bucket:'),
+		accepts: tagValues(tags, 'accepts:'),
+	});
+	if (!labels.success) throw new Error(`${filePath}: ${formatIssues(labels.error)}`);
+	return { kind: 'case', routingCase: { id, userMessage: conversation[0].text, ...labels.data } };
+}
+
+/**
+ * Loads every `route-*.json` file in `dir`, sorted by name, so notes or
+ * results kept next to the cases are not read as cases. `filter` keeps the
+ * files whose name contains one of its comma-separated tokens. All invalid
+ * files are reported in one error, so an author can fix them in one pass.
+ * Cases that need setup are returned by id in `needsSetup` and do not run.
+ */
+export function loadRoutingCases(
+	dir: string,
+	filter?: string,
+): { cases: RoutingCase[]; needsSetup: string[] } {
+	const root = resolve(dir);
+	const files = getJsonFiles(root, filter)
+		.filter((file) => basename(file).startsWith('route-'))
+		.sort();
+
+	const cases: RoutingCase[] = [];
+	const needsSetup: string[] = [];
+	const errors: string[] = [];
+	for (const file of files) {
+		try {
+			const parsed = parseRoutingCaseFile(file);
+			if (parsed.kind === 'case') cases.push(parsed.routingCase);
+			else needsSetup.push(parsed.id);
+		} catch (error) {
+			errors.push(error instanceof Error ? error.message : String(error));
+		}
+	}
+	if (errors.length > 0) {
+		throw new Error(
+			`Invalid routing case(s) in ${root}:\n${errors.map((e) => `  - ${e}`).join('\n')}`,
+		);
+	}
+	return { cases, needsSetup };
+}

packages/@n8n/instance-ai/evaluations/routing/grade.ts

@@ -0,0 +1,178 @@
+// ---------------------------------------------------------------------------
+// Route resolution and scoring for the routing eval.
+//
+// A route watcher asks the judge before each orchestrator tool call runs (see
+// judge.ts). The first `stop` verdict ends the run and decides the route. When
+// the turn ends first, the judge picks the route from the full trace. Only
+// the read-only actions below skip the check, so a new or renamed tool still
+// gets a check and needs no grader change.
+// ---------------------------------------------------------------------------
+
+import type { InstanceAiEvent } from '@n8n/api-types';
+
+import type { AcceptToken, RoutingCase } from './cases';
+import type { JudgeInput, JudgeVerdict, Route, Steer, TraceStep } from './judge';
+import {
+	ORCHESTRATOR_AGENT_ID,
+	type DiscoveryStreamStatus,
+	type PendingToolCall,
+} from '../discovery/types';
+import { DOMAIN_TOOL_IDS } from '../../src/tools/tool-ids';
+
+/**
+ * A check before a read almost never stops the run, and the next check sees
+ * the read, so the run skips it. A tool or action not listed here gets a check.
+ */
+const READ_ONLY_ACTIONS: Readonly<Record<string, ReadonlySet<unknown>>> = {
+	// `load_skill` has no action.
+	load_skill: new Set([undefined]),
+	[DOMAIN_TOOL_IDS.NODES]: new Set([
+		'list',
+		'search',
+		'describe',
+		'type-definition',
+		'suggested',
+		'explore-resources',
+	]),
+	[DOMAIN_TOOL_IDS.CREDENTIALS]: new Set(['list', 'get', 'search-types']),
+	[DOMAIN_TOOL_IDS.N8N_DOCS]: new Set(['lookup', 'search', 'read']),
+	[DOMAIN_TOOL_IDS.RESEARCH]: new Set(['web-search', 'fetch-url']),
+};
+
+const isReadOnly = ({ toolName, args }: PendingToolCall): boolean =>
+	READ_ONLY_ACTIONS[toolName]?.has(args.action) ?? false;
+
+export interface RouteResolution {
+	route: Route;
+	/** Unset only when the judge failed. */
+	steer?: Steer;
+	/** Where the route was decided: the tool call that the run stopped before, or the end of the turn. */
+	evidence: string;
+	judgeReason?: string;
+	/** Set when a judge call in the trial failed. */
+	judgeError?: string;
+}
+
+/**
+ * The orchestrator's text and calls in order. A pending call that has no
+ * event yet goes last, after the text that the run published before it.
+ */
+export function traceSteps(
+	events: readonly InstanceAiEvent[],
+	pending?: PendingToolCall,
+): TraceStep[] {
+	const steps: TraceStep[] = [];
+	let text = '';
+	let pendingSeen = false;
+	for (const event of events) {
+		if (event.agentId !== ORCHESTRATOR_AGENT_ID) continue;
+		if (event.type === 'text-delta') {
+			text += event.payload.text;
+			continue;
+		}
+		if (event.type !== 'tool-call') continue;
+		if (text.trim()) steps.push({ kind: 'text', text: text.trim() });
+		text = '';
+		const { toolCallId, toolName, args } = event.payload;
+		steps.push({ kind: 'call', toolName, args });
+		if (toolCallId === pending?.toolCallId) pendingSeen = true;
+	}
+	if (text.trim()) steps.push({ kind: 'text', text: text.trim() });
+	if (pending && !pendingSeen) {
+		steps.push({ kind: 'call', toolName: pending.toolName, args: pending.args });
+	}
+	return steps;
+}
+
+export interface RouteWatcher {
+	/** The runner's `stopBeforeTool` hook: `true` ends the run before the call runs. */
+	beforeToolCall: (call: PendingToolCall, events: readonly InstanceAiEvent[]) => Promise<boolean>;
+	/** The route of the finished turn. */
+	resolve: (turn: {
+		instanceEvents: readonly InstanceAiEvent[];
+		streamStatus: DiscoveryStreamStatus;
+	}) => Promise<RouteResolution>;
+}
+
+/** One watcher for each trial. */
+export function createRouteWatcher(
+	judge: (input: JudgeInput) => Promise<JudgeVerdict>,
+): RouteWatcher {
+	let stopped: RouteResolution | undefined;
+	let judgeError: string | undefined;
+	// Calls of one step can run in parallel, so the judge takes them one at a time.
+	let queue: Promise<unknown> = Promise.resolve();
+
+	const ask = async (input: JudgeInput): Promise<JudgeVerdict | undefined> => {
+		try {
+			return await judge(input);
+		} catch (error) {
+			// A failed check before a call lets the run go on; the next check decides.
+			judgeError ??= error instanceof Error ? error.message : String(error);
+			return undefined;
+		}
+	};
+	const withError = (resolution: RouteResolution): RouteResolution =>
+		judgeError ? { ...resolution, judgeError } : resolution;
+
+	return {
+		beforeToolCall: async (call, events) => {
+			const check = queue.then(async () => {
+				if (stopped) return true;
+				if (isReadOnly(call)) return false;
+				const verdict = await ask({ steps: traceSteps(events, call) });
+				// A stop without a route lets the run go on.
+				if (verdict?.decision !== 'stop' || verdict.route === 'none') return false;
+				stopped = {
+					route: verdict.route,
+					steer: verdict.steer,
+					evidence: `${call.toolName} call`,
+					judgeReason: verdict.reason,
+				};
+				return true;
+			});
+			queue = check;
+			return await check;
+		},
+
+		resolve: async ({ instanceEvents, streamStatus }) => {
+			// A check can still run after a timeout, and its verdict counts.
+			await queue;
+			if (stopped) return withError(stopped);
+			const evidence = `end of turn (${streamStatus})`;
+			const verdict = await ask({ steps: traceSteps(instanceEvents), endStatus: streamStatus });
+			if (!verdict) return withError({ route: 'none', evidence: `${evidence}, judge failed` });
+			return withError({
+				route: verdict.route,
+				steer: verdict.steer,
+				evidence,
+				judgeReason: verdict.reason,
+			});
+		},
+	};
+}
+
+/** Short label for output: the route, with the steer for `clarify` and `answer`. */
+export function routeLabel(resolution: RouteResolution): string {
+	return resolution.route === 'clarify' || resolution.route === 'answer'
+		? `${resolution.route}:${resolution.steer ?? 'none'}`
+		: resolution.route;
+}
+
+// `<route>:agent` passes a steer toward an Agent; `<route>:open` passes any steer except workflow only.
+function acceptTokenMatches(token: AcceptToken, { route, steer }: RouteResolution): boolean {
+	const [tokenRoute, tokenSteer] = token.split(':');
+	if (route !== tokenRoute) return false;
+	if (tokenSteer === 'agent') return steer === 'agent' || steer === 'both';
+	if (tokenSteer === 'open') return steer !== undefined && steer !== 'workflow';
+	return true;
+}
+
+export function trialPasses(routingCase: RoutingCase, resolution: RouteResolution): boolean {
+	return routingCase.accepts.some((token) => acceptTokenMatches(token, resolution));
+}
+
+/** A case passes when at least 2 of 3 trials pass; other trial counts keep the ratio. */
+export function casePasses(passedTrials: number, totalTrials: number): boolean {
+	return totalTrials > 0 && passedTrials * 3 >= totalTrials * 2;
+}

packages/@n8n/instance-ai/evaluations/routing/judge.ts

@@ -0,0 +1,142 @@
+// ---------------------------------------------------------------------------
+// LLM judge for the routing eval.
+//
+// The judge watches one orchestrator turn. Before each orchestrator tool call
+// runs, it reads the trace so far and decides to stop with a route or to let
+// the run continue. When the turn ends first, it picks the route from the full
+// trace. It never sees the user's message, so it grades what the Assistant did.
+// ---------------------------------------------------------------------------
+
+import { z } from 'zod';
+
+import { createEvalAgent } from '../../src/utils/eval-agents';
+import type { DiscoveryStreamStatus } from '../discovery/types';
+
+// `reason` comes first, so the judge reasons before it picks the labels.
+export const judgeVerdictSchema = z.object({
+	reason: z
+		.string()
+		.describe('What the Assistant did, then the rules that decide the route and the steer.'),
+	decision: z
+		.enum(['continue', 'stop'])
+		.describe('stop when the Assistant has picked a route, else continue.'),
+	route: z
+		.enum([
+			'agent',
+			'workflow',
+			'one-off',
+			'debug',
+			'multi',
+			'clarify',
+			'answer',
+			'decline',
+			'none',
+		])
+		.describe('The route that the Assistant picked. Use none on continue.'),
+	steer: z
+		.enum(['agent', 'workflow', 'both', 'none'])
+		.describe('The artifact that the Assistant pushes the user toward.'),
+});
+
+export type JudgeVerdict = z.infer<typeof judgeVerdictSchema>;
+export type Route = JudgeVerdict['route'];
+export type Steer = JudgeVerdict['steer'];
+
+export type TraceStep =
+	| { kind: 'text'; text: string }
+	| { kind: 'call'; toolName: string; args: Record<string, unknown> };
+
+export interface JudgeInput {
+	/** The orchestrator's text and tool calls, in order. */
+	steps: TraceStep[];
+	/** Set when the turn has ended. The judge must then pick a route. */
+	endStatus?: DiscoveryStreamStatus;
+}
+
+// Labels steer more consistently than the shared eval model (Sonnet 4.6).
+const JUDGE_MODEL = 'anthropic/claude-sonnet-5-5';
+const MAX_ATTEMPTS = 2;
+const ATTEMPT_TIMEOUT_MS = 60_000;
+const MAX_STEP_CHARS = 3_000;
+
+const SYSTEM_PROMPT = `You watch the n8n Assistant work on one user request. You do not see the request. You see what the Assistant did so far: its text and its tool calls, in order. Decide if the Assistant has picked a route.
+
+n8n is a workflow automation product. The Assistant can build two different artifacts:
+
+- A workflow: a trigger and a fixed graph of steps (nodes) that runs the same way each time. A workflow can contain AI steps, including an "AI Agent" node. It is still a workflow.
+- An Agent: a standalone n8n Agent made with the Agent Builder. It has instructions, tools, and memory. It holds conversations or owns an ongoing role that needs judgment. It can also run scheduled tasks.
+
+route:
+- agent: The Assistant starts to build or change an Agent.
+- workflow: The Assistant starts to build or change a workflow that the user keeps and that runs again later.
+- one-off: The Assistant does a task once, now, and the task has an effect. For example, it runs a node or a workflow, writes or deletes data, publishes, unpublishes, or deletes a workflow, or tests a credential. A workflow that it builds only to run once is one-off. Opening a credential setup for the user is not one-off, because the user must still finish the setup.
+- debug: The Assistant looks for why a run failed or gave a wrong result. It reads the executions of the run, or it reads the workflow or the Agent to find the cause. When it then explains or fixes the cause that it found, the route is still debug.
+- multi: The Assistant plans two or more separate pieces of work, for example two workflows, or an Agent and a workflow. A plan for one artifact has the route of that artifact. An Agent with a workflow that only connects it to a channel (for example, a workflow that passes WhatsApp messages to the Agent) is one artifact: agent.
+- clarify: The Assistant asks the user for a decision or a missing detail and waits. It shows a question card, or its reply ends with questions that it must have answered before it continues.
+- answer: The Assistant gives information, instructions, an explanation, or a result, and it needs nothing from the user to continue. An answer that ends with an optional offer ("Do you want me to build this?") is still an answer. But when the reply asks the user to pick one of the options that it lists, asks what the user wants to build or do ("What do you want to build?"), or asks for details that it needs before it builds, it is clarify.
+- decline: The Assistant refuses the request, or it says that it cannot help with it. It can suggest an alternative.
+- none: None of the routes above.
+
+decision:
+- continue: The Assistant only explored. Exploration is loading skills or tools, searching docs, nodes, templates, or the web, and reading workflows, tables, credentials, or settings. Creating an empty table or adding columns is also exploration. Exploration never picks a route, even when the name of the skill, the tool, or the item names a route, or when the Assistant says what it will build or do. Reading a workflow is not debug, unless the Assistant reads it to find why a run failed or gave a wrong result. Listing or reading credentials is not one-off.
+- continue: The latest call starts work that can still end as more than one route. For example, it writes workflow code, but no call shows yet if the workflow runs once or stays.
+- stop: The latest call picks a route. It starts a build, does a task with an effect, plans tasks, looks for why a run failed or gave a wrong result, or asks the user. Stop as soon as the route is clear. Do not wait for the work to finish.
+- continue: The latest call only asks the user to confirm a build that the Assistant already proposed (for example, "Should I build this now?" with yes and no options). The build that follows decides the route. When the turn ends on this confirmation, pick the route of the proposed build.
+- When the turn has ended, stop and pick the route from all that the Assistant did and wrote. If the turn did not complete, its last text is not a reply to the user. Then pick the route only from the calls. What the Assistant said that it will build or do does not pick a route. When no call picked a route, pick none.
+
+steer is the artifact that the Assistant pushes the user toward. An answer has the steer of what it explains or recommends: an answer that explains only how to build a workflow, also one with an AI Agent node, is workflow.
+- agent: It recommends or assumes an Agent, or its questions only make sense for an Agent (persona, tone, what the Agent remembers, where it talks to people). It does not offer a workflow as a real option.
+- workflow: It recommends or assumes a workflow, or its questions are about workflow details (nodes, steps, field mappings, or which app event starts the workflow). It does not offer an Agent as a real option. A workflow with an AI Agent node is a workflow. It is also workflow when the Assistant says that it will build a workflow or lists the steps that it will build. A question about when or how often the work runs is not a workflow detail, also when one option is an app event, because an Agent can also run on a schedule.
+- both: It offers an Agent and a workflow as real options, and the user can pick either one. This applies also when it marks one of them as recommended. An option where an AI keeps a conversation going with people (for example, it keeps chatting with leads or answers customers) is an Agent option.
+- none: It pushes toward neither artifact. It asks about the goal or the use case without favouring one, it asks only about details that fit both (which app holds the data, which channel, when or how often the work runs, or whether to do the task once now or on a repeat), or it declines. A question whether to do the task once now or keep doing it is none, also when the repeat option is a workflow.
+
+Rules:
+- Judge what the Assistant does, not what it should do.
+- The words "agent", "assistant", or "bot" alone do not decide the route or the steer. Decide from what the Assistant proposes to build.
+- reason: two or three sentences. Say what the Assistant did, then name the rules that decide the route and the steer.`;
+
+function clip(text: string): string {
+	return text.length > MAX_STEP_CHARS
+		? `${text.slice(0, MAX_STEP_CHARS)}… [${String(text.length - MAX_STEP_CHARS)} more characters]`
+		: text;
+}
+
+export function renderJudgePrompt({ steps, endStatus }: JudgeInput): string {
+	let callNumber = 0;
+	const lines = steps.map((step) =>
+		step.kind === 'text'
+			? `Assistant text: ${clip(step.text)}`
+			: `Tool call ${String(++callNumber)}: ${step.toolName} ${clip(JSON.stringify(step.args))}`,
+	);
+	const footer = endStatus
+		? `The turn has ended (status: ${endStatus}). Stop and pick the route.`
+		: 'The turn is still running. The last tool call has not run yet. Decide: stop or continue.';
+	return `<trace>\n${lines.join('\n') || '(nothing)'}\n</trace>\n\n${footer}`;
+}
+
+/** Throws when no attempt returns a valid verdict. */
+export async function judgeRoute(input: JudgeInput): Promise<JudgeVerdict> {
+	let lastError = 'no attempt made';
+	for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+		const agent = createEvalAgent('eval-routing-judge', {
+			instructions: SYSTEM_PROMPT,
+			cache: true,
+			model: JUDGE_MODEL,
+		}).structuredOutput(judgeVerdictSchema);
+		try {
+			const result = await agent.generate(renderJudgePrompt(input), {
+				abortSignal: AbortSignal.timeout(ATTEMPT_TIMEOUT_MS),
+			});
+			const parsed = judgeVerdictSchema.safeParse(result.structuredOutput);
+			if (parsed.success) return parsed.data;
+			// generate() reports API failures in `result.error` instead of throwing.
+			lastError =
+				result.error instanceof Error
+					? result.error.message
+					: `no valid verdict (${JSON.stringify(result.error ?? null)})`;
+		} catch (error) {
+			lastError = error instanceof Error ? error.message : String(error);
+		}
+	}
+	throw new Error(`Routing judge failed: ${lastError}`);
+}