fix(ai-builder): Make eval file fixtures match the scenario (no-changelog) (#40725)

Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>

packages/cli/src/modules/instance-ai/eval/__tests__/execution.service.test.ts

@@ -1883,6 +1883,48 @@ describe('EvalExecutionService', () => {
 			});
 		});
 
+		it('writes the embedded text into a synthesized PDF', async () => {
+			const hints = makeEmptyHints();
+			hints.triggerContent = {
+				body: {},
+				binary: {
+					attachment_0: {
+						mimeType: 'application/pdf',
+						fileName: 'invoice.pdf',
+						text: 'Invoice Number: INV-2026-0042',
+					},
+				},
+			};
+			generateMockHintsMock.mockResolvedValue(hints);
+
+			await service.executeWithLlmMock('wf-1', makeUser());
+
+			const runData = workflowRunner.run.mock.calls[0][0];
+			const item = runData.pinData?.['Webhook']?.[0];
+			const pdf = Buffer.from(item?.binary?.attachment_0.data ?? '', 'base64');
+			expect(pdf.toString('latin1')).toContain('(Invoice Number: INV-2026-0042) Tj');
+		});
+
+		it('does not add the requirement key when the embedded map names other files', async () => {
+			const hints = makeEmptyHints();
+			hints.triggerContent = {
+				body: {},
+				binary: { CV_0: { mimeType: 'application/pdf', fileName: 'cv.pdf' } },
+			};
+			generateMockHintsMock.mockResolvedValue(hints);
+			detectBinaryDependenciesMock.mockReturnValueOnce({
+				propertyName: 'data',
+				contentType: 'application/octet-stream',
+				filename: 'input.bin',
+			});
+
+			await service.executeWithLlmMock('wf-1', makeUser());
+
+			const runData = workflowRunner.run.mock.calls[0][0];
+			const item = runData.pinData?.['Webhook']?.[0];
+			expect(Object.keys(item?.binary ?? {})).toEqual(['CV_0']);
+		});
+
 		it('does not create pin data when triggerContent is empty', async () => {
 			const hints = makeEmptyHints();
 			hints.triggerContent = {};

packages/cli/src/modules/instance-ai/eval/__tests__/mock-handler.test.ts

@@ -270,6 +270,23 @@ describe('createLlmMockHandler', () => {
 		expect(sniffed?.ext).toBe('pdf');
 	});
 
+	it('should write textBody into a binary PDF response', async () => {
+		llmSubmits({
+			type: 'binary',
+			contentType: 'application/pdf',
+			filename: 'invoice.pdf',
+			textBody: 'Invoice Number: INV-2026-0042',
+		});
+		const handler = createLlmMockHandler();
+		const result = await callHandler(handler);
+
+		const sniffed = await fileTypeFromBuffer(result.body as Buffer);
+		expect(sniffed?.mime).toBe('application/pdf');
+		expect((result.body as Buffer).toString('latin1')).toContain(
+			'(Invoice Number: INV-2026-0042) Tj',
+		);
+	});
+
 	it('should populate content-disposition and content-length headers for binary responses', async () => {
 		llmSubmits({ type: 'binary', contentType: 'image/png', filename: 'logo.png' });
 		const handler = createLlmMockHandler();

packages/cli/src/modules/instance-ai/eval/__tests__/workflow-analysis.test.ts

@@ -1139,6 +1139,32 @@ describe('generateMockHints', () => {
 		expect(result.warnings).toEqual([]);
 	});
 
+	it('should unwrap trigger content that comes back as an n8n item', async () => {
+		const binary = { CV_0: { mimeType: 'application/pdf', fileName: 'cv.pdf' } };
+		mockAgentResponses(
+			JSON.stringify({
+				globalContext: '',
+				triggerContent: { json: { 'Full Name': 'Ada Lovelace' }, binary },
+				nodeHints: { Slack: 'post a message' },
+			}),
+		);
+
+		const result = await generateMockHints({ workflow, nodeNames: ['Schedule', 'Slack'] });
+
+		expect(result.triggerContent).toEqual({ 'Full Name': 'Ada Lovelace', binary });
+	});
+
+	it('should keep trigger content whose json field sits next to other fields', async () => {
+		const triggerContent = { json: { id: 1 }, headers: {}, body: {} };
+		mockAgentResponses(
+			JSON.stringify({ globalContext: '', triggerContent, nodeHints: { Slack: 'post' } }),
+		);
+
+		const result = await generateMockHints({ workflow, nodeNames: ['Schedule', 'Slack'] });
+
+		expect(result.triggerContent).toEqual(triggerContent);
+	});
+
 	it('should retry when the first attempt returns empty triggerContent, then succeed', async () => {
 		const generate = mockAgentResponses(
 			JSON.stringify({ globalContext: '', triggerContent: {}, nodeHints: { Slack: 'foo' } }),

packages/cli/src/modules/instance-ai/eval/execution.service.ts

@@ -973,7 +973,12 @@ export class EvalExecutionService {
 		const item: INodeExecutionData = { json: triggerContent as IDataObject };
 		const binary: IBinaryKeyData = {};
 
-		if (binaryRequirement) {
+		// The embedded files are what the trigger emits; the requirement only fills in when they name none.
+		const embeddedKeys = Object.keys(embedded);
+		if (
+			binaryRequirement &&
+			(embeddedKeys.length === 0 || embeddedKeys.includes(binaryRequirement.propertyName))
+		) {
 			// Requirement wins for its key, except embedded meta beats the generic fallback.
 			const embeddedMeta = embedded[binaryRequirement.propertyName];
 			const isGenericFallback =
@@ -982,6 +987,7 @@ export class EvalExecutionService {
 			binary[binaryRequirement.propertyName] = synthesizeBinaryEntry(
 				(isGenericFallback && embeddedMeta?.mimeType) || binaryRequirement.contentType,
 				(isGenericFallback && embeddedMeta?.fileName) || binaryRequirement.filename,
+				embeddedMeta?.text,
 			);
 		}
 
@@ -990,6 +996,7 @@ export class EvalExecutionService {
 			binary[key] = synthesizeBinaryEntry(
 				meta.mimeType ?? 'application/octet-stream',
 				meta.fileName ?? 'input.bin',
+				meta.text,
 			);
 		}
 
@@ -1538,8 +1545,8 @@ function withInterceptionGaps(
 }
 
 /** Synthesize a structurally valid binary entry (real bytes, base64-inlined). */
-function synthesizeBinaryEntry(contentType: string, filename: string): IBinaryData {
-	const bytes = synthesizeBinaryFixture(contentType, filename);
+function synthesizeBinaryEntry(contentType: string, filename: string, text?: string): IBinaryData {
+	const bytes = synthesizeBinaryFixture(contentType, filename, { text });
 	const extension = filename.includes('.') ? filename.slice(filename.lastIndexOf('.') + 1) : 'bin';
 	return {
 		mimeType: contentType,
@@ -1553,6 +1560,8 @@ function synthesizeBinaryEntry(contentType: string, filename: string): IBinaryDa
 interface EmbeddedBinaryMeta {
 	mimeType?: string;
 	fileName?: string;
+	/** The document's text, written into the synthesized file. */
+	text?: string;
 }
 
 function nonEmptyString(value: unknown): string | undefined {
@@ -1587,6 +1596,7 @@ function readEmbeddedBinaryMeta(
 			mimeType,
 			// `name` is a fileName fallback, not qualification evidence.
 			fileName: strictFileName ?? nonEmptyString(v.name),
+			text: nonEmptyString(v.text),
 		};
 	}
 

packages/cli/src/modules/instance-ai/eval/mock-handler.ts

@@ -76,7 +76,7 @@ Match THIS request only (URL + method): a node may make multiple sequential call
 
 For APIs that return empty responses on success (204/202), call submit_response with type="json" and body={}.
 
-**Binary / file responses.** Pick \`type: "binary"\` when the request URL or node parameters indicate a file download — Telegram \`getFile\` / \`/file/bot...\`, Google Drive \`alt=media\`, Dropbox \`/files/download\`, OneDrive \`/items/{id}/content\`, S3 \`GetObject\`, OpenAI \`audio/transcriptions\` source file, or any path containing \`/download\`, \`/file\`, \`/attachment\`, \`/media\`, \`/image\`, \`/voice\`, \`/audio\`, \`/export\`. Always set \`contentType\` (real MIME like \`application/pdf\`, \`audio/ogg\`, \`image/png\`) and \`filename\` (with the correct extension). Use \`sizeHint\` only when the scenario hints mention file size constraints (e.g. "rejects files > 100KB"). Do NOT pick \`binary\` for JSON metadata endpoints like Slack \`files.upload\`, \`files.info\`, or Telegram \`getFile\` (which returns a JSON envelope describing the file — the binary comes from the follow-up \`/file/bot.../path\` request). Exception: when the document at a download/export path is itself textual (CSV, XML, HTML, plain text) and the scenario or node hints specify its contents, use \`type: "text"\` with the exact document — reserve \`binary\` for opaque formats (PDF, images, audio, video, archives).
+**Binary / file responses.** Pick \`type: "binary"\` when the request URL or node parameters indicate a file download — Telegram \`getFile\` / \`/file/bot...\`, Google Drive \`alt=media\`, Dropbox \`/files/download\`, OneDrive \`/items/{id}/content\`, S3 \`GetObject\`, OpenAI \`audio/transcriptions\` source file, or any path containing \`/download\`, \`/file\`, \`/attachment\`, \`/media\`, \`/image\`, \`/voice\`, \`/audio\`, \`/export\`. Always set \`contentType\` (real MIME like \`application/pdf\`, \`audio/ogg\`, \`image/png\`) and \`filename\` (with the correct extension). For a PDF whose content the scenario or node hints state, also put that content in \`textBody\` as plain text — the harness writes it into the PDF; without it the PDF has no text. Use \`sizeHint\` only when the scenario hints mention file size constraints (e.g. "rejects files > 100KB"). Do NOT pick \`binary\` for JSON metadata endpoints like Slack \`files.upload\`, \`files.info\`, or Telegram \`getFile\` (which returns a JSON envelope describing the file — the binary comes from the follow-up \`/file/bot.../path\` request). Exception: when the document at a download/export path is itself textual (CSV, XML, HTML, plain text) and the scenario or node hints specify its contents, use \`type: "text"\` with the exact document — reserve \`binary\` for opaque formats (PDF, images, audio, video, archives).
 
 **Raw text / non-JSON responses.** When the real endpoint returns a non-JSON text document — XML/SOAP, CSV, HTML, RSS/Atom, plain text — use \`type: "text"\` and put the EXACT raw document in \`textBody\`, with \`contentType\` set to the real MIME (\`text/xml\`, \`application/xml\`, \`text/csv\`, \`text/html\`, \`text/plain\`). NEVER wrap such a document inside a JSON object like \`{"data": "<?xml..."}\` — the node consumes the raw text, and a JSON wrapper corrupts it. SOAP endpoints return the full SOAP envelope as \`textBody\`; SOAP faults use \`type: "error"\` with the fault XML in \`textBody\`.`;
 
@@ -628,7 +628,7 @@ const submitResponseSchema = z.object({
 		.string()
 		.optional()
 		.describe(
-			'The raw text document exactly as the service would send it (e.g. "<?xml version=\\"1.0\\"?>..."). Required for type="text"; allowed with type="error" when the service returns a text/XML error document. Omit otherwise.',
+			'The raw text document exactly as the service would send it (e.g. "<?xml version=\\"1.0\\"?>..."). Required for type="text"; allowed with type="error" when the service returns a text/XML error document; with type="binary" and a PDF, the document\'s text as plain text. Omit otherwise.',
 		),
 	statusCode: z
 		.number()
@@ -1019,7 +1019,10 @@ function materializeSpec(spec: MockResponseSpec): EvalMockHttpResponse {
 		case 'binary': {
 			const filename = spec.filename ?? 'mock-file.dat';
 			const contentType = spec.contentType ?? 'application/octet-stream';
-			const body = synthesizeBinaryFixture(contentType, filename, { sizeHint: spec.sizeHint });
+			const body = synthesizeBinaryFixture(contentType, filename, {
+				sizeHint: spec.sizeHint,
+				text: spec.textBody,
+			});
 			return {
 				body,
 				headers: {

packages/cli/src/modules/instance-ai/eval/workflow-analysis.ts

@@ -2,6 +2,7 @@ import { extractJsonCandidate } from '@n8n/ai-utilities/llm-output';
 import { Logger } from '@n8n/backend-common';
 import { Container } from '@n8n/di';
 import { createEvalAgent, extractText } from '@n8n/instance-ai';
+import { isRecord } from '@n8n/utils/is-record';
 // AI root node types (single source in @n8n/workflow-sdk mock-data) — lets
 // the typo guard accept a no-sub-node Agent.
 import { isAiRootNodeType } from '@n8n/workflow-sdk';
@@ -613,7 +614,7 @@ RULES:
    - CRITICAL: apart from a Manual Trigger, triggerContent must NEVER be an empty object ({}). Even for scenarios that test empty payloads ("empty submission", "no data", "missing fields"), emit the trigger envelope with empty *nested* fields — an empty webhook is { headers: {}, query: {}, body: {} }, a schedule with no context is { timestamp: "..." }. The workflow cannot execute without trigger output. The one exception is a polling or event trigger that the scenario says has NOTHING to emit ("no new emails", "no new rows", "no results"): then set "triggerEmitsNoItems": true and omit triggerContent — the harness pins the trigger to zero items so downstream nodes do not run.
    - CRITICAL: check what downstream nodes reference (e.g., $json.body.email, $json.subject, $json.text) and ensure those paths exist in triggerContent
    - CRITICAL: when the workflow has MULTIPLE trigger nodes, pick the ONE the Test Scenario targets (the trigger whose firing the scenario describes, e.g. "The weekly Schedule Trigger fires") and return its exact node name in a "startNodeName" field. triggerContent must be THAT trigger's output.
-   - CRITICAL: triggerContent must NEVER contain binary file CONTENT — no base64 blobs, no fake file-bytes placeholders. When the trigger carries a file (form upload, email attachment, incoming media), declare it with a METADATA-ONLY binary map instead: "binary": { "<propertyKey>": { "mimeType": "<real MIME>", "fileName": "<name.ext>" } } — the harness synthesizes real file bytes from that metadata and attaches them at the item level. The MIME type and file name MUST match the scenario: an image/png upload scenario needs mimeType "image/png" and a .png fileName, never a generic application/octet-stream. Use "data" as the propertyKey unless downstream nodes reference a different binary property name.
+   - CRITICAL: triggerContent must NEVER contain binary file CONTENT — no base64 blobs, no fake file-bytes placeholders. When the trigger carries a file (form upload, email attachment, incoming media), declare it with a METADATA-ONLY binary map instead: "binary": { "<propertyKey>": { "mimeType": "<real MIME>", "fileName": "<name.ext>" } } — the harness synthesizes real file bytes from that metadata and attaches them at the item level. For a PDF or text file whose words the scenario states, also add "text": "<those words as plain text>" to its entry — the harness writes them into the file. The MIME type and file name MUST match the scenario: an image/png upload scenario needs mimeType "image/png" and a .png fileName, never a generic application/octet-stream. Use "data" as the propertyKey unless downstream nodes reference a different binary property name.
 3. Create a "nodeHints" object with one entry per node. Each hint describes what data that specific node's API response should contain, referencing entities from the global context.
 4. Hints should describe the DATA CONTENT, not the API response format. The mock server already knows the API schema.
 5. Ensure data flows logically through the workflow. If node A fetches items that node B processes, the items in A's hint should match what B expects.
@@ -704,6 +705,14 @@ function isManualTriggerNode(workflow: IWorkflowBase, nodeName?: string): boolea
 	return node?.type === MANUAL_TRIGGER_NODE_TYPE && !node.disabled;
 }
 
+/** The model sometimes answers with an n8n item (`{ json, binary }`) instead of the item's json. */
+function unwrapItemShape(content: object): object {
+	if (!isRecord(content)) return content;
+	const { json, ...rest } = content;
+	if (!isRecord(json) || Object.keys(rest).some((key) => key !== 'binary')) return content;
+	return rest.binary === undefined ? json : { ...json, binary: rest.binary };
+}
+
 /** One LLM call → globalContext + triggerContent + per-node hints. Retried once on structural issues. */
 export async function generateMockHints(options: GenerateMockHintsOptions): Promise<MockHints> {
 	const { workflow, nodeNames, scenarioHints } = options;
@@ -759,12 +768,13 @@ export async function generateMockHints(options: GenerateMockHintsOptions): Prom
 			) {
 				reason = `invalid nodeHints structure (raw: ${text.slice(0, 200)})`;
 			} else {
-				const triggerContent =
+				const triggerContent = unwrapItemShape(
 					typeof parsed.triggerContent === 'object' &&
-					parsed.triggerContent !== null &&
-					!Array.isArray(parsed.triggerContent)
+						parsed.triggerContent !== null &&
+						!Array.isArray(parsed.triggerContent)
 						? parsed.triggerContent
-						: {};
+						: {},
+				);
 				// The model answers a bool as a word often enough to read both spellings.
 				const triggerEmitsNoItems =
 					parsed.triggerEmitsNoItems === true || parsed.triggerEmitsNoItems === 'true';

packages/core/src/execution-engine/__tests__/eval-mock-fixtures.test.ts

@@ -77,6 +77,29 @@ describe('eval-mock-fixtures', () => {
 			});
 		});
 
+		describe('text option', () => {
+			it('should write the text into a PDF', () => {
+				const buf = synthesizeBinaryFixture('application/pdf', 'invoice.pdf', {
+					text: 'Invoice Number: INV-7\nTotal Due: 12.50 EUR',
+				});
+				const pdf = buf.toString('latin1');
+				expect(pdf.startsWith('%PDF')).toBe(true);
+				expect(pdf).toContain('(Invoice Number: INV-7) Tj');
+				expect(pdf).toContain('(Total Due: 12.50 EUR) Tj');
+			});
+
+			it('should use the text as the body of a text MIME', () => {
+				const buf = synthesizeBinaryFixture('text/csv', 'prices.csv', {
+					text: 'sku,price\nA-1,9.99\n',
+				});
+				expect(buf.toString('utf8')).toBe('sku,price\nA-1,9.99\n');
+			});
+
+			it('should keep an explicitly empty text body for a text MIME', () => {
+				expect(synthesizeBinaryFixture('text/plain', 'empty.txt', { text: '' }).length).toBe(0);
+			});
+		});
+
 		describe('octet-stream fallback', () => {
 			it('should produce deterministic bytes for application/octet-stream', () => {
 				const a = synthesizeBinaryFixture('application/octet-stream', 'blob.dat');
@@ -251,9 +274,11 @@ describe('eval-mock-fixtures', () => {
 			expect(pdf.slice(startxref, startxref + 4)).toBe('xref');
 		});
 
-		it('replaces non-ASCII characters and skips blank lines', () => {
-			const pdf = buildPdfWithText('café £100\n\n  \nnext').toString('latin1');
-			expect(pdf).toContain('(caf? ?100) Tj');
+		it('keeps WinAnsi characters, replaces others and skips blank lines', () => {
+			const pdf = buildPdfWithText('café £100 10 €\n✓ ok\n\n  \nnext').toString('latin1');
+			expect(pdf).toContain('/Encoding/WinAnsiEncoding');
+			expect(pdf).toContain('(café £100 10 \x80) Tj');
+			expect(pdf).toContain('(? ok) Tj');
 			expect(pdf).toContain('(next) Tj');
 		});
 

packages/core/src/execution-engine/eval-mock-fixtures.ts

@@ -22,6 +22,8 @@ export interface SynthesizeBinaryFixtureOptions {
 	sizeHint?: FixtureSizeHint;
 	/** Scenario-pinned override — when present, returned untouched (Step 4 precedence). */
 	override?: Buffer;
+	/** The document's text: a PDF carries it as its text layer, a text MIME uses it as the body. */
+	text?: string;
 }
 
 // ---------------------------------------------------------------------------
@@ -267,8 +269,8 @@ export function evalCanvasPng(): Buffer {
 const PDF_TEXT_MAX_LINES = 40;
 
 /**
- * Build a valid one-page PDF whose extractable text is `text` (ASCII-printable
- * subset; other characters become `?`). The xref offsets are computed, so
+ * Build a valid one-page PDF whose extractable text is `text` (WinAnsi subset:
+ * Latin-1 plus `€`; other characters become `?`). The xref offsets are computed, so
  * strict parsers (pdfjs-dist behind Extract From File) accept it — LLM-authored
  * "PDF bytes" never parse, so mock layers substitute these and keep the
  * scenario's document CONTENT model-authored as plaintext.
@@ -276,7 +278,12 @@ const PDF_TEXT_MAX_LINES = 40;
 export function buildPdfWithText(text: string): Buffer {
 	const lines = text
 		.split(/\r?\n/)
-		.map((line) => line.replace(/[^\x20-\x7e]/g, '?').trimEnd())
+		.map((line) =>
+			line
+				.replace(/€/g, '\x80')
+				.replace(/[^\x20-\x7e\x80\xa0-\xff]/g, '?')
+				.trimEnd(),
+		)
 		.filter((line) => line.length > 0)
 		.slice(0, PDF_TEXT_MAX_LINES);
 	if (lines.length === 0) lines.push('Mock PDF document');
@@ -295,7 +302,7 @@ export function buildPdfWithText(text: string): Buffer {
 		'<</Type/Catalog/Pages 2 0 R>>',
 		'<</Type/Pages/Kids[3 0 R]/Count 1>>',
 		'<</Type/Page/Parent 2 0 R/MediaBox[0 0 612 792]/Resources<</Font<</F1 4 0 R>>>>/Contents 5 0 R>>',
-		'<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>',
+		'<</Type/Font/Subtype/Type1/BaseFont/Helvetica/Encoding/WinAnsiEncoding>>',
 		`<</Length ${String(Buffer.byteLength(contentStream, 'latin1'))}>>\nstream\n${contentStream}\nendstream`,
 	];
 
@@ -484,9 +491,10 @@ function applySizeHint(base: Buffer, hint: FixtureSizeHint, mime: string): Buffe
  *
  * Precedence:
  *  1. `options.override` — scenario-pinned bytes returned untouched.
- *  2. MIME-keyed minimal fixture (PDF, PNG, OGG, …).
- *  3. Text MIME plaintext stub seeded with `filename`.
- *  4. Deterministic-random 256-byte payload (octet-stream fallback).
+ *  2. `options.text` — a PDF with that text, or the text itself for a text MIME.
+ *  3. MIME-keyed minimal fixture (PDF, PNG, OGG, …).
+ *  4. Text MIME plaintext stub seeded with `filename`.
+ *  5. Deterministic-random 256-byte payload (octet-stream fallback).
  *
  * `options.sizeHint` only pads the tail of MIMEs whose decoder tolerates
  * trailing bytes (everything except ZIP-based formats and MP4).
@@ -501,6 +509,13 @@ export function synthesizeBinaryFixture(
 	const mime = (contentType || 'application/octet-stream').toLowerCase().split(';')[0].trim();
 	const hint: FixtureSizeHint = options.sizeHint ?? 'small';
 
+	if (options.text?.trim() && mime === 'application/pdf') {
+		return applySizeHint(buildPdfWithText(options.text), hint, mime);
+	}
+	if (options.text !== undefined && isTextMime(mime)) {
+		return applySizeHint(Buffer.from(options.text, 'utf8'), hint, mime);
+	}
+
 	const binary = pickBinaryFixture(mime, filename);
 	if (binary) return applySizeHint(binary, hint, mime);