diff --git a/packages/core/test/tool-webfetch.test.ts b/packages/core/test/tool-webfetch.test.ts index 6a1617b3d74..d746fddf2b1 100644 --- a/packages/core/test/tool-webfetch.test.ts +++ b/packages/core/test/tool-webfetch.test.ts @@ -114,18 +114,24 @@ describe("WebFetchTool helpers", () => { expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe("before after") }) - test("is deterministic and bounded for malformed maximum-size input", () => { - const html = `

${"visible & text ".repeat(250_000)}

` + test("is deterministic and bounded for malformed input across parser chunks", () => { + const html = `

${"visible & text ".repeat(4_096)}

` const first = WebFetchTool.convertHTMLToMarkdown(html) expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(first) expect(first.startsWith("visible & text visible & text")).toBe(true) expect(first.length).toBeLessThanOrEqual(html.length) }) + test("defaults to the production byte budget with room for closing syntax", () => { + const output = WebFetchTool.convertHTMLToMarkdown("x".repeat(WebFetchTool.MAX_RESPONSE_BYTES)) + expect(WebFetchTool.MAX_RESPONSE_BYTES).toBe(5 * 1024 * 1024) + expect(output).toHaveLength(WebFetchTool.MAX_RESPONSE_BYTES - 64 * 1024) + }) + test("bounds deeply nested list output and fragmented code fences", () => { const lists = `${"".repeat(2_000)}` const quotes = `${"

item".repeat(2_000)}${"

".repeat(2_000)}` - const code = `
${"` x ".repeat(250_000)}
` + const code = `
${"` x ".repeat(4_096)}
` expect(WebFetchTool.convertHTMLToMarkdown(lists).length).toBeLessThan(lists.length * 4) expect(WebFetchTool.convertHTMLToMarkdown(quotes).length).toBeLessThan(quotes.length * 4) expect(() => WebFetchTool.convertHTMLToMarkdown(code)).not.toThrow() @@ -250,46 +256,35 @@ describe("WebFetchTool helpers", () => { expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`| a\\|b next | \`x\\|y\` |\n| --- | --- |`) }) - test("keeps each near-boundary inline construct closed and UTF-8-safe", () => { - const payload = "😀".repeat(WebFetchTool.MAX_RESPONSE_BYTES / 4) + test("preserves Unicode in inline constructs", () => { + const payload = "😀".repeat(16) const cases = [ - [`${payload}`, /^\*\*[\s\S]*\*\*$/], - [`${payload}`, /^\[[\s\S]*\]\(\/docs\)$/], - [`${payload}`, /^!\[[\s\S]*\]\(image\.png\)$/], - [`${payload}`, /^`[\s\S]*`$/], + [`${payload}`, `**${payload}**`], + [`${payload}`, `[${payload}](/docs)`], + [`${payload}`, `![${payload}](image.png)`], + [`${payload}`, `\`${payload}\``], ] as const - for (const [html, pattern] of cases) { - const output = WebFetchTool.convertHTMLToMarkdown(html) - expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES) - expect(output).not.toContain("�") - expect(output).toMatch(pattern) + for (const [html, expected] of cases) { + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(expected) } }) - test("keeps near-boundary block constructs syntactically complete", () => { - const payload = "x".repeat(WebFetchTool.MAX_RESPONSE_BYTES) + test("preserves block content and following lists", () => { + const payload = "x".repeat(256) const table = WebFetchTool.convertHTMLToMarkdown( `
Name
${payload}
`, ) - const list = WebFetchTool.convertHTMLToMarkdown(``) + const list = WebFetchTool.convertHTMLToMarkdown(``) const code = WebFetchTool.convertHTMLToMarkdown(`
${payload}
`) - for (const output of [table, list, code]) { - expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES) - expect(output).not.toContain("�") - } - expect(table).toMatch(/^\| Name \|\n\| --- \|\n\| [\s\S]* \|$/) - expect(list).toMatch(/^- [\s\S]*$/) - expect(list.includes("nested")).toBe(false) - expect(code.match(/^(`{3,}|~{3,})$/gm)).toHaveLength(2) + expect(table).toBe(`| Name |\n| --- |\n| ${payload} |`) + expect(list).toBe(`- ${payload}\n\n- next`) + expect(code).toBe(`\`\`\`\n${payload}\n\`\`\``) }) - test("keeps quoted code within budget with a safe closed fence", () => { - const html = `
${"`".repeat(32)}${"~".repeat(32)}${"x".repeat(WebFetchTool.MAX_RESPONSE_BYTES)}
` - const output = WebFetchTool.convertHTMLToMarkdown(html) - expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES) - const lines = output.split("\n") - expect(lines[0]).toMatch(/^> (`{33}|~{33})$/) - expect(lines.at(-1)).toBe(lines[0]) + test("keeps quoted code with long delimiter runs inside a safe closed fence", () => { + const payload = `${"`".repeat(32)}${"~".repeat(32)}${"x".repeat(64)}` + const output = WebFetchTool.convertHTMLToMarkdown(`
${payload}
`) + expect(output).toBe(`> ${"`".repeat(33)}\n> ${payload}\n> ${"`".repeat(33)}`) }) test("separates reconstructed tables from adjacent inline and quoted content", () => { @@ -299,13 +294,9 @@ describe("WebFetchTool helpers", () => { ) }) - test("keeps multiline quoted code closed at the content budget", () => { - const html = `
${"x\n".repeat(WebFetchTool.MAX_RESPONSE_BYTES / 2)}

tail

` - const output = WebFetchTool.convertHTMLToMarkdown(html) - expect(Buffer.byteLength(output)).toBeLessThanOrEqual(WebFetchTool.MAX_RESPONSE_BYTES) - expect((output.match(/(`{3}|~{3})/g) ?? []).length).toBe(2) - expect(output.includes("\uFFFD")).toBe(false) - expect(output.endsWith("tail")).toBe(true) + test("keeps multiline quoted code closed before following prose", () => { + const html = `
${"x\n".repeat(16)}

tail

` + expect(WebFetchTool.convertHTMLToMarkdown(html)).toBe(`> \`\`\`\n${"> x\n".repeat(16)}> \`\`\`\n\ntail`) }) test("keeps active content suppressed when depth fallback begins", () => {