Compare commits

...

1 Commits

Author SHA1 Message Date
Ryan Vogel 50dde6a646 fix(core): restore webfetch image output 2026-07-21 14:05:22 +00:00
3 changed files with 96 additions and 13 deletions
+37 -5
View File
@@ -36,6 +36,14 @@ const Output = Schema.Struct({
contentType: Schema.String, contentType: Schema.String,
format: Input.fields.format, format: Input.fields.format,
output: Schema.String, output: Schema.String,
data: Schema.String.pipe(Schema.optional),
mime: Schema.String.pipe(Schema.optional),
})
const StructuredOutput = Schema.Struct({
url: Output.fields.url,
contentType: Output.fields.contentType,
format: Output.fields.format,
output: Output.fields.output,
}) })
type Format = (typeof Input.Type)["format"] type Format = (typeof Input.Type)["format"]
@@ -104,7 +112,9 @@ const isTextualMime = (mime: string) =>
mime === "application/xml" || mime === "application/xml" ||
mime.endsWith("+xml") || mime.endsWith("+xml") ||
mime === "application/javascript" || mime === "application/javascript" ||
mime === "application/x-javascript" mime === "application/x-javascript" ||
mime === "image/svg+xml" ||
mime === "image/vnd.fastbidsheet"
const convert = (content: string, contentType: string, format: Format) => { const convert = (content: string, contentType: string, format: Format) => {
if (!contentType.includes("text/html")) return content if (!contentType.includes("text/html")) return content
if (format === "markdown") return convertHTMLToMarkdown(content) if (format === "markdown") return convertHTMLToMarkdown(content)
@@ -126,7 +136,21 @@ export const Plugin = {
description, description,
input: Input, input: Input,
output: Output, output: Output,
toModelOutput: ({ output }) => [{ type: "text", text: output.output }], structured: StructuredOutput,
// Image base64 belongs in the native file content only so it is not persisted twice.
toStructuredOutput: ({ output }) => ({
url: output.url,
contentType: output.contentType,
format: output.format,
output: output.output,
}),
toModelOutput: ({ input, output }) => {
if (output.data === undefined || output.mime === undefined) return [{ type: "text", text: output.output }]
return [
{ type: "text", text: output.output },
{ type: "file", data: output.data, mime: output.mime, name: input.url },
]
},
execute: (input, context) => execute: (input, context) =>
Effect.gen(function* () { Effect.gen(function* () {
yield* Effect.try({ yield* Effect.try({
@@ -150,9 +174,7 @@ export const Plugin = {
) )
const contentType = response.headers["content-type"] || "" const contentType = response.headers["content-type"] || ""
const mime = mimeFrom(contentType) const mime = mimeFrom(contentType)
if (isImageAttachment(mime)) if (!isTextualMime(mime) && !isImageAttachment(mime))
return yield* Effect.fail(new Error(`Unsupported fetched image content type: ${mime}`))
if (!isTextualMime(mime))
return yield* Effect.fail(new Error(`Unsupported fetched file content type: ${mime}`)) return yield* Effect.fail(new Error(`Unsupported fetched file content type: ${mime}`))
return { body: yield* collectBody(response), contentType } return { body: yield* collectBody(response), contentType }
}).pipe( }).pipe(
@@ -161,6 +183,16 @@ export const Plugin = {
orElse: () => Effect.fail(new Error("Request timed out")), orElse: () => Effect.fail(new Error("Request timed out")),
}), }),
) )
const mime = mimeFrom(contentType)
if (isImageAttachment(mime))
return {
url: input.url,
contentType,
format: input.format,
output: "Image fetched successfully",
data: Buffer.from(body).toString("base64"),
mime,
}
const content = new TextDecoder().decode(body) const content = new TextDecoder().decode(body)
const output = yield* Effect.try({ const output = yield* Effect.try({
try: () => convert(content, contentType, input.format), try: () => convert(content, contentType, input.format),
+52 -5
View File
@@ -234,14 +234,41 @@ describe("WebFetchTool registration", () => {
}), }),
) )
it.effect("keeps images and files unsupported until typed settlement can carry attachments", () => it.effect("returns images as native media and keeps other files unsupported", () =>
Effect.gen(function* () { Effect.gen(function* () {
reset() reset()
const registry = yield* ToolRegistry.Service const registry = yield* ToolRegistry.Service
respond = () => Effect.succeed(new Response("png", { headers: { "content-type": "image/png" } })) respond = () => Effect.succeed(new Response("png", { headers: { "content-type": "IMAGE/PNG; charset=binary" } }))
expect(yield* executeTool(registry, call({ url: "https://1.1.1.1/image", format: "html" }))).toEqual({ expect(yield* settleTool(registry, call({ url: "https://1.1.1.1/image", format: "html" }))).toEqual({
type: "error", result: {
value: "Unable to fetch https://1.1.1.1/image", type: "content",
value: [
{ type: "text", text: "Image fetched successfully" },
{
type: "file",
uri: "data:image/png;base64,cG5n",
mime: "image/png",
name: "https://1.1.1.1/image",
},
],
},
output: {
structured: {
url: "https://1.1.1.1/image",
contentType: "IMAGE/PNG; charset=binary",
format: "html",
output: "Image fetched successfully",
},
content: [
{ type: "text", text: "Image fetched successfully" },
{
type: "file",
uri: "data:image/png;base64,cG5n",
mime: "image/png",
name: "https://1.1.1.1/image",
},
],
},
}) })
respond = () => Effect.succeed(new Response("pdf", { headers: { "content-type": "application/pdf" } })) respond = () => Effect.succeed(new Response("pdf", { headers: { "content-type": "application/pdf" } }))
@@ -252,6 +279,26 @@ describe("WebFetchTool registration", () => {
}), }),
) )
it.effect("keeps SVG and FastBid sheets as text instead of image media", () =>
Effect.gen(function* () {
reset()
const registry = yield* ToolRegistry.Service
respond = () =>
Effect.succeed(new Response("<svg><text>hello</text></svg>", { headers: { "content-type": "image/svg+xml" } }))
expect(yield* executeTool(registry, call({ url: "https://1.1.1.1/image.svg", format: "html" }))).toEqual({
type: "text",
value: "<svg><text>hello</text></svg>",
})
respond = () =>
Effect.succeed(new Response("table Sheet {}", { headers: { "content-type": "image/vnd.fastbidsheet" } }))
expect(yield* executeTool(registry, call({ url: "https://1.1.1.1/sheet.fbs", format: "text" }))).toEqual({
type: "text",
value: "table Sheet {}",
})
}),
)
it.effect("retries Cloudflare challenges with an honest user agent", () => it.effect("retries Cloudflare challenges with an honest user agent", () =>
Effect.gen(function* () { Effect.gen(function* () {
reset() reset()
+7 -3
View File
@@ -17,6 +17,10 @@ and other binary prompt attachments are not currently included in the model
request. Some clients may let you select a PDF, but V2 does not yet make that request. Some clients may let you select a PDF, but V2 does not yet make that
PDF visible to the model. PDF visible to the model.
The built-in `webfetch` tool returns HTTP image responses as image media when
the response uses an `image/*` content type. SVG and FastBid sheet responses
remain text.
<Warning> <Warning>
Use a model that supports image input before attaching an image. OpenCode Use a model that supports image input before attaching an image. OpenCode
passes supported image media to the selected provider, but the provider and passes supported image media to the selected provider, but the provider and
@@ -133,9 +137,9 @@ All fields are optional:
<Note> <Note>
In the current V2 runtime, these settings apply to image media produced by In the current V2 runtime, these settings apply to image media produced by
the built-in `read` tool. Images attached directly through the TUI, desktop, built-in tools such as `read` and `webfetch`. Images attached directly
web, CLI, or API bypass this normalization. Resize direct attachments before through the TUI, desktop, web, CLI, or API bypass this normalization. Resize
adding them if the provider requires smaller media. direct attachments before adding them if the provider requires smaller media.
</Note> </Note>
The `read` tool recognizes PNG, JPEG, GIF, and WebP by their contents and will The `read` tool recognizes PNG, JPEG, GIF, and WebP by their contents and will