From 100b2cf308aa1a08852d5e73ff9f3b8a3b33297f Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Fri, 14 Aug 2026 12:14:07 -0500 Subject: [PATCH] feat(opencode): route cloudflare ai gateway openai and anthropic models through native passthroughs Co-authored-by: Keefe Tang --- packages/opencode/src/plugin/cloudflare.ts | 11 - packages/opencode/src/provider/provider.ts | 31 ++- .../opencode/test/plugin/cloudflare.test.ts | 53 +--- .../test/provider/cf-ai-gateway-e2e.test.ts | 228 ++++++++++++++---- 4 files changed, 217 insertions(+), 106 deletions(-) diff --git a/packages/opencode/src/plugin/cloudflare.ts b/packages/opencode/src/plugin/cloudflare.ts index c4bf6bb8e70..2ccf5168d8a 100644 --- a/packages/opencode/src/plugin/cloudflare.ts +++ b/packages/opencode/src/plugin/cloudflare.ts @@ -61,16 +61,5 @@ export async function CloudflareAIGatewayAuthPlugin(_input: PluginInput): Promis }, ], }, - "chat.params": async (input, output) => { - if (input.model.providerID !== "cloudflare-ai-gateway") return - // The unified gateway routes through @ai-sdk/openai-compatible, which - // always emits max_tokens. OpenAI reasoning models (gpt-5.x, o-series) - // reject that field and require max_completion_tokens instead, and the - // compatible SDK has no way to rename it. Drop the cap so OpenAI falls - // back to the model's default output budget. - if (!input.model.api.id.toLowerCase().startsWith("openai/")) return - if (!input.model.capabilities.reasoning) return - output.maxOutputTokens = undefined - }, } } diff --git a/packages/opencode/src/provider/provider.ts b/packages/opencode/src/provider/provider.ts index 44a5b3f6213..ab3ca72f225 100644 --- a/packages/opencode/src/provider/provider.ts +++ b/packages/opencode/src/provider/provider.ts @@ -800,9 +800,10 @@ function custom(dep: CustomDep): Record { ) } - // Use official ai-gateway-provider package (v2.x for AI SDK v5 compatibility) const { createAiGateway } = yield* Effect.promise(() => import("ai-gateway-provider")) const { createUnified } = yield* Effect.promise(() => import("ai-gateway-provider/providers/unified")) + const { createOpenAI } = yield* Effect.promise(() => import("ai-gateway-provider/providers/openai")) + const { createAnthropic } = yield* Effect.promise(() => import("ai-gateway-provider/providers/anthropic")) const metadata = iife(() => { if (input.options?.metadata) return input.options.metadata @@ -833,6 +834,13 @@ function custom(dep: CustomDep): Record { autoload: true, async getModel(_sdk: any, modelID: string, _options?: Record) { // Model IDs use Unified API format: provider/model (e.g., "anthropic/claude-sonnet-4-5"). + // OpenAI and Anthropic ride their native passthrough routes so agents get the Responses + // and Messages APIs; new OpenAI models reject tools+reasoning_effort on chat completions. + // The passthrough wrappers inject a CF_TEMP_TOKEN sentinel that the gateway strips before + // dispatch, so upstream billing stays on the gateway (Unified Billing / stored BYOK). + if (modelID.startsWith("openai/")) return aigateway(createOpenAI()(modelID.slice("openai/".length))) + if (modelID.startsWith("anthropic/")) + return aigateway(createAnthropic()(modelID.slice("anthropic/".length))) // Workers AI is the only first-party provider whose upstream is Cloudflare itself, so it is // the only one that should receive the Cloudflare token as its upstream Authorization header. // The Unified API addresses Workers AI both with the explicit "workers-ai/" prefix and as @@ -1214,6 +1222,17 @@ function cost(c: ModelsDev.Model["cost"]): Model["cost"] { return result } +// Cloudflare AI Gateway routes OpenAI and Anthropic models through their native +// passthrough SDKs (Responses / Messages APIs). Resolving the native npm before +// variants are computed makes reasoning variants produce payloads the native +// SDKs understand (e.g. anthropic `effort` instead of compat `reasoningEffort`). +function cloudflareGatewayNpm(providerID: string, modelID: string) { + if (providerID !== "cloudflare-ai-gateway") return undefined + if (modelID.startsWith("openai/")) return "@ai-sdk/openai" + if (modelID.startsWith("anthropic/")) return "@ai-sdk/anthropic" + return undefined +} + function fromModelsDevModel(provider: ModelsDev.Provider, model: ModelsDev.Model): Model { const base: Model = { id: ModelV2.ID.make(model.id), @@ -1223,7 +1242,11 @@ function fromModelsDevModel(provider: ModelsDev.Provider, model: ModelsDev.Model api: { id: model.id, url: model.provider?.api ?? provider.api ?? "", - npm: model.provider?.npm ?? provider.npm ?? "@ai-sdk/openai-compatible", + npm: + cloudflareGatewayNpm(provider.id, model.id) ?? + model.provider?.npm ?? + provider.npm ?? + "@ai-sdk/openai-compatible", }, status: model.status ?? "active", headers: {}, @@ -1445,6 +1468,9 @@ const layer = Layer.effect( model.provider?.npm ?? provider.npm ?? existingModel?.api.npm ?? + // Config-defined gateway models bypass fromModelsDevModel, so resolve the + // native passthrough npm here before falling back to the catalog default. + cloudflareGatewayNpm(providerID, apiID) ?? modelsDev[providerID]?.npm ?? "@ai-sdk/openai-compatible" const name = iife(() => { @@ -1624,6 +1650,7 @@ const layer = Layer.effect( for (const [modelID, model] of Object.entries(provider.models)) { model.api.id = model.api.id ?? model.id ?? modelID + if ( // These chat aliases are invalid for the special handling in the // built-in providers below, but custom providers may support them. diff --git a/packages/opencode/test/plugin/cloudflare.test.ts b/packages/opencode/test/plugin/cloudflare.test.ts index 5fa41068358..ab3d27df47c 100644 --- a/packages/opencode/test/plugin/cloudflare.test.ts +++ b/packages/opencode/test/plugin/cloudflare.test.ts @@ -13,56 +13,13 @@ const pluginInput = { $: {} as never, } -function makeHookInput(overrides: { providerID?: string; apiId?: string; reasoning?: boolean }) { - return { - sessionID: "s", - agent: "a", - provider: {} as never, - message: {} as never, - model: { - providerID: overrides.providerID ?? "cloudflare-ai-gateway", - api: { id: overrides.apiId ?? "openai/gpt-5.2-codex", url: "", npm: "ai-gateway-provider" }, - capabilities: { - reasoning: overrides.reasoning ?? true, - temperature: false, - attachment: true, - toolcall: true, - input: { text: true, audio: false, image: false, video: false, pdf: false }, - output: { text: true, audio: false, image: false, video: false, pdf: false }, - interleaved: false, - }, - } as never, - } -} - -function makeHookOutput() { - return { temperature: 0, topP: 1, topK: 0, maxOutputTokens: 32_000 as number | undefined, options: {} } -} - -test("omits maxOutputTokens for openai reasoning models on cloudflare-ai-gateway", async () => { +test("registers the cloudflare-ai-gateway auth method", async () => { const hooks = await CloudflareAIGatewayAuthPlugin(pluginInput) - const out = makeHookOutput() - await hooks["chat.params"]!(makeHookInput({ apiId: "openai/gpt-5.2-codex", reasoning: true }), out) - expect(out.maxOutputTokens).toBeUndefined() + expect(hooks.auth?.provider).toBe("cloudflare-ai-gateway") + expect(hooks.auth?.methods).toHaveLength(1) }) -test("keeps maxOutputTokens for openai non-reasoning models", async () => { +test("no longer drops maxOutputTokens; OpenAI models ride the Responses API passthrough", async () => { const hooks = await CloudflareAIGatewayAuthPlugin(pluginInput) - const out = makeHookOutput() - await hooks["chat.params"]!(makeHookInput({ apiId: "openai/gpt-4-turbo", reasoning: false }), out) - expect(out.maxOutputTokens).toBe(32_000) -}) - -test("keeps maxOutputTokens for non-openai reasoning models on cloudflare-ai-gateway", async () => { - const hooks = await CloudflareAIGatewayAuthPlugin(pluginInput) - const out = makeHookOutput() - await hooks["chat.params"]!(makeHookInput({ apiId: "anthropic/claude-sonnet-4-5", reasoning: true }), out) - expect(out.maxOutputTokens).toBe(32_000) -}) - -test("ignores non-cloudflare-ai-gateway providers", async () => { - const hooks = await CloudflareAIGatewayAuthPlugin(pluginInput) - const out = makeHookOutput() - await hooks["chat.params"]!(makeHookInput({ providerID: "openai", apiId: "gpt-5.2-codex", reasoning: true }), out) - expect(out.maxOutputTokens).toBe(32_000) + expect(hooks["chat.params"]).toBeUndefined() }) diff --git a/packages/opencode/test/provider/cf-ai-gateway-e2e.test.ts b/packages/opencode/test/provider/cf-ai-gateway-e2e.test.ts index 603a2ecd1b9..cb1654006e6 100644 --- a/packages/opencode/test/provider/cf-ai-gateway-e2e.test.ts +++ b/packages/opencode/test/provider/cf-ai-gateway-e2e.test.ts @@ -1,16 +1,20 @@ -// End-to-end regression test for opencode#24432. +// End-to-end regression tests for opencode#24432 and opencode#32051/#32052. // -// Routes through the actual ai-gateway-provider + @ai-sdk/openai-compatible -// chain that provider.ts:811 builds at runtime, with only the network boundary -// stubbed. Asserts that `reasoning_effort` (and other provider options the -// transform emits) actually land in the body Cloudflare AI Gateway forwards -// upstream, which is the only place the bug was observable. +// Routes through the actual ai-gateway-provider chain that provider.ts builds at +// runtime, with only the network boundary stubbed: +// - openai/* -> native OpenAI passthrough (Responses API) +// - anthropic/* -> native Anthropic passthrough (Messages API) +// - everything else -> unified /compat (openai-compatible chat completions) +// Asserts what actually lands in the envelope body Cloudflare AI Gateway +// forwards upstream, which is the only place these bugs were observable. import { afterEach, beforeEach, describe, expect, test } from "bun:test" import type { JSONValue } from "ai" import { generateText } from "ai" import { createAiGateway } from "ai-gateway-provider" import { createUnified } from "ai-gateway-provider/providers/unified" +import { createOpenAI } from "ai-gateway-provider/providers/openai" +import { createAnthropic } from "ai-gateway-provider/providers/anthropic" import { ProviderTransform } from "@/provider/transform" import type * as Provider from "@/provider/provider" import { ProviderV2 } from "@opencode-ai/core/provider" @@ -26,28 +30,76 @@ function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value) } +// The gateway returns the upstream provider's response body verbatim, so the +// mock must answer in the wire format of the step's target provider. +function upstreamResponseBody(provider: string | undefined) { + if (provider === "openai") + return { + id: "resp_test", + object: "response", + created_at: 0, + model: "gpt-5.4", + status: "completed", + error: null, + incomplete_details: null, + output: [ + { + type: "message", + role: "assistant", + id: "msg_1", + status: "completed", + content: [{ type: "output_text", text: "ok", annotations: [] }], + }, + ], + usage: { + input_tokens: 1, + input_tokens_details: { cached_tokens: 0 }, + output_tokens: 1, + output_tokens_details: { reasoning_tokens: 0 }, + total_tokens: 2, + }, + } + if (provider === "anthropic") + return { + id: "msg_test", + type: "message", + role: "assistant", + model: "claude-sonnet-4-6", + content: [{ type: "text", text: "ok" }], + stop_reason: "end_turn", + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 1 }, + } + return { + id: "chatcmpl-test", + object: "chat.completion", + created: 0, + model: "test", + choices: [{ index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + } +} + beforeEach(() => { captured = null const handle = async (input: Parameters[0], init?: Parameters[1]): Promise => { const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url if (url.startsWith("https://gateway.ai.cloudflare.com/")) { const bodyText = typeof init?.body === "string" ? init.body : "" + const outerBody = bodyText ? JSON.parse(bodyText) : null captured = { url, - outerBody: bodyText ? JSON.parse(bodyText) : null, + outerBody, headers: Object.fromEntries(new Headers(init?.headers).entries()), } - return new Response( - JSON.stringify({ - id: "chatcmpl-test", - object: "chat.completion", - created: 0, - model: "openai/gpt-5.4", - choices: [{ index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], - usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, - }), - { status: 200, headers: { "Content-Type": "application/json" } }, - ) + const provider = + Array.isArray(outerBody) && isRecord(outerBody[0]) && typeof outerBody[0].provider === "string" + ? outerBody[0].provider + : undefined + return new Response(JSON.stringify(upstreamResponseBody(provider)), { + status: 200, + headers: { "Content-Type": "application/json" }, + }) } return realFetch(input, init) } @@ -60,11 +112,19 @@ afterEach(() => { globalThis.fetch = realFetch }) +// Mirrors the runtime npm rewrite in provider.ts: openai/anthropic models carry +// their native SDK package so transforms key provider options correctly. +const cfNpm = (apiId: string) => { + if (apiId.startsWith("openai/")) return "@ai-sdk/openai" + if (apiId.startsWith("anthropic/")) return "@ai-sdk/anthropic" + return "ai-gateway-provider" +} + const cfModel = (apiId: string, releaseDate = "2026-03-05"): Provider.Model => ({ id: ModelV2.ID.make(`cloudflare-ai-gateway/${apiId}`), providerID: ProviderV2.ID.make("cloudflare-ai-gateway"), name: apiId, - api: { id: apiId, url: "https://gateway.ai.cloudflare.com/v1/compat", npm: "ai-gateway-provider" }, + api: { id: apiId, url: "https://gateway.ai.cloudflare.com/v1/compat", npm: cfNpm(apiId) }, capabilities: { reasoning: true, temperature: false, @@ -84,68 +144,138 @@ const cfModel = (apiId: string, releaseDate = "2026-03-05"): Provider.Model => ( // ai-gateway-provider sends an array of step descriptors; each entry's `query` // is the body forwarded to the upstream provider. -function extractUpstreamQuery(body: unknown): Record | undefined { +function firstStep(body: unknown): Record | undefined { if (!Array.isArray(body) || body.length === 0) return undefined const first = body[0] - if (!isRecord(first)) return undefined - const query = first.query + return isRecord(first) ? first : undefined +} + +function extractUpstreamQuery(body: unknown): Record | undefined { + const query = firstStep(body)?.query return isRecord(query) ? query : undefined } // Each step descriptor also carries the `headers` forwarded to the upstream provider. function extractUpstreamHeaders(body: unknown): Record | undefined { - if (!Array.isArray(body) || body.length === 0) return undefined - const first = body[0] - if (!isRecord(first)) return undefined - const headers = first.headers + const headers = firstStep(body)?.headers return isRecord(headers) ? headers : undefined } -async function callThroughGateway(apiId: string, providerOptions: ProviderOptions, gatewayToken = "test") { +// Mirrors the runtime routing in provider.ts getModel. +function gatewayModel(apiId: string, gatewayToken = "test") { const aigateway = createAiGateway({ accountId: "test", gateway: "test", apiKey: gatewayToken }) - // Mirrors the runtime: only first-party Workers AI sub-requests (workers-ai/ or bare @cf/) get the token. + if (apiId.startsWith("openai/")) return aigateway(createOpenAI()(apiId.slice("openai/".length))) + if (apiId.startsWith("anthropic/")) return aigateway(createAnthropic()(apiId.slice("anthropic/".length))) const isWorkersAi = apiId.startsWith("workers-ai/") || apiId.startsWith("@cf/") const unified = createUnified(isWorkersAi ? { apiKey: gatewayToken } : {}) - await generateText({ model: aigateway(unified(apiId)), prompt: "hi", providerOptions }) + return aigateway(unified(apiId)) +} + +async function callThroughGateway(apiId: string, providerOptions: ProviderOptions, gatewayToken = "test") { + await generateText({ model: gatewayModel(apiId, gatewayToken), prompt: "hi", providerOptions }) return extractUpstreamQuery(captured?.outerBody) } +describe("cf-ai-gateway routing", () => { + test("openai/* rides the native OpenAI passthrough on the Responses API", async () => { + await callThroughGateway("openai/gpt-5.4", {}) + const step = firstStep(captured?.outerBody) + expect(step?.provider).toBe("openai") + expect(step?.endpoint).toBe("v1/responses") + const upstream = extractUpstreamQuery(captured?.outerBody) + expect(upstream?.model).toBe("gpt-5.4") + }) + + test("anthropic/* rides the native Anthropic passthrough on the Messages API", async () => { + await callThroughGateway("anthropic/claude-sonnet-4-6", {}) + const step = firstStep(captured?.outerBody) + expect(step?.provider).toBe("anthropic") + expect(step?.endpoint).toBe("v1/messages") + const upstream = extractUpstreamQuery(captured?.outerBody) + expect(upstream?.model).toBe("claude-sonnet-4-6") + }) + + test("workers-ai models stay on the unified /compat route", async () => { + await callThroughGateway("workers-ai/@cf/moonshotai/kimi-k2.6", {}) + const step = firstStep(captured?.outerBody) + expect(step?.provider).toBe("compat") + expect(step?.endpoint).toBe("chat/completions") + const upstream = extractUpstreamQuery(captured?.outerBody) + expect(upstream?.model).toBe("workers-ai/@cf/moonshotai/kimi-k2.6") + }) +}) + describe("cf-ai-gateway end-to-end (regression: #24432)", () => { - test("ProviderTransform.providerOptions output puts reasoning_effort on the wire", async () => { - // The full chain the runtime exercises: - // transform.providerOptions() -> openaiCompatible key - // -> @ai-sdk/openai-compatible reads it as compatibleOptions - // -> emits body.reasoning_effort + test("ProviderTransform.providerOptions output puts reasoning effort on the Responses wire", async () => { + // The full chain the runtime exercises for OpenAI models: + // transform.providerOptions() -> "openai" key (npm rewritten to @ai-sdk/openai) + // -> OpenAIResponsesLanguageModel emits body.reasoning.effort // -> ai-gateway-provider wraps the body and forwards to gateway.ai.cloudflare.com const opts = ProviderTransform.providerOptions(cfModel("openai/gpt-5.4"), { reasoningEffort: "xhigh" }) - expect(opts).toEqual({ openaiCompatible: { reasoningEffort: "xhigh" } }) + expect(Object.keys(opts)).toEqual(["openai"]) + expect(opts.openai.reasoningEffort).toBe("xhigh") const upstream = await callThroughGateway("openai/gpt-5.4", opts) - expect(upstream?.reasoning_effort).toBe("xhigh") + expect((upstream?.reasoning as Record | undefined)?.effort).toBe("xhigh") }) test("variants() output for openai/gpt-5.4 lands xhigh on the wire", async () => { - // The other half of the bug: workflow `variant: xhigh` flows through variants() - // and must reach the wire. variants() returns the providerOptions payload - // unwrapped; providerOptions() wraps it under the SDK key. + // fromModelsDevModel resolves the native npm before computing variants, so + // OpenAI models get full Responses-flavored payloads (summary + encrypted + // reasoning include for stateless multi-turn reasoning). const variants = ProviderTransform.variants(cfModel("openai/gpt-5.4")) - expect(variants.xhigh).toEqual({ reasoningEffort: "xhigh" }) + expect(variants.xhigh).toEqual({ + reasoningEffort: "xhigh", + reasoningSummary: "auto", + include: ["reasoning.encrypted_content"], + }) const opts = ProviderTransform.providerOptions(cfModel("openai/gpt-5.4"), variants.xhigh) const upstream = await callThroughGateway("openai/gpt-5.4", opts) - expect(upstream?.reasoning_effort).toBe("xhigh") + const reasoning = upstream?.reasoning as Record | undefined + expect(reasoning?.effort).toBe("xhigh") + expect(reasoning?.summary).toBe("auto") + }) + + test("reasoning effort variants for anthropic models land as native adaptive thinking", async () => { + // Mirrors the runtime catalog path: models.dev reasoning_options -> reasoningVariants + // computed on the native @ai-sdk/anthropic npm -> adaptive thinking + output_config.effort. + const model = cfModel("anthropic/claude-sonnet-4-6") + const variants = ProviderTransform.reasoningVariants( + { reasoning_options: [{ type: "effort", values: ["low", "medium", "high"] }] } as never, + model, + ) + expect(variants?.high).toMatchObject({ effort: "high" }) + + const opts = ProviderTransform.providerOptions(model, variants!.high) + expect(Object.keys(opts)).toEqual(["anthropic"]) + + const upstream = await callThroughGateway("anthropic/claude-sonnet-4-6", opts) + expect((upstream?.thinking as Record | undefined)?.type).toBe("adaptive") + expect((upstream?.output_config as Record | undefined)?.effort).toBe("high") + }) + + test("reasoning_effort still reaches the /compat wire for workers-ai models", async () => { + const model = cfModel("workers-ai/@cf/moonshotai/kimi-k2.6") + const opts = ProviderTransform.providerOptions(model, { reasoningEffort: "high" }) + expect(opts).toEqual({ openaiCompatible: { reasoningEffort: "high" } }) + + const upstream = await callThroughGateway("workers-ai/@cf/moonshotai/kimi-k2.6", opts) + expect(upstream?.reasoning_effort).toBe("high") }) test("legacy buggy key 'cloudflare-ai-gateway' does NOT reach the wire (proves the bug)", async () => { // Sanity: confirms the bug class. If a future change accidentally restores // providerID-keyed providerOptions, this test fails before users notice. - const upstream = await callThroughGateway("openai/gpt-5.4", { + const upstream = await callThroughGateway("workers-ai/@cf/moonshotai/kimi-k2.6", { "cloudflare-ai-gateway": { reasoningEffort: "high" }, }) expect(upstream?.reasoning_effort).toBeUndefined() }) +}) - test("third-party models do NOT forward the Cloudflare token upstream (regression: #32052)", async () => { +describe("cf-ai-gateway token scoping (regression: #32051/#32052)", () => { + test("openai passthrough does NOT forward the Cloudflare token upstream", async () => { await callThroughGateway("openai/gpt-5.4", {}, "cf-gateway-secret") expect(captured?.headers["cf-aig-authorization"]).toBe("Bearer cf-gateway-secret") @@ -154,14 +284,22 @@ describe("cf-ai-gateway end-to-end (regression: #24432)", () => { expect(JSON.stringify(captured?.outerBody)).not.toContain("cf-gateway-secret") }) - test("workers-ai models DO forward the Cloudflare token upstream (regression: #32051)", async () => { + test("anthropic passthrough does NOT forward the Cloudflare token upstream", async () => { + await callThroughGateway("anthropic/claude-sonnet-4-6", {}, "cf-gateway-secret") + + expect(captured?.headers["cf-aig-authorization"]).toBe("Bearer cf-gateway-secret") + expect(extractUpstreamHeaders(captured?.outerBody)?.["x-api-key"]).toBeUndefined() + expect(JSON.stringify(captured?.outerBody)).not.toContain("cf-gateway-secret") + }) + + test("workers-ai models DO forward the Cloudflare token upstream", async () => { await callThroughGateway("workers-ai/@cf/google/gemma-4-26b-a4b-it", {}, "cf-gateway-secret") expect(captured?.headers["cf-aig-authorization"]).toBe("Bearer cf-gateway-secret") expect(extractUpstreamHeaders(captured?.outerBody)?.["authorization"]).toBe("Bearer cf-gateway-secret") }) - test("bare @cf/ Workers AI models DO forward the Cloudflare token upstream (regression: #32051)", async () => { + test("bare @cf/ Workers AI models DO forward the Cloudflare token upstream", async () => { await callThroughGateway("@cf/meta/llama-3.1-8b-instruct", {}, "cf-gateway-secret") expect(captured?.headers["cf-aig-authorization"]).toBe("Bearer cf-gateway-secret")