From 9ca650f97cd9a1557b2fc1f2b0a08e70fd01ac3e Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:43:00 -0500 Subject: [PATCH] refactor(ai): promote prompt cache key (#39965) --- packages/ai/README.md | 7 ++++--- packages/ai/example/tutorial.ts | 9 ++++----- packages/ai/src/protocols/open-responses.ts | 2 +- .../ai/src/protocols/utils/open-responses-options.ts | 2 -- packages/ai/src/providers/open-responses-options.ts | 1 - packages/ai/src/providers/openai-options.ts | 1 - packages/ai/src/providers/openrouter.ts | 3 +-- packages/ai/src/schema/messages.ts | 3 +++ packages/ai/test/provider-options/cloudflare.types.ts | 6 +++--- .../provider/openai-responses-cache.recorded.test.ts | 2 +- packages/ai/test/provider/openai-responses.test.ts | 7 +++---- packages/ai/test/provider/openrouter.test.ts | 2 +- packages/core/src/aisdk-native.ts | 2 +- packages/core/src/session/compaction.ts | 2 ++ packages/core/src/session/generate-node.ts | 6 ++---- packages/core/src/session/model-request.ts | 4 ++-- packages/core/src/session/prompt-cache-key.ts | 6 ++++++ packages/core/test/aisdk-native.test.ts | 4 ---- packages/core/test/session-compaction.test.ts | 1 + packages/core/test/session-generate.test.ts | 2 +- packages/core/test/session-runner.test.ts | 4 ++-- 21 files changed, 38 insertions(+), 38 deletions(-) create mode 100644 packages/core/src/session/prompt-cache-key.ts diff --git a/packages/ai/README.md b/packages/ai/README.md index 6f088d2b2a4..048db1620ba 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -368,11 +368,12 @@ Other provider exports listed above remain direct facades until they explicitly ## Provider options & HTTP overlays -Three escape hatches in order of stability: +Request options in order of stability: 1. **`generation`** — portable knobs (`maxTokens`, `temperature`, `topP`, `topK`, penalties, seed, stop). -2. **`providerOptions: { : {...} }`** — typed-at-the-facade provider-specific knobs (OpenAI `promptCacheKey`, Anthropic `thinking`, Gemini `thinkingConfig`, OpenRouter routing). -3. **`http: { body, headers, query }`** — last-resort serializable overlays merged into the final HTTP request. Reach for this only when a stable typed path doesn't yet exist. +2. **`promptCacheKey`** — stable cache affinity lowered by every protocol that supports it. +3. **`providerOptions: { : {...} }`** — typed-at-the-facade provider-specific knobs (OpenAI `store`, Anthropic `thinking`, Gemini `thinkingConfig`, OpenRouter routing). +4. **`http: { body, headers, query }`** — last-resort serializable overlays merged into the final HTTP request. Reach for this only when a stable typed path doesn't yet exist. Route/provider defaults are overridden by request-level values for each axis. diff --git a/packages/ai/example/tutorial.ts b/packages/ai/example/tutorial.ts index 3924a57dd28..47af153c461 100644 --- a/packages/ai/example/tutorial.ts +++ b/packages/ai/example/tutorial.ts @@ -33,9 +33,10 @@ const model = OpenAI.configure({ // // - `generation`: common controls such as max tokens, temperature, topP/topK, // penalties, seed, and stop sequences. +// - `promptCacheKey`: stable cache affinity for protocols that support it. // - `providerOptions`: namespaced provider-native behavior. For example, -// OpenAI cache keys and store behavior, Anthropic thinking, Gemini thinking -// config, or OpenRouter routing/reasoning. +// OpenAI store behavior, Anthropic thinking, Gemini thinking config, or +// OpenRouter routing/reasoning. // - `http`: last-resort serializable overlays for final request body, headers, // and query params. Prefer typed `providerOptions` when a field is stable. // @@ -45,9 +46,7 @@ const request = LLM.request({ system: "You are concise and practical.", prompt: "Tell me a joke", generation: { maxTokens: 80, temperature: 0.7 }, - providerOptions: { - openai: { promptCacheKey: "tutorial-joke" }, - }, + promptCacheKey: "tutorial-joke", }) // 3. `generate` sends the request and collects the event stream into one diff --git a/packages/ai/src/protocols/open-responses.ts b/packages/ai/src/protocols/open-responses.ts index c34f509bbb8..03068b24fc1 100644 --- a/packages/ai/src/protocols/open-responses.ts +++ b/packages/ai/src/protocols/open-responses.ts @@ -539,7 +539,7 @@ const lowerOptions = (request: LLMRequest) => { return { ...(options.instructions ? { instructions: options.instructions } : {}), ...(options.store !== undefined ? { store: options.store } : {}), - ...(options.promptCacheKey ? { prompt_cache_key: options.promptCacheKey } : {}), + ...(request.promptCacheKey ? { prompt_cache_key: request.promptCacheKey } : {}), ...(options.include ? { include: options.include } : {}), ...(options.reasoningEffort || options.reasoningSummary ? { reasoning: { effort: options.reasoningEffort, summary: options.reasoningSummary } } diff --git a/packages/ai/src/protocols/utils/open-responses-options.ts b/packages/ai/src/protocols/utils/open-responses-options.ts index 80288ca5cb1..38a0e7063f1 100644 --- a/packages/ai/src/protocols/utils/open-responses-options.ts +++ b/packages/ai/src/protocols/utils/open-responses-options.ts @@ -33,7 +33,6 @@ export const ServiceTierSchema = Schema.Literals(ServiceTiers) export interface Resolved { readonly instructions?: string readonly store?: boolean - readonly promptCacheKey?: string readonly reasoningEffort?: string readonly reasoningSummary?: "auto" | "concise" | "detailed" readonly include?: ReadonlyArray @@ -50,7 +49,6 @@ export const resolve = (request: LLMRequest): Resolved => { return { instructions: typeof input?.instructions === "string" ? input.instructions : undefined, store: typeof input?.store === "boolean" ? input.store : undefined, - promptCacheKey: typeof input?.promptCacheKey === "string" ? input.promptCacheKey : undefined, reasoningEffort: typeof input?.reasoningEffort === "string" ? input.reasoningEffort : undefined, reasoningSummary: reasoningSummary === "auto" || reasoningSummary === "concise" || reasoningSummary === "detailed" diff --git a/packages/ai/src/providers/open-responses-options.ts b/packages/ai/src/providers/open-responses-options.ts index 686cdcb8083..b0194da2a46 100644 --- a/packages/ai/src/providers/open-responses-options.ts +++ b/packages/ai/src/providers/open-responses-options.ts @@ -5,7 +5,6 @@ export interface OpenResponsesOptionsInput { readonly [key: string]: unknown readonly instructions?: string readonly store?: boolean - readonly promptCacheKey?: string readonly reasoningEffort?: ReasoningEffort readonly reasoningSummary?: "auto" | "concise" | "detailed" readonly include?: ReadonlyArray diff --git a/packages/ai/src/providers/openai-options.ts b/packages/ai/src/providers/openai-options.ts index 86c7c2e3345..c2336b59686 100644 --- a/packages/ai/src/providers/openai-options.ts +++ b/packages/ai/src/providers/openai-options.ts @@ -17,7 +17,6 @@ const openAIProviderOptions = (options: OpenAIOptionsInput | undefined): Provide const openai = Object.fromEntries( definedEntries({ store: options?.store, - promptCacheKey: options?.promptCacheKey, reasoningEffort: options?.reasoningEffort, reasoningSummary: options?.reasoningSummary, include: options?.include, diff --git a/packages/ai/src/providers/openrouter.ts b/packages/ai/src/providers/openrouter.ts index 3fe7bf13585..98da6cbcd80 100644 --- a/packages/ai/src/providers/openrouter.ts +++ b/packages/ai/src/providers/openrouter.ts @@ -55,7 +55,6 @@ export interface OpenRouterOptions { readonly debug?: Readonly<{ echo_upstream_body?: boolean }> readonly models?: ReadonlyArray readonly plugins?: ReadonlyArray - readonly promptCacheKey?: string readonly provider?: OpenRouterProviderRouting readonly reasoning?: Readonly<{ enabled?: boolean @@ -122,6 +121,7 @@ export const protocol = Protocol.make({ ...body, messages, ...bodyOptions(request.providerOptions?.openrouter), + ...(request.promptCacheKey ? { prompt_cache_key: request.promptCacheKey } : {}), } as OpenRouterBody }), ), @@ -161,7 +161,6 @@ const bodyOptions = (input: unknown) => { ...(isRecord(debug) ? { debug } : {}), ...(typeof user === "string" ? { user } : {}), ...(isRecord(reasoning) ? { reasoning } : {}), - ...(typeof promptCacheKey === "string" ? { prompt_cache_key: promptCacheKey } : {}), } } diff --git a/packages/ai/src/schema/messages.ts b/packages/ai/src/schema/messages.ts index 94b12206fb4..9fae163370e 100644 --- a/packages/ai/src/schema/messages.ts +++ b/packages/ai/src/schema/messages.ts @@ -272,6 +272,8 @@ export class LLMRequest extends Schema.Class("LLM.Request")({ providerOptions: Schema.optional(ProviderOptions), http: Schema.optional(HttpOptions), cache: Schema.optional(CachePolicy), + // Stable cache affinity for protocols that support provider-managed prompt caching. + promptCacheKey: Schema.optional(Schema.String), metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), }) {} @@ -289,6 +291,7 @@ export namespace LLMRequest { providerOptions: request.providerOptions, http: request.http, cache: request.cache, + promptCacheKey: request.promptCacheKey, metadata: request.metadata, }) diff --git a/packages/ai/test/provider-options/cloudflare.types.ts b/packages/ai/test/provider-options/cloudflare.types.ts index 6e2af83bae4..e98d5153caf 100644 --- a/packages/ai/test/provider-options/cloudflare.types.ts +++ b/packages/ai/test/provider-options/cloudflare.types.ts @@ -3,11 +3,11 @@ import { CloudflareWorkersAI } from "../../src/providers" const model = CloudflareWorkersAI.configure({ accountId: "account", apiKey: "test" }).model("model") -LLM.request({ model, prompt: "Hello", providerOptions: { openai: { promptCacheKey: "cache" } } }) +LLM.request({ model, prompt: "Hello", promptCacheKey: "cache" }) LLM.request({ model, prompt: "Hello", - // @ts-expect-error Cloudflare's OpenAI-compatible prompt cache key must be a string. - providerOptions: { openai: { promptCacheKey: 1 } }, + // @ts-expect-error Prompt cache keys must be strings. + promptCacheKey: 1, }) diff --git a/packages/ai/test/provider/openai-responses-cache.recorded.test.ts b/packages/ai/test/provider/openai-responses-cache.recorded.test.ts index 638c30e6671..fa764f35359 100644 --- a/packages/ai/test/provider/openai-responses-cache.recorded.test.ts +++ b/packages/ai/test/provider/openai-responses-cache.recorded.test.ts @@ -20,7 +20,7 @@ const cacheRequest = LLM.request({ system: LARGE_CACHEABLE_SYSTEM, prompt: "Say hi.", generation: { maxTokens: 16, temperature: 0 }, - providerOptions: { openai: { promptCacheKey: "recorded-cache-test" } }, + promptCacheKey: "recorded-cache-test", }) const recorded = recordedTests({ diff --git a/packages/ai/test/provider/openai-responses.test.ts b/packages/ai/test/provider/openai-responses.test.ts index 02722c5cc98..ebfa6c4f331 100644 --- a/packages/ai/test/provider/openai-responses.test.ts +++ b/packages/ai/test/provider/openai-responses.test.ts @@ -682,9 +682,9 @@ describe("OpenAI Responses route", () => { LLM.request({ model: OpenAI.configure({ baseURL: "https://api.openai.test/v1/", apiKey: "test" }).model("gpt-5.2"), prompt: "think", + promptCacheKey: "session_123", providerOptions: { openai: { - promptCacheKey: "session_123", reasoningEffort: "high", reasoningSummary: "auto", include: ["reasoning.encrypted_content"], @@ -803,17 +803,16 @@ describe("OpenAI Responses route", () => { }), ) - it.effect("request OpenAI provider options override route defaults", () => + it.effect("maps the request prompt cache key", () => Effect.gen(function* () { const prepared = yield* compileRequest( LLM.request({ model: OpenAI.configure({ baseURL: "https://api.openai.test/v1/", apiKey: "test", - providerOptions: { openai: { promptCacheKey: "model_cache" } }, }).model("gpt-4.1-mini"), prompt: "no cache", - providerOptions: { openai: { promptCacheKey: "request_cache" } }, + promptCacheKey: "request_cache", }), ) diff --git a/packages/ai/test/provider/openrouter.test.ts b/packages/ai/test/provider/openrouter.test.ts index 9f6efe5bfce..6077e5c797d 100644 --- a/packages/ai/test/provider/openrouter.test.ts +++ b/packages/ai/test/provider/openrouter.test.ts @@ -162,7 +162,6 @@ describe("OpenRouter", () => { openrouter: { usage: true, reasoning: { effort: "high" }, - promptCacheKey: "session_123", models: ["anthropic/claude-sonnet-4.6", "google/gemini-3.1-pro"], provider: { order: ["anthropic", "google"], require_parameters: true }, plugins: [{ id: "response-healing" }], @@ -174,6 +173,7 @@ describe("OpenRouter", () => { }, }).model("anthropic/claude-3.7-sonnet:thinking"), prompt: "Think briefly.", + promptCacheKey: "session_123", }), ) diff --git a/packages/core/src/aisdk-native.ts b/packages/core/src/aisdk-native.ts index de27a782537..d22211f4d3e 100644 --- a/packages/core/src/aisdk-native.ts +++ b/packages/core/src/aisdk-native.ts @@ -263,6 +263,7 @@ function mapOpenRouterOptions(settings: Readonly>) { "extraBody", "fetch", "headers", + "promptCacheKey", "timeout", ].includes(key), ), @@ -279,7 +280,6 @@ function mapXAIOptions(settings: Readonly>) { const options = { ...(typeof settings.reasoningEffort === "string" ? { reasoningEffort: settings.reasoningEffort } : {}), ...(typeof settings.store === "boolean" ? { store: settings.store } : {}), - ...(typeof settings.promptCacheKey === "string" ? { promptCacheKey: settings.promptCacheKey } : {}), } if (Object.keys(options).length === 0) return {} return { providerOptions: { xai: options } } diff --git a/packages/core/src/session/compaction.ts b/packages/core/src/session/compaction.ts index 3437fb7c6e2..69c20200bca 100644 --- a/packages/core/src/session/compaction.ts +++ b/packages/core/src/session/compaction.ts @@ -11,6 +11,7 @@ import { llmClient } from "../effect/app-node-platform" import { SessionEvent } from "./event" import type { SessionMessage } from "./message" import { SessionModelHeaders } from "./model-headers" +import { SessionPromptCacheKey } from "./prompt-cache-key" import { App } from "../app" import { SessionRunnerModel } from "./runner/model" import { SessionSchema } from "./schema" @@ -258,6 +259,7 @@ const make = (dependencies: Dependencies) => { .stream( LLM.request({ model: plan.model, + promptCacheKey: SessionPromptCacheKey.make(plan.session.id), http: { headers: SessionModelHeaders.make(plan.session, dependencies.app) }, messages: [Message.user(plan.prompt)], tools: [], diff --git a/packages/core/src/session/generate-node.ts b/packages/core/src/session/generate-node.ts index eed4d9763d6..a36582ce386 100644 --- a/packages/core/src/session/generate-node.ts +++ b/packages/core/src/session/generate-node.ts @@ -11,6 +11,7 @@ import { SessionContext } from "./context" import { SessionGenerate } from "./generate" import { SessionHistory } from "./history" import { SessionModelHeaders } from "./model-headers" +import { SessionPromptCacheKey } from "./prompt-cache-key" import { SessionRunnerModel } from "./runner/model" import PROMPT_DEFAULT from "./runner/prompt/base.txt" import { toLLMMessages } from "./runner/to-llm-message" @@ -31,9 +32,6 @@ export const layer = Layer.effect( const model = yield* models.resolve(selection.session) const history = yield* SessionHistory.preview(database.db, selection.session.id, selection.instructions) const providerMetadataKey = model.model.route.providerMetadataKey ?? model.model.provider - const promptCacheKey = /^ses_[0-9a-f]{64}$/.test(selection.session.id) - ? selection.session.id.slice(4) - : selection.session.id const tools = selection.tools const toolDefinitions = tools.definitions const toolsByName = new Map(toolDefinitions.map((tool) => [tool.name, tool])) @@ -71,7 +69,7 @@ export const layer = Layer.effect( LLM.request({ model: model.model, http: { headers: SessionModelHeaders.make(selection.session, app) }, - providerOptions: { [providerMetadataKey]: { promptCacheKey } }, + promptCacheKey: SessionPromptCacheKey.make(selection.session.id), system: contextEvent.system, messages: contextEvent.messages, tools: hookedTools, diff --git a/packages/core/src/session/model-request.ts b/packages/core/src/session/model-request.ts index 37dd134c4b0..11106f99f1b 100644 --- a/packages/core/src/session/model-request.ts +++ b/packages/core/src/session/model-request.ts @@ -15,6 +15,7 @@ import { QuestionTool } from "../tool/plugin/question" import { Tool } from "../tool" import { SessionContext } from "./context" import { SessionModelHeaders } from "./model-headers" +import { SessionPromptCacheKey } from "./prompt-cache-key" import { PromptCacheDiagnostics } from "./prompt-cache-diagnostics" import { MAX_STEPS_PROMPT } from "./runner/max-steps" import PROMPT_DEFAULT from "./runner/prompt/base.txt" @@ -181,7 +182,6 @@ export const layer = Layer.effect( // The final Step keeps definitions available to protocols with native "none", // preserving their prompt cache prefix. Calls are still rejected at execution. const tools = input.context.tools - const promptCacheKey = /^ses_[0-9a-f]{64}$/.test(session.id) ? session.id.slice(4) : session.id const system = [agent.info.system ? agent.info.system : PROMPT_DEFAULT, input.context.initial] .filter((part) => part.length > 0) .map(SystemPart.make) @@ -220,7 +220,7 @@ export const layer = Layer.effect( http: { headers: SessionModelHeaders.make(session, app), }, - providerOptions: { [providerMetadataKey]: { promptCacheKey } }, + promptCacheKey: SessionPromptCacheKey.make(session.id), system: context.system, messages: boundImages(unsupportedParts(context.messages, resolved.capabilities)), tools: Array.from(hooked, ([name, tool]) => ({ ...tool, name })), diff --git a/packages/core/src/session/prompt-cache-key.ts b/packages/core/src/session/prompt-cache-key.ts new file mode 100644 index 00000000000..6c8ec873b20 --- /dev/null +++ b/packages/core/src/session/prompt-cache-key.ts @@ -0,0 +1,6 @@ +export * as SessionPromptCacheKey from "./prompt-cache-key" + +import { SessionSchema } from "./schema" + +export const make = (sessionID: SessionSchema.ID) => + /^ses_[0-9a-f]{64}$/.test(sessionID) ? sessionID.slice(4) : sessionID diff --git a/packages/core/test/aisdk-native.test.ts b/packages/core/test/aisdk-native.test.ts index fafcc5d683c..a127ed2acd6 100644 --- a/packages/core/test/aisdk-native.test.ts +++ b/packages/core/test/aisdk-native.test.ts @@ -179,7 +179,6 @@ describe("AISDKNative", () => { models: ["anthropic/claude-sonnet-4.6"], provider: { only: ["anthropic"], require_parameters: true }, reasoning: { effort: "high" }, - promptCacheKey: "session_123", future_option: { enabled: true }, }), ).toEqual({ @@ -190,7 +189,6 @@ describe("AISDKNative", () => { models: ["anthropic/claude-sonnet-4.6"], provider: { only: ["anthropic"], require_parameters: true }, reasoning: { effort: "high" }, - promptCacheKey: "session_123", future_option: { enabled: true }, }, }, @@ -271,7 +269,6 @@ describe("AISDKNative", () => { baseURL: "https://xai.example/v1", reasoningEffort: "custom", store: true, - promptCacheKey: "cache-key", }), ).toEqual({ package: "@opencode-ai/ai/providers/xai", @@ -282,7 +279,6 @@ describe("AISDKNative", () => { xai: { reasoningEffort: "custom", store: true, - promptCacheKey: "cache-key", }, }, }, diff --git a/packages/core/test/session-compaction.test.ts b/packages/core/test/session-compaction.test.ts index b5ce5baec7e..5fb39460fb4 100644 --- a/packages/core/test/session-compaction.test.ts +++ b/packages/core/test/session-compaction.test.ts @@ -236,6 +236,7 @@ it.effect("manual compaction summarizes short context instead of no-op", () => expect(Array.from(yield* Fiber.join(delta)).map((event) => event.data.text)).toEqual(["manual summary"]) expect(requests).toHaveLength(1) + expect(requests[0]?.promptCacheKey).toBe(sessionID) expect(requests[0]?.http?.headers).toEqual({ "x-session-affinity": sessionID, "X-Session-Id": sessionID, diff --git a/packages/core/test/session-generate.test.ts b/packages/core/test/session-generate.test.ts index 3b6de779e1a..b89019056f8 100644 --- a/packages/core/test/session-generate.test.ts +++ b/packages/core/test/session-generate.test.ts @@ -296,7 +296,7 @@ it.effect("generates from fresh settled Session context without durable mutation expect(requests[0]?.system[0]?.text).toBe("Hooked system") expect(requests[0]?.system.map((part) => part.text)).toContain("Initial context") expect(requests[0]?.http?.headers).toMatchObject({ "X-Session-Id": sessionID }) - expect(requests[0]?.providerOptions).toMatchObject({ openai: { promptCacheKey: sessionID } }) + expect(requests[0]?.promptCacheKey).toBe(sessionID) const instructionUpdates = requests[0]?.messages.flatMap((message) => message.role === "system" ? message.content.flatMap((content) => (content.type === "text" ? [content.text] : [])) diff --git a/packages/core/test/session-runner.test.ts b/packages/core/test/session-runner.test.ts index 72d15f16b18..547b930d7e2 100644 --- a/packages/core/test/session-runner.test.ts +++ b/packages/core/test/session-runner.test.ts @@ -3254,7 +3254,7 @@ describe("SessionRunnerLLM", () => { yield* stream.started expect(requests).toHaveLength(2) - expect(requests.map((request) => request.providerOptions?.openai?.promptCacheKey)).toEqual([ + expect(requests.map((request) => request.promptCacheKey)).toEqual([ sessionID, otherSessionID, ]) @@ -3285,7 +3285,7 @@ describe("SessionRunnerLLM", () => { yield* session.resume(longSessionID) yield* session.resume(otherLongSessionID) - const keys = requests.map((request) => request.providerOptions?.openai?.promptCacheKey) + const keys = requests.map((request) => request.promptCacheKey) expect(keys).toEqual([longSessionID.slice(4), otherLongSessionID.slice(4)]) expect(keys.every((key) => typeof key === "string" && key.length === 64)).toBe(true) expect(keys[0]).not.toBe(keys[1])