From 751f4ba93f7fe738cb38da4e43ec2da455b2fca2 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Thu, 20 Aug 2026 14:03:33 -0500 Subject: [PATCH] feat(ai): support server-side response compaction --- packages/ai/src/protocols/open-responses.ts | 39 ++++++++- packages/ai/src/protocols/openai-responses.ts | 14 ++++ .../ai/src/protocols/utils/openai-options.ts | 20 ++++- packages/ai/src/providers/openai-options.ts | 5 +- .../ai/test/provider-options/openai.types.ts | 5 ++ .../ai/test/provider/openai-responses.test.ts | 83 +++++++++++++++++++ .../core/src/session/runner/to-llm-message.ts | 14 +++- .../core/test/session-runner-message.test.ts | 34 ++++++++ 8 files changed, 207 insertions(+), 7 deletions(-) diff --git a/packages/ai/src/protocols/open-responses.ts b/packages/ai/src/protocols/open-responses.ts index d30065e4acf..853376a8343 100644 --- a/packages/ai/src/protocols/open-responses.ts +++ b/packages/ai/src/protocols/open-responses.ts @@ -69,6 +69,13 @@ const OpenResponsesReasoningItem = Schema.Struct({ encrypted_content: optionalNull(Schema.String), }) +const OpenResponsesCompactionItem = Schema.Struct({ + type: Schema.tag("compaction"), + id: Schema.optionalKey(Schema.String), + encrypted_content: Schema.String, +}) +type OpenResponsesCompactionItem = Schema.Schema.Type + const OpenResponsesItemReference = Schema.Struct({ type: Schema.tag("item_reference"), id: Schema.String, @@ -100,6 +107,7 @@ export const InputItem = Schema.Union([ phase: Schema.optionalKey(MessagePhase), }), OpenResponsesReasoningItem, + OpenResponsesCompactionItem, OpenResponsesItemReference, Schema.Struct({ type: Schema.tag("function_call"), @@ -339,6 +347,7 @@ export interface ParserState { readonly messagePhase: (value: unknown) => MessagePhase | null | undefined readonly messagePhases: Readonly> readonly reasoningItems: Readonly> + readonly compactionItems: ReadonlyArray readonly store: boolean | undefined } @@ -418,6 +427,12 @@ const hostedToolItemID = (part: ToolResultPart, providerMetadataKey: string) => return itemID(part.providerMetadata, providerMetadataKey) } +const compactionItems = (message: LLMRequest["messages"][number], providerMetadataKey: string) => { + const native = message.native?.[providerMetadataKey] + if (!ProviderShared.isRecord(native) || !Array.isArray(native.compactionItems)) return [] + return native.compactionItems.filter(Schema.is(OpenResponsesCompactionItem)) +} + const lowerMedia = Effect.fn("OpenResponses.lowerMedia")(function* ( part: MediaPart, request: LLMRequest, @@ -499,6 +514,7 @@ const lowerMessages = Effect.fn("OpenResponses.lowerMessages")(function* (reques } if (message.role === "assistant") { + input.push(...compactionItems(message, providerMetadataKey)) const content: TextPart[] = [] const reasoningItems: Record = {} const reasoningReferences = new Set() @@ -1029,6 +1045,21 @@ const onOutputItemDone = Effect.fn("OpenResponses.onOutputItemDone")(function* ( ] satisfies StepResult } + if (item.type === "compaction") { + if (!item.id || typeof item.encrypted_content !== "string") + return yield* ProviderShared.eventError(state.id, "Open Responses compaction item is malformed") + return [ + { + ...state, + compactionItems: [ + ...state.compactionItems, + { type: "compaction", id: item.id, encrypted_content: item.encrypted_content }, + ], + }, + NO_EVENTS, + ] satisfies StepResult + } + return [state, NO_EVENTS] satisfies StepResult }) @@ -1049,10 +1080,11 @@ const onResponseFinish = Effect.fn("OpenResponses.onResponseFinish")(function* ( }, usage: mapUsage(event.response?.usage, state.providerMetadataKey), providerMetadata: - event.response?.id || event.response?.service_tier + event.response?.id || event.response?.service_tier || state.compactionItems.length > 0 ? providerMetadata(state, { - responseId: event.response.id, - serviceTier: event.response.service_tier, + responseId: event.response?.id, + serviceTier: event.response?.service_tier, + ...(state.compactionItems.length > 0 ? { compactionItems: state.compactionItems } : {}), }) : undefined, }) @@ -1162,6 +1194,7 @@ export const initial = (request: LLMRequest, extension: Extension = BASE): Parse messagePhase: (value) => messagePhase(value, extension), messagePhases: {}, reasoningItems: {}, + compactionItems: [], store: OpenResponsesOptions.resolve(request).store, }) diff --git a/packages/ai/src/protocols/openai-responses.ts b/packages/ai/src/protocols/openai-responses.ts index 915c5dfd851..cbfc7a211c7 100644 --- a/packages/ai/src/protocols/openai-responses.ts +++ b/packages/ai/src/protocols/openai-responses.ts @@ -13,6 +13,7 @@ import { OpenAIImage } from "./utils/openai-image.js" import { ToolSchemaProjection } from "./utils/tool-schema.js" import { OpenResponsesChannel } from "./open-responses-channel.js" import { OpenAIResponsesChannel } from "./openai-responses-channel.js" +import { OpenAIOptions } from "./utils/openai-options.js" const ADAPTER = "openai-responses" const NAME = "OpenAI Responses" @@ -56,6 +57,14 @@ const OpenAIResponsesCoreFields = { input: Schema.Array(OpenAIResponsesInputItem), tools: optionalArray(OpenAIResponsesTools), tool_choice: Schema.optional(OpenAIResponsesToolChoice), + context_management: Schema.optional( + Schema.Array( + Schema.Struct({ + type: Schema.tag("compaction"), + compact_threshold: Schema.optional(Schema.Int.check(Schema.isGreaterThan(0))), + }), + ), + ), } const OpenAIResponsesBody = Schema.Struct({ @@ -115,6 +124,7 @@ const fromRequest = Effect.fn("OpenAIResponses.fromRequest")(function* (request: extension, ) const toolSchemaCompatibility = request.model.compatibility?.toolSchema + const contextManagement = OpenAIOptions.resolve(request).contextManagement return { ...body, tools: @@ -125,6 +135,10 @@ const fromRequest = Effect.fn("OpenAIResponses.fromRequest")(function* (request: ), tool_choice: body.tool_choice ?? (request.toolChoice ? yield* lowerToolChoice(request.toolChoice, request.tools) : undefined), + context_management: contextManagement?.map((item) => ({ + type: item.type, + compact_threshold: item.compactThreshold, + })), } satisfies OpenAIResponsesBody }) diff --git a/packages/ai/src/protocols/utils/openai-options.ts b/packages/ai/src/protocols/utils/openai-options.ts index bb47053093c..5fe13ae5702 100644 --- a/packages/ai/src/protocols/utils/openai-options.ts +++ b/packages/ai/src/protocols/utils/openai-options.ts @@ -1,3 +1,5 @@ +import { Option, Schema } from "effect" +import type { LLMRequest } from "../../schema/index.js" import { OpenResponsesOptions } from "./open-responses-options.js" export const OpenAIReasoningEfforts = OpenResponsesOptions.ReasoningEfforts @@ -19,6 +21,22 @@ export const OpenAIServiceTier = OpenResponsesOptions.ServiceTierSchema export const isReasoningEffort = (effort: unknown): effort is OpenAIReasoningEffort => typeof effort === "string" -export const resolve = OpenResponsesOptions.resolve +export const ContextManagement = Schema.Array( + Schema.Struct({ + type: Schema.tag("compaction"), + compactThreshold: Schema.optional(Schema.Int.check(Schema.isGreaterThan(0))), + }), +) +export type ContextManagement = typeof ContextManagement.Type + +const Options = Schema.Struct({ + contextManagement: Schema.optional(ContextManagement), +}) +const decodeOptions = Schema.decodeUnknownOption(Options) + +export const resolve = (request: LLMRequest) => ({ + ...OpenResponsesOptions.resolve(request), + ...Option.getOrElse(decodeOptions(request.providerOptions), () => ({})), +}) export * as OpenAIOptions from "./openai-options.js" diff --git a/packages/ai/src/providers/openai-options.ts b/packages/ai/src/providers/openai-options.ts index ca85642ef3a..1689a25bbf0 100644 --- a/packages/ai/src/providers/openai-options.ts +++ b/packages/ai/src/providers/openai-options.ts @@ -1,9 +1,12 @@ import { mergeProviderOptions, type ProviderOptions } from "../schema/index.js" import type { OpenResponsesOptionsInput } from "./open-responses-options.js" +import type { ContextManagement } from "../protocols/utils/openai-options.js" export type { OpenAIResponseIncludable, OpenAIServiceTier } from "../protocols/utils/openai-options.js" -export type OpenAIOptionsInput = OpenResponsesOptionsInput +export type OpenAIOptionsInput = OpenResponsesOptionsInput & { + readonly contextManagement?: ContextManagement +} export type OpenAIProviderOptionsInput = OpenAIOptionsInput diff --git a/packages/ai/test/provider-options/openai.types.ts b/packages/ai/test/provider-options/openai.types.ts index 6419070b355..4004c2a15d8 100644 --- a/packages/ai/test/provider-options/openai.types.ts +++ b/packages/ai/test/provider-options/openai.types.ts @@ -10,6 +10,11 @@ LLM.request({ model: selected, prompt: "Hello", providerOptions: { textVerbosity LLM.request({ model: selected, prompt: "Hello", providerOptions: { textVerbosity: "verbose" } }) LLM.request({ model: chat, prompt: "Hello", providerOptions: { reasoningEffort: "max" } }) LLM.request({ model: chat, prompt: "Hello", providerOptions: { reasoningEffort: "experimental" } }) +LLM.request({ + model: selected, + prompt: "Hello", + providerOptions: { contextManagement: [{ type: "compaction", compactThreshold: 100_000 }] }, +}) LLM.request({ model: selected, diff --git a/packages/ai/test/provider/openai-responses.test.ts b/packages/ai/test/provider/openai-responses.test.ts index 4336e5ac4ec..764537aa543 100644 --- a/packages/ai/test/provider/openai-responses.test.ts +++ b/packages/ai/test/provider/openai-responses.test.ts @@ -168,6 +168,50 @@ describe("OpenAI Responses route", () => { }), ) + it.effect("enables server-side compaction", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLMRequest.update(request, { + providerOptions: { + contextManagement: [{ type: "compaction", compactThreshold: 100_000 }], + }, + }), + ) + + expect(prepared.body.context_management).toEqual([{ type: "compaction", compact_threshold: 100_000 }]) + }), + ) + + it.effect("replays durable server-side compaction items", () => + Effect.gen(function* () { + const prepared = yield* compileRequest( + LLM.request({ + model, + messages: [ + Message.make({ + role: "assistant", + content: "After compaction", + native: { + openai: { + compactionItems: [{ type: "compaction", id: "cmp_1", encrypted_content: "opaque-state" }], + }, + }, + }), + ], + }), + ) + + expect(prepared.body.input).toEqual([ + { type: "compaction", id: "cmp_1", encrypted_content: "opaque-state" }, + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "After compaction" }], + }, + ]) + }), + ) + it.effect("passes through custom OpenAI reasoning effort strings", () => Effect.gen(function* () { const prepared = yield* compileRequest( @@ -1506,6 +1550,45 @@ describe("OpenAI Responses route", () => { }), ) + it.effect("retains server-side compaction output for continuation", () => + Effect.gen(function* () { + const response = yield* LLMClient.generate(request).pipe( + Effect.provide( + fixedResponse( + sseEvents( + { + type: "response.output_item.done", + item: { + type: "compaction", + id: "cmp_1", + encrypted_content: "opaque-state", + status: "completed", + }, + }, + { type: "response.completed", response: { id: "resp_1" } }, + ), + ), + ), + ) + + expect(response.events.filter(LLMEvent.is.stepFinish)).toEqual([ + { + type: "step-finish", + index: 0, + reason: { normalized: "stop", raw: undefined }, + providerMetadata: { + openai: { + responseId: "resp_1", + serviceTier: undefined, + compactionItems: [{ type: "compaction", id: "cmp_1", encrypted_content: "opaque-state" }], + }, + }, + usage: undefined, + }, + ]) + }), + ) + it.effect("preserves standard refusal content as ordinary assistant text", () => Effect.gen(function* () { const response = yield* LLMClient.generate(request).pipe( diff --git a/packages/core/src/session/runner/to-llm-message.ts b/packages/core/src/session/runner/to-llm-message.ts index 698af2795cb..a3d35d68da6 100644 --- a/packages/core/src/session/runner/to-llm-message.ts +++ b/packages/core/src/session/runner/to-llm-message.ts @@ -146,6 +146,10 @@ const assistant = (message: SessionMessage.Assistant, model: Model.Ref, provider const sameProvider = String(message.model.providerID) === String(model.providerID) const sameModel = sameProvider && String(message.model.id) === String(model.id) const reuseProviderMetadata = sameModel && message.error === undefined + const native = + reuseProviderMetadata && Array.isArray(message.providerState?.compactionItems) + ? { [providerMetadataKey]: message.providerState } + : undefined const content = message.content.flatMap((item): ContentPart[] => { if (item.type === "text") return [ @@ -204,9 +208,15 @@ const assistant = (message: SessionMessage.Assistant, model: Model.Ref, provider ) .filter((message) => message !== undefined) .map(Message.tool) - if (meaningful.length === 0) return results + if (meaningful.length === 0 && native === undefined) return results return [ - Message.make({ id: message.id, role: "assistant", content: meaningful, metadata: message.metadata }), + Message.make({ + id: message.id, + role: "assistant", + content: meaningful, + metadata: message.metadata, + native, + }), ...results, ] } diff --git a/packages/core/test/session-runner-message.test.ts b/packages/core/test/session-runner-message.test.ts index a3a677329c4..2204897dd8a 100644 --- a/packages/core/test/session-runner-message.test.ts +++ b/packages/core/test/session-runner-message.test.ts @@ -1019,6 +1019,40 @@ Recent work ]) }) + test("carries same-model server compaction state as native message data", () => { + const messages = toLLMMessages( + [ + SessionMessage.Assistant.make({ + id: id("assistant-compaction"), + type: "assistant", + agent: build, + model, + content: [], + providerState: { + responseId: "resp_1", + compactionItems: [{ type: "compaction", id: "cmp_1", encrypted_content: "opaque-state" }], + }, + time: { created, completed: created }, + }), + ], + model, + ) + + expect(messages).toEqual([ + Message.make({ + id: id("assistant-compaction"), + role: "assistant", + content: [], + native: { + provider: { + responseId: "resp_1", + compactionItems: [{ type: "compaction", id: "cmp_1", encrypted_content: "opaque-state" }], + }, + }, + }), + ]) + }) + test("preserves assistant text provider state across same-provider model changes and failures", () => { const messages = toLLMMessages( [