Compare commits

...

4 Commits

Author SHA1 Message Date
starptech dcca3f76b4 Enable required tool choice for structured output tests 2026-05-21 12:37:09 +02:00
starptech f11ff4022c Handle StructuredOutput tool refusal errors 2026-05-21 11:29:50 +02:00
starptech 935163cf07 Rename test provider helper 2026-05-21 11:17:44 +02:00
starptech 93b5f5c229 Add tool-choice capability support 2026-05-21 11:05:47 +02:00
9 changed files with 169 additions and 1 deletions
+1
View File
@@ -49,6 +49,7 @@ export const Model = Schema.Struct({
reasoning: Schema.Boolean,
temperature: Schema.Boolean,
tool_call: Schema.Boolean,
tool_choice_required: Schema.optional(Schema.Boolean),
interleaved: Schema.optional(
Schema.Union([
Schema.Literal(true),
+1
View File
@@ -11,6 +11,7 @@ export const Model = Schema.Struct({
reasoning: Schema.optional(Schema.Boolean),
temperature: Schema.optional(Schema.Boolean),
tool_call: Schema.optional(Schema.Boolean),
tool_choice_required: Schema.optional(Schema.Boolean),
interleaved: Schema.optional(
Schema.Union([
Schema.Literal(true),
@@ -867,6 +867,7 @@ const ProviderCapabilities = Schema.Struct({
reasoning: Schema.Boolean,
attachment: Schema.Boolean,
toolcall: Schema.Boolean,
toolChoiceRequired: Schema.optional(Schema.Boolean),
input: ProviderModalities,
output: ProviderModalities,
interleaved: ProviderInterleaved,
@@ -1068,6 +1069,7 @@ function fromModelsDevModel(provider: ModelsDev.Provider, model: ModelsDev.Model
reasoning: model.reasoning ?? false,
attachment: model.attachment ?? false,
toolcall: model.tool_call ?? true,
toolChoiceRequired: model.tool_choice_required ?? true,
input: {
text: model.modalities?.input?.includes("text") ?? false,
audio: model.modalities?.input?.includes("audio") ?? false,
@@ -1296,6 +1298,7 @@ export const layer = Layer.effect(
reasoning: model.reasoning ?? existingModel?.capabilities.reasoning ?? false,
attachment: model.attachment ?? existingModel?.capabilities.attachment ?? false,
toolcall: model.tool_call ?? existingModel?.capabilities.toolcall ?? true,
toolChoiceRequired: model.tool_choice_required ?? existingModel?.capabilities.toolChoiceRequired ?? true,
input: {
text: model.modalities?.input?.includes("text") ?? existingModel?.capabilities.input.text ?? true,
audio: model.modalities?.input?.includes("audio") ?? existingModel?.capabilities.input.audio ?? false,
+6 -1
View File
@@ -1436,7 +1436,12 @@ export const layer = Layer.effect(
messages: [...modelMsgs, ...(isLastStep ? [{ role: "assistant" as const, content: MAX_STEPS }] : [])],
tools,
model,
toolChoice: format.type === "json_schema" ? "required" : undefined,
toolChoice:
format.type === "json_schema"
? model.capabilities.toolChoiceRequired === false
? "auto"
: "required"
: undefined,
})
if (structured !== undefined) {
@@ -757,6 +757,7 @@ test("model inherits properties from existing database model", async () => {
const model = providers[ProviderID.anthropic].models["claude-sonnet-4-20250514"]
expect(model.name).toBe("Custom Name for Sonnet")
expect(model.capabilities.toolcall).toBe(true)
expect(model.capabilities.toolChoiceRequired).toBe(true)
expect(model.capabilities.attachment).toBe(true)
expect(model.limit.context).toBeGreaterThan(0)
},
@@ -1376,6 +1377,44 @@ test("model defaults tool_call to true when not specified", async () => {
})
})
test("model can disable required tool_choice separately from tool_call", async () => {
await using tmp = await tmpdir({
init: async (dir) => {
await Bun.write(
path.join(dir, "opencode.json"),
JSON.stringify({
$schema: "https://opencode.ai/config.json",
provider: {
"tool-choice": {
name: "Tool Choice Provider",
npm: "@ai-sdk/openai-compatible",
env: [],
models: {
model: {
name: "Model",
tool_call: true,
tool_choice_required: false,
limit: { context: 4000, output: 1000 },
},
},
options: { apiKey: "test" },
},
},
}),
)
},
})
await withTestInstance({
directory: tmp.path,
fn: async (ctx) => {
const providers = await list(ctx)
const model = providers[ProviderID.make("tool-choice")].models.model
expect(model.capabilities.toolcall).toBe(true)
expect(model.capabilities.toolChoiceRequired).toBe(false)
},
})
})
test("model headers are preserved", async () => {
await using tmp = await tmpdir({
init: async (dir) => {
@@ -2009,15 +2048,32 @@ test("models.dev normalization fills required response fields", () => {
output: 128_000,
},
},
"deepseek-reasoner": {
id: "deepseek-reasoner",
name: "DeepSeek Reasoner",
family: "deepseek",
tool_choice_required: false,
cost: {
input: 0.5,
output: 2,
},
limit: {
context: 64_000,
output: 8_000,
},
},
},
} as unknown as ModelsDev.Provider
const model = Provider.fromModelsDevProvider(provider).models["gpt-5.4"]
const reasoner = Provider.fromModelsDevProvider(provider).models["deepseek-reasoner"]
expect(model.api.url).toBe("")
expect(model.capabilities.temperature).toBe(false)
expect(model.capabilities.reasoning).toBe(false)
expect(model.capabilities.attachment).toBe(false)
expect(model.capabilities.toolcall).toBe(true)
expect(model.capabilities.toolChoiceRequired).toBe(true)
expect(reasoner.capabilities.toolChoiceRequired).toBe(false)
expect(model.release_date).toBe("")
})
@@ -294,6 +294,30 @@ function providerCfg(url: string) {
}
}
function providerCfgWithoutRequiredToolChoice(url: string) {
return {
...providerCfg(url),
provider: {
test: {
...cfg.provider.test,
models: {
...cfg.provider.test.models,
"tool-choice-disabled": {
...cfg.provider.test.models["test-model"],
id: "tool-choice-disabled",
name: "Tool Choice Disabled",
tool_choice_required: false,
},
},
options: {
...cfg.provider.test.options,
baseURL: url,
},
},
},
}
}
const writeText = Effect.fn("test.writeText")(function* (file: string, text: string) {
const fs = yield* AppFileSystem.Service
yield* fs.writeWithDirs(file, text)
@@ -482,6 +506,80 @@ it.instance("loop calls LLM and returns assistant message", () =>
}),
)
it.instance("structured output uses auto tool choice and errors when the model ignores the tool", () =>
Effect.gen(function* () {
const { llm } = yield* useServerConfig(providerCfgWithoutRequiredToolChoice)
const prompt = yield* SessionPrompt.Service
const sessions = yield* Session.Service
const chat = yield* sessions.create({
title: "Pinned",
permission: [{ permission: "*", pattern: "*", action: "allow" }],
})
yield* llm.text("plain text")
const result = yield* prompt.prompt({
sessionID: chat.id,
agent: "build",
model: { providerID: ProviderID.make("test"), modelID: ModelID.make("tool-choice-disabled") },
parts: [{ type: "text", text: "say hello" }],
format: {
type: "json_schema",
schema: {
type: "object",
properties: { answer: { type: "string" } },
required: ["answer"],
},
},
})
const inputs = yield* llm.inputs
expect(inputs).toHaveLength(1)
expect(inputs[0].tool_choice).toBe("auto")
expect(inputs[0].tools).toEqual(
expect.arrayContaining([
expect.objectContaining({ function: expect.objectContaining({ name: "StructuredOutput" }) }),
]),
)
expect(result.info.role).toBe("assistant")
if (result.info.role === "assistant") {
expect(result.info.error?.name).toBe("StructuredOutputError")
expect(result.info.structured).toBeUndefined()
}
}),
)
it.instance("structured output uses required tool choice by default", () =>
Effect.gen(function* () {
const { llm } = yield* useServerConfig(providerCfg)
const prompt = yield* SessionPrompt.Service
const sessions = yield* Session.Service
const chat = yield* sessions.create({
title: "Pinned",
permission: [{ permission: "*", pattern: "*", action: "allow" }],
})
yield* llm.text("plain text")
yield* prompt.prompt({
sessionID: chat.id,
agent: "build",
model: { providerID: ProviderID.make("test"), modelID: ModelID.make("test-model") },
parts: [{ type: "text", text: "say hello" }],
format: {
type: "json_schema",
schema: {
type: "object",
properties: { answer: { type: "string" } },
required: ["answer"],
},
},
})
const inputs = yield* llm.inputs
expect(inputs).toHaveLength(1)
expect(inputs[0].tool_choice).toBe("required")
}),
)
noLLMServer.instance(
"prompt emits v2 prompted and synthetic events",
() =>
@@ -58320,6 +58320,7 @@
"attachment": false,
"reasoning": true,
"tool_call": true,
"tool_choice_required": false,
"interleaved": {
"field": "reasoning_content"
},
+1
View File
@@ -1044,6 +1044,7 @@ export type ProviderConfig = {
reasoning?: boolean
temperature?: boolean
tool_call?: boolean
tool_choice_required?: boolean
cost?: {
input: number
output: number
+2
View File
@@ -1049,6 +1049,7 @@ export type ProviderConfig = {
reasoning?: boolean
temperature?: boolean
tool_call?: boolean
tool_choice_required?: boolean
interleaved?:
| true
| {
@@ -1313,6 +1314,7 @@ export type Model = {
reasoning: boolean
attachment: boolean
toolcall: boolean
toolChoiceRequired?: boolean
input: {
text: boolean
audio: boolean