diff --git a/llms/openai/internal/openaiclient/chat.go b/llms/openai/internal/openaiclient/chat.go index 7c80f23e..ff627920 100644 --- a/llms/openai/internal/openaiclient/chat.go +++ b/llms/openai/internal/openaiclient/chat.go @@ -30,25 +30,38 @@ type StreamOptions struct { IncludeUsage bool `json:"include_usage,omitempty"` } +// ReasoningOptions is enabling reasoning if the model supports it. +// There should have to use one of the fields: effort or max_tokens. +type ReasoningOptions struct { + Effort llms.ReasoningEffort `json:"effort,omitempty"` + MaxTokens int `json:"max_tokens,omitempty"` +} + // ChatRequest is a request to complete a chat completion.. type ChatRequest struct { - Model string `json:"model"` - Messages []*ChatMessage `json:"messages"` - Temperature float64 `json:"temperature,omitempty"` - TopP float64 `json:"top_p,omitempty"` - // Deprecated: Use MaxCompletionTokens - MaxTokens int `json:"-"` - MaxCompletionTokens int `json:"max_completion_tokens,omitempty"` - N int `json:"n,omitempty"` - StopWords []string `json:"stop,omitempty"` - Stream bool `json:"stream,omitempty"` - FrequencyPenalty float64 `json:"frequency_penalty,omitempty"` - PresencePenalty float64 `json:"presence_penalty,omitempty"` - Seed int `json:"seed,omitempty"` + Model string `json:"model"` + Messages []*ChatMessage `json:"messages"` + Temperature float64 `json:"temperature,omitempty"` + TopP float64 `json:"top_p,omitempty"` + MaxCompletionTokens int `json:"max_completion_tokens,omitempty"` + N int `json:"n,omitempty"` + StopWords []string `json:"stop,omitempty"` + Stream bool `json:"stream,omitempty"` + FrequencyPenalty float64 `json:"frequency_penalty,omitempty"` + PresencePenalty float64 `json:"presence_penalty,omitempty"` + Seed int `json:"seed,omitempty"` - // Reasoning effort is enabling reasoning if the model supports it. + // ReasoningEffort enables reasoning mode for models that support it. + // Set this field when you want to use the legacy reasoning configuration. + // Do not use ReasoningEffort together with Reasoning; only one should be set at a time. ReasoningEffort *llms.ReasoningEffort `json:"reasoning_effort,omitempty"` + // Reasoning provides advanced reasoning configuration for models that support it. + // Use either the Effort or MaxTokens field to control reasoning behavior. + // This field should be set when using the modern reasoning format. + // Do not set both Reasoning and ReasoningEffort at the same time, as they are mutually exclusive. + Reasoning *ReasoningOptions `json:"reasoning,omitempty"` + // ResponseFormat is the format of the response. ResponseFormat *ResponseFormat `json:"response_format,omitempty"` @@ -164,8 +177,13 @@ type ChatMessage struct { //nolint:musttag // Only present in tool messages. ToolCallID string `json:"tool_call_id,omitempty"` - // This field is only used with the deepseek-reasoner model and represents the reasoning contents of the assistant message before the final answer. + // This field is primarily used by reasoning-capable models. It contains + // the assistant's step-by-step reasoning or thought process, provided before the final answer. ReasoningContent string `json:"reasoning_content,omitempty"` + + // This field serves as a fallback for ReasoningContent. If ReasoningContent is empty, + // Reasoning may contain the assistant's reasoning or explanation. + Reasoning string `json:"reasoning,omitempty"` } func (m ChatMessage) MarshalJSON() ([]byte, error) { @@ -191,9 +209,13 @@ func (m ChatMessage) MarshalJSON() ([]byte, error) { // Only present in tool messages. ToolCallID string `json:"tool_call_id,omitempty"` - // This field is only used with the deepseek-reasoner model and represents the reasoning contents of the assistant message before the final answer. + // Reasoning content result fields ReasoningContent string `json:"reasoning_content,omitempty"` + Reasoning string `json:"reasoning,omitempty"` }(m) + if msg.ReasoningContent == "" && msg.Reasoning != "" { + msg.ReasoningContent = msg.Reasoning + } return json.Marshal(msg) } msg := struct { @@ -209,9 +231,13 @@ func (m ChatMessage) MarshalJSON() ([]byte, error) { // Only present in tool messages. ToolCallID string `json:"tool_call_id,omitempty"` - // This field is only used with the deepseek-reasoner model and represents the reasoning contents of the assistant message before the final answer. + // Reasoning content result fields ReasoningContent string `json:"reasoning_content,omitempty"` + Reasoning string `json:"reasoning,omitempty"` }(m) + if msg.ReasoningContent == "" && msg.Reasoning != "" { + msg.ReasoningContent = msg.Reasoning + } return json.Marshal(msg) } @@ -237,13 +263,17 @@ func (m *ChatMessage) UnmarshalJSON(data []byte) error { // Only present in tool messages. ToolCallID string `json:"tool_call_id,omitempty"` - // This field is only used with the deepseek-reasoner model and represents the reasoning contents of the assistant message before the final answer. + // Reasoning content result fields ReasoningContent string `json:"reasoning_content,omitempty"` + Reasoning string `json:"reasoning,omitempty"` }{} err := json.Unmarshal(data, &msg) if err != nil { return err } + if msg.ReasoningContent == "" && msg.Reasoning != "" { + msg.ReasoningContent = msg.Reasoning + } *m = ChatMessage(msg) return nil } @@ -407,8 +437,8 @@ func (c *Client) createChat(ctx context.Context, payload *ChatRequest) (*ChatCom payload.StreamOptions = &StreamOptions{IncludeUsage: true} } } - // Build request payload + // Build request payload payloadBytes, err := json.Marshal(payload) if err != nil { return nil, err @@ -457,6 +487,9 @@ func parseChatResponse(body io.Reader) (*ChatCompletionResponse, error) { // Try to restore reasoning content (some model providers don't return reasoning content) for _, choice := range response.Choices { + if choice.Message.ReasoningContent == "" { + choice.Message.ReasoningContent = choice.Message.Reasoning + } if choice.Message.ReasoningContent == "" { choice.Message.ReasoningContent, choice.Message.Content = reasoning.SplitContent(choice.Message.Content) } diff --git a/llms/openai/internal/openaiclient/openaiclient.go b/llms/openai/internal/openaiclient/openaiclient.go index c3653a07..9dc55234 100644 --- a/llms/openai/internal/openaiclient/openaiclient.go +++ b/llms/openai/internal/openaiclient/openaiclient.go @@ -38,6 +38,9 @@ type Client struct { apiVersion string ResponseFormat *ResponseFormat + + UseReasoningMaxTokens bool + ModernReasoningFormat bool } // Option is an option for the OpenAI client. @@ -51,19 +54,21 @@ type Doer interface { // New returns a new OpenAI client. func New(token string, model string, baseURL string, organization string, apiType APIType, apiVersion string, httpClient Doer, embeddingModel string, - responseFormat *ResponseFormat, + responseFormat *ResponseFormat, useReasoningMaxTokens bool, modernReasoningFormat bool, opts ...Option, ) (*Client, error) { c := &Client{ - token: token, - Model: model, - EmbeddingModel: embeddingModel, - baseURL: strings.TrimSuffix(baseURL, "/"), - organization: organization, - apiType: apiType, - apiVersion: apiVersion, - httpClient: httpClient, - ResponseFormat: responseFormat, + token: token, + Model: model, + EmbeddingModel: embeddingModel, + baseURL: strings.TrimSuffix(baseURL, "/"), + organization: organization, + apiType: apiType, + apiVersion: apiVersion, + httpClient: httpClient, + ResponseFormat: responseFormat, + UseReasoningMaxTokens: useReasoningMaxTokens, + ModernReasoningFormat: modernReasoningFormat, } if c.baseURL == "" { c.baseURL = defaultBaseURL diff --git a/llms/openai/llm.go b/llms/openai/llm.go index 598aecef..483e509a 100644 --- a/llms/openai/llm.go +++ b/llms/openai/llm.go @@ -49,7 +49,7 @@ func newClient(opts ...Option) (*options, *openaiclient.Client, error) { cli, err := openaiclient.New(options.token, options.model, options.baseURL, options.organization, openaiclient.APIType(options.apiType), options.apiVersion, options.httpClient, options.embeddingModel, - options.responseFormat, + options.responseFormat, options.useReasoningMaxTokens, options.modernReasoningFormat, ) return options, cli, err } diff --git a/llms/openai/multicontent_test.go b/llms/openai/multicontent_test.go index 05e612b2..189ebdf1 100644 --- a/llms/openai/multicontent_test.go +++ b/llms/openai/multicontent_test.go @@ -67,6 +67,10 @@ type testEnv struct { name string init func(t *testing.T, opts ...Option) *LLM opts []Option + + // reasoning options + ropt llms.CallOption + rout bool } func getCompletionTests() []testEnv { @@ -113,6 +117,80 @@ func getCompletionTests() []testEnv { return tests } +func getReasoningTests() []testEnv { + var openRouterModels = []string{ //nolint:gofumpt + "anthropic/claude-3.7-sonnet:thinking", + "deepseek/deepseek-r1", + "google/gemini-2.5-flash-preview:thinking", + "openai/o3-mini-high", + "openai/o3-mini", + "openai/o4-mini-high", + "openai/o4-mini", + } + reasoningOption := llms.WithReasoning(llms.ReasoningHigh, 0) + tests := []testEnv{ + { + name: "openai-o1", + init: newTestOpenAIClient, + opts: []Option{WithModel("o1")}, + ropt: reasoningOption, + rout: false, + }, + { + name: "openai-o3", + init: newTestOpenAIClient, + opts: []Option{WithModel("o3")}, + ropt: reasoningOption, + rout: false, + }, + { + name: "openai-o3-mini", + init: newTestOpenAIClient, + opts: []Option{WithModel("o3-mini")}, + ropt: reasoningOption, + rout: false, + }, + { + name: "openai-o4-mini", + init: newTestOpenAIClient, + opts: []Option{WithModel("o4-mini")}, + ropt: reasoningOption, + rout: false, + }, + { + name: "deepseek", + init: newTestDeepSeekClient, + opts: []Option{WithModel("deepseek-reasoner")}, + rout: true, + }, + } + clientReasoningOptions := []Option{ + WithUsingReasoningMaxTokens(), + WithModernReasoningFormat(), + } + for _, model := range openRouterModels { + expectResoningContent := !strings.Contains(model, "openai") // OpenAI doesn't provide reasoning content + normModelName := strings.ReplaceAll(strings.Split(model, "/")[1], ":", "-") + tests = append(tests, testEnv{ + name: "openrouter-" + normModelName + "-effort", + init: newTestOpenRouterClient, + opts: append([]Option{WithModel(model)}, clientReasoningOptions...), + ropt: reasoningOption, + rout: expectResoningContent, + }) + if strings.HasSuffix(model, ":thinking") { + tests = append(tests, testEnv{ + name: "openrouter-" + normModelName + "-tokens", + init: newTestOpenRouterClient, + opts: append([]Option{WithModel(model)}, clientReasoningOptions...), + ropt: llms.WithReasoning(llms.ReasoningNone, 2048), + rout: expectResoningContent, + }) + } + } + return tests +} + func getToolCallTests(multiToolCalls bool) []testEnv { var openRouterModels = []string{ //nolint:gofumpt "deepseek/deepseek-chat", @@ -164,7 +242,7 @@ func TestMultiContentText(t *testing.T) { llms.TextPart("I'm a pomeranian"), llms.TextPart("What kind of mammal am I?"), } - content := []llms.MessageContent{ + messages := []llms.MessageContent{ { Role: llms.ChatMessageTypeHuman, Parts: parts, @@ -179,7 +257,7 @@ func TestMultiContentText(t *testing.T) { llm := test.init(t, test.opts...) - resp, err := llm.GenerateContent(t.Context(), content) + resp, err := llm.GenerateContent(t.Context(), messages) require.NoError(t, err) assert.NotEmpty(t, resp.Choices) @@ -189,10 +267,85 @@ func TestMultiContentText(t *testing.T) { } } +func TestMultiContentTextWithReasoning(t *testing.T) { + t.Parallel() + + parts := []llms.ContentPart{ + llms.TextPart("What is the factorial of 5? Show your work. Think before you answer."), + } + messages := []llms.MessageContent{ + { + Role: llms.ChatMessageTypeHuman, + Parts: parts, + }, + } + + testFunc := func(t *testing.T, test testEnv, isStreaming bool) { + t.Helper() + + var content, reasoningContent strings.Builder + opts := []llms.CallOption{ + llms.WithMaxTokens(8192), // enough max tokens for reasoning + } + + if isStreaming { + opts = append(opts, llms.WithStreamingFunc(func(_ context.Context, chunk streaming.Chunk) error { + switch chunk.Type { + case streaming.ChunkTypeText: + content.WriteString(chunk.Content) + case streaming.ChunkTypeReasoning: + reasoningContent.WriteString(chunk.ReasoningContent) + case streaming.ChunkTypeToolCall: + // skip tool calls + } + return nil + })) + } + + if test.ropt != nil { + opts = append(opts, test.ropt) + } + + llm := test.init(t, test.opts...) + resp, err := llm.GenerateContent(t.Context(), messages, opts...) + require.NoError(t, err) + + assert.NotEmpty(t, resp.Choices) + c1 := resp.Choices[0] + assert.Regexp(t, "120", strings.ToLower(c1.Content)) + + if test.rout { + assert.NotEmpty(t, c1.ReasoningContent) + } else { + assert.Empty(t, c1.ReasoningContent) + } + + if isStreaming { + assert.Equal(t, content.String(), c1.Content) + assert.Equal(t, reasoningContent.String(), c1.ReasoningContent) + } + } + + tests := getReasoningTests() + for idx := range tests { + test := tests[idx] + t.Run("synchronous-"+test.name, func(t *testing.T) { + t.Parallel() + + testFunc(t, test, false) + }) + t.Run("streaming-"+test.name, func(t *testing.T) { + t.Parallel() + + testFunc(t, test, true) + }) + } +} + func TestMultiContentTextChatSequence(t *testing.T) { t.Parallel() - content := []llms.MessageContent{ + messages := []llms.MessageContent{ { Role: llms.ChatMessageTypeHuman, Parts: []llms.ContentPart{llms.TextPart("Name some countries")}, @@ -215,7 +368,7 @@ func TestMultiContentTextChatSequence(t *testing.T) { llm := test.init(t, test.opts...) - resp, err := llm.GenerateContent(t.Context(), content) + resp, err := llm.GenerateContent(t.Context(), messages) require.NoError(t, err) assert.NotEmpty(t, resp.Choices) @@ -234,14 +387,14 @@ func TestMultiContentImage(t *testing.T) { llms.ImageURLPart("https://github.com/vxcontrol/langchaingo/blob/main/docs/static/img/parrot-icon.png?raw=true"), //nolint:lll llms.TextPart("describe this image in detail"), } - content := []llms.MessageContent{ + messages := []llms.MessageContent{ { Role: llms.ChatMessageTypeHuman, Parts: parts, }, } - resp, err := llm.GenerateContent(t.Context(), content, llms.WithMaxTokens(300)) + resp, err := llm.GenerateContent(t.Context(), messages, llms.WithMaxTokens(300)) require.NoError(t, err) assert.NotEmpty(t, resp.Choices) @@ -256,7 +409,7 @@ func TestWithStreaming(t *testing.T) { llms.TextPart("I'm a pomeranian"), llms.TextPart("Tell me more about my taxonomy"), } - content := []llms.MessageContent{ + messages := []llms.MessageContent{ { Role: llms.ChatMessageTypeHuman, Parts: parts, @@ -275,7 +428,7 @@ func TestWithStreaming(t *testing.T) { text strings.Builder reasoning strings.Builder ) - resp, err := llm.GenerateContent(t.Context(), content, + resp, err := llm.GenerateContent(t.Context(), messages, llms.WithStreamingFunc(func(_ context.Context, chunk streaming.Chunk) error { switch chunk.Type { case streaming.ChunkTypeText: @@ -306,7 +459,7 @@ func TestFunctionCall(t *testing.T) { parts := []llms.ContentPart{ llms.TextPart("What is the weather like in Boston, MA?"), } - content := []llms.MessageContent{ + messages := []llms.MessageContent{ { Role: llms.ChatMessageTypeHuman, Parts: parts, @@ -332,12 +485,12 @@ func TestFunctionCall(t *testing.T) { for idx := range tests { test := tests[idx] - t.Run("function call: "+test.name, func(t *testing.T) { + t.Run("synchronous-"+test.name, func(t *testing.T) { t.Parallel() llm := test.init(t, test.opts...) - resp, err := llm.GenerateContent(t.Context(), content, llms.WithFunctions(functions)) + resp, err := llm.GenerateContent(t.Context(), messages, llms.WithFunctions(functions)) require.NoError(t, err) assert.NotEmpty(t, resp.Choices) @@ -356,7 +509,7 @@ func TestFunctionCall(t *testing.T) { for idx := range tests { test := tests[idx] - t.Run("function call with streaming: "+test.name, func(t *testing.T) { + t.Run("streaming-"+test.name, func(t *testing.T) { t.Parallel() llm := test.init(t, test.opts...) @@ -378,7 +531,7 @@ func TestFunctionCall(t *testing.T) { resp, err := llm.GenerateContent( t.Context(), - content, + messages, llms.WithFunctions(functions), llms.WithStreamingFunc(streamingFunc), ) @@ -408,7 +561,7 @@ func TestFunctionParallelCall(t *testing.T) { parts := []llms.ContentPart{ llms.TextPart("What are the weather and time in Boston, MA?"), } - content := []llms.MessageContent{ + messages := []llms.MessageContent{ { Role: llms.ChatMessageTypeHuman, Parts: parts, @@ -445,12 +598,12 @@ func TestFunctionParallelCall(t *testing.T) { for idx := range tests { test := tests[idx] - t.Run("parallel tool calls: "+test.name, func(t *testing.T) { + t.Run("synchronous-"+test.name, func(t *testing.T) { t.Parallel() llm := test.init(t, test.opts...) - resp, err := llm.GenerateContent(t.Context(), content, llms.WithFunctions(functions)) + resp, err := llm.GenerateContent(t.Context(), messages, llms.WithFunctions(functions)) require.NoError(t, err) assert.NotEmpty(t, resp.Choices) @@ -474,7 +627,7 @@ func TestFunctionParallelCall(t *testing.T) { for idx := range tests { test := tests[idx] - t.Run("parallel tool calls with streaming: "+test.name, func(t *testing.T) { + t.Run("streaming-"+test.name, func(t *testing.T) { t.Parallel() llm := test.init(t, test.opts...) @@ -501,7 +654,7 @@ func TestFunctionParallelCall(t *testing.T) { resp, err := llm.GenerateContent( t.Context(), - content, + messages, llms.WithFunctions(functions), llms.WithStreamingFunc(streamingFunc), ) diff --git a/llms/openai/openaillm.go b/llms/openai/openaillm.go index b26446ab..6b9e17d3 100644 --- a/llms/openai/openaillm.go +++ b/llms/openai/openaillm.go @@ -236,16 +236,44 @@ func (o *LLM) createChatRequest(chatMsgs []*ChatMessage, opts llms.CallOptions) req.ResponseFormat = o.client.ResponseFormat } - if opts.Reasoning.IsEnabled() { - reasoningEffort := opts.Reasoning.GetEffort(opts.MaxTokens) - if reasoningEffort != llms.ReasoningNone { - req.ReasoningEffort = &reasoningEffort + // set reasoning options, depends on the client and request options + o.setReasoning(req, opts) + + return req, nil +} + +// setReasoning sets reasoning options, depends on the client and request options. +func (o *LLM) setReasoning(req *openaiclient.ChatRequest, opts llms.CallOptions) { + if !opts.Reasoning.IsEnabled() { + return + } + + defer func() { + if req.Reasoning != nil || req.ReasoningEffort != nil { // must of all reasoning models can't use temperature and top_p with reasoning at the same time req.Temperature, req.TopP = 0.0, 0.0 } + }() + + reasoningEffort := opts.Reasoning.GetEffort(opts.MaxTokens) + reasoningTokens := opts.Reasoning.GetTokens(opts.MaxTokens) + if !o.client.ModernReasoningFormat { + if reasoningEffort != llms.ReasoningNone { + req.ReasoningEffort = &reasoningEffort + } + return } - return req, nil + // using modern reasoning format + if o.client.UseReasoningMaxTokens && opts.Reasoning.Tokens != 0 && reasoningTokens != 0 { + req.Reasoning = &openaiclient.ReasoningOptions{ + MaxTokens: reasoningTokens, + } + } else if reasoningEffort != llms.ReasoningNone { + req.Reasoning = &openaiclient.ReasoningOptions{ + Effort: reasoningEffort, + } + } } // addToolsToRequest adds tools to the request from functions and tool definitions. diff --git a/llms/openai/openaillm_option.go b/llms/openai/openaillm_option.go index a184e7ee..0709778f 100644 --- a/llms/openai/openaillm_option.go +++ b/llms/openai/openaillm_option.go @@ -35,6 +35,10 @@ type options struct { responseFormat *ResponseFormat + // fine tuning reasoning options for various LLM providers + useReasoningMaxTokens bool + modernReasoningFormat bool + // required when APIType is APITypeAzure or APITypeAzureAD apiVersion string embeddingModel string @@ -135,3 +139,20 @@ func WithResponseFormat(responseFormat *ResponseFormat) Option { opts.responseFormat = responseFormat } } + +// WithUsingReasoningMaxTokens allows to use reasoning max_tokens instead of effort. +// If reasoning max_tokens is set, it will be sent to the server instead of effort. +// Note: you must use this option within WithModernReasoningFormat(), otherwise it will be ignored. +func WithUsingReasoningMaxTokens() Option { + return func(opts *options) { + opts.useReasoningMaxTokens = true + } +} + +// WithModernReasoningFormat includes "reasoning" key and object value in the request payload. +// Otherways, it will be sent as a "reasoning_effort" string value. +func WithModernReasoningFormat() Option { + return func(opts *options) { + opts.modernReasoningFormat = true + } +}