run-llama/LlamaIndexTS
1
0
Fork 0
mirror of https://github.com/run-llama/LlamaIndexTS.git synced 2026-07-16 07:14:29 -04:00
Code Issues 138 Packages Projects Releases 10 Wiki Activity

Add "thinking" and "thinking_signature" to chat response, and "thinking_signature" to chat stream (#1825)

Browse Source
This commit is contained in:
r3rer3
2025-04-07 10:57:55 -04:00
committed by GitHub
parent 0d852d6fdc
commit 0dfa371fc9
4 changed files with 255 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/yellow-carpets-return.md
+9
View File
@@ -0,0 +1,9 @@
---
"@llamaindex/anthropic": patch
---
added `option.thinking` and `option.thinking_signature` to Anthropic's chat response
added `option.thinking_signature` to Anthropic's chat stream response
handle `ChatMessages` with `option.thinking` and `option.thinking_signature`
examples/anthropic/thinking.ts
+17
View File
@@ -27,6 +27,23 @@ import { Anthropic } from "@llamaindex/anthropic";
process.stdout.write(chunk.delta);
} else if (chunk.options?.thinking) {
process.stdout.write(chunk.options.thinking);
} else if (chunk.options?.thinking_signature) {
process.stdout.write(chunk.options.thinking_signature);
}
}
console.log("Again, but without streaming");
const resultNoStream = await anthropic.chat({
messages: [
{
role: "user",
content:
"Are there an infinite number of prime numbers such that n mod 4 == 3?",
},
],
});
console.log(resultNoStream.message.options?.thinking);
console.log(resultNoStream.message.options?.thinking_signature);
console.log(resultNoStream.message.content);
})();
packages/providers/anthropic/src/llm.ts
+67 -18
View File
@@ -10,6 +10,7 @@ import type {
MessageCreateParamsBase,
MessageParam,
Model,
ThinkingBlock,
Tool,
ToolUseBlock,
} from "@anthropic-ai/sdk/resources/messages";
@@ -135,6 +136,7 @@ export type AnthropicAdditionalChatOptions = Pick<
export type AnthropicToolCallLLMMessageOptions = ToolCallLLMMessageOptions & {
cache_control?: BetaCacheControlEphemeral | null;
thinking?: string | undefined;
thinking_signature?: string | undefined;
};
export class Anthropic extends ToolCallLLM<
@@ -215,7 +217,7 @@ export class Anthropic extends ToolCallLLM<
};
formatMessages(
messages: ChatMessage<ToolCallLLMMessageOptions>[],
messages: ChatMessage<AnthropicToolCallLLMMessageOptions>[],
): MessageParam[] {
const formattedMessages = messages.flatMap((message) => {
const options = message.options ?? {};
@@ -224,10 +226,22 @@ export class Anthropic extends ToolCallLLM<
return [];
}
if ("toolCall" in options) {
const text = extractText(message.content);
const content: MessageParam["content"] = [];
const content: MessageParam["content"] = [];
if (options?.thinking) {
if (options.thinking_signature == null) {
throw new Error(
"`thinking_signature` is required if `thinking` is provided",
);
}
content.push({
type: "thinking",
thinking: options.thinking,
signature: options.thinking_signature,
});
const text = extractText(message.content);
if (text && text.trim().length > 0) {
// don't add empty text blocks
content.push({
@@ -235,6 +249,24 @@ export class Anthropic extends ToolCallLLM<
text: text,
});
}
if (!("toolCall" in options)) {
return { role: "assistant", content } satisfies MessageParam;
}
}
if ("toolCall" in options) {
if (content.length === 0 || !content.some((c) => c.type === "text")) {
const text = extractText(message.content);
if (text && text.trim().length > 0) {
// don't add empty text blocks
content.push({
type: "text" as const,
text: text,
});
}
}
content.push(
...options.toolCall.map((tool) => ({
type: "tool_use" as const,
@@ -460,10 +492,28 @@ export class Anthropic extends ToolCallLLM<
const response = await anthropic.messages.create(apiParams);
const thinkingBlock = response.content.find(
(content): content is ThinkingBlock => content.type === "thinking",
);
const toolUseBlock = response.content.filter(
(content): content is ToolUseBlock => content.type === "tool_use",
);
const toolCall =
toolUseBlock.length > 0
? {
toolCall: toolUseBlock.map((block) => ({
id: block.id,
name: block.name,
input:
typeof block.input === "string"
? block.input
: JSON.stringify(block.input),
})),
}
: {};
return {
raw: response,
message: {
@@ -477,19 +527,11 @@ export class Anthropic extends ToolCallLLM<
text: content.text,
})),
role: "assistant",
options:
toolUseBlock.length > 0
? {
toolCall: toolUseBlock.map((block) => ({
id: block.id,
name: block.name,
input:
typeof block.input === "string"
? block.input
: JSON.stringify(block.input),
})),
}
: {},
options: {
...toolCall,
thinking: thinkingBlock?.thinking,
thinking_signature: thinkingBlock?.signature,
},
},
};
}
@@ -519,6 +561,12 @@ export class Anthropic extends ToolCallLLM<
? part.delta.thinking
: undefined;
const thinkingSignature =
part.type === "content_block_delta" &&
part.delta.type === "signature_delta"
? part.delta.signature
: undefined;
if (
part.type === "content_block_start" &&
part.content_block.type === "tool_use"
@@ -559,13 +607,14 @@ export class Anthropic extends ToolCallLLM<
continue;
}
if (!textContent && !thinking) continue;
if (!textContent && !thinking && !thinkingSignature) continue;
yield {
raw: part,
delta: textContent ?? "",
options: {
thinking: thinking,
thinking_signature: thinkingSignature,
},
};
}
packages/providers/anthropic/tests/index.test.ts
+162
View File
@@ -276,4 +276,166 @@ describe("Message Formatting", () => {
expect(() => anthropic.formatMessages(stringToolMessages)).toThrow();
});
});
describe("Extended Thinking Formatting", () => {
test("Anthropic formats thinking messages correctly", () => {
const thinkingMessages: ChatMessage[] = [
{
role: "user",
content:
"Are there an infinite number of prime numbers such that n mod 4 == 3?",
},
{
role: "assistant",
content: [
{
type: "text",
text: "Yes, there are infinitely many prime numbers of the form n mod 4 == 3.",
},
],
options: {
thinking: "Hmm, let me think about that...",
thinking_signature: "thinking_123",
},
},
];
const anthropic = new Anthropic();
const expectedOutput: MessageParam[] = [
{
role: "user",
content:
"Are there an infinite number of prime numbers such that n mod 4 == 3?",
},
{
role: "assistant",
content: [
{
type: "thinking",
thinking: "Hmm, let me think about that...",
signature: "thinking_123",
},
{
type: "text",
text: "Yes, there are infinitely many prime numbers of the form n mod 4 == 3.",
},
],
},
];
expect(anthropic.formatMessages(thinkingMessages)).toEqual(
expectedOutput,
);
});
test("Anthropic throws error if thinking_signature is not provided", () => {
const thinkingMessages: ChatMessage[] = [
{
role: "user",
content:
"Are there an infinite number of prime numbers such that n mod 4 == 3?",
},
{
role: "assistant",
content: [
{
type: "text",
text: "Yes, there are infinitely many prime numbers of the form n mod 4 == 3.",
},
],
options: {
thinking: "Hmm, let me think about that...",
},
},
];
const anthropic = new Anthropic();
expect(() => anthropic.formatMessages(thinkingMessages)).toThrow(
"`thinking_signature` is required if `thinking` is provided",
);
});
test("Thinking block comes before all other blocks", () => {
const thinkingMessages: ChatMessage[] = [
{
role: "user",
content: "What's the weather in London?",
},
{
role: "assistant",
content: [
{
type: "text",
text: "Let me check the weather.",
},
],
options: {
thinking: "Hmm, let me think about that...",
thinking_signature: "thinking_123",
toolCall: [
{
id: "call_123",
name: "weather",
input: JSON.stringify({ location: "London" }),
},
],
},
},
{
role: "assistant",
content: "The weather in London is sunny, +20°C",
options: {
toolResult: {
id: "call_123",
},
},
},
];
const anthropic = new Anthropic();
const expectedOutput: MessageParam[] = [
{
role: "user",
content: "What's the weather in London?",
},
{
role: "assistant",
content: [
{
type: "thinking",
thinking: "Hmm, let me think about that...",
signature: "thinking_123",
},
{
type: "text",
text: "Let me check the weather.",
},
{
type: "tool_use",
id: "call_123",
name: "weather",
input: {
location: "London",
},
},
],
},
{
role: "user",
content: [
{
type: "tool_result",
tool_use_id: "call_123",
content: "The weather in London is sunny, +20°C",
},
],
},
];
expect(anthropic.formatMessages(thinkingMessages)).toEqual(
expectedOutput,
);
});
});
});