feat: stream thinking tokens for claude 3.7 (#1679)

.changeset/flat-candles-count.md

@@ -0,0 +1,5 @@
+---
+"@llamaindex/anthropic": patch
+---
+
+Stream thinking tokens

examples/anthropic/thinking.ts

@@ -2,7 +2,6 @@ import { Anthropic } from "@llamaindex/anthropic";
 
 (async () => {
   const anthropic = new Anthropic({
-    apiKey: process.env.ANTHROPIC_API_KEY,
     model: "claude-3-7-sonnet",
     maxTokens: 20000,
     additionalChatOptions: {
@@ -20,6 +19,14 @@ import { Anthropic } from "@llamaindex/anthropic";
           "Are there an infinite number of prime numbers such that n mod 4 == 3?",
       },
     ],
+    stream: true,
   });
-  console.log(result.message);
+  console.log("Thinking...");
+  for await (const chunk of result) {
+    if (chunk.delta) {
+      process.stdout.write(chunk.delta);
+    } else if (chunk.options?.thinking) {
+      process.stdout.write(chunk.options.thinking);
+    }
+  }
 })();

packages/providers/anthropic/src/llm.ts

@@ -133,6 +133,7 @@ export type AnthropicAdditionalChatOptions = Pick<
 >;
 export type AnthropicToolCallLLMMessageOptions = ToolCallLLMMessageOptions & {
   cache_control?: BetaCacheControlEphemeral | null;
+  thinking?: string | undefined;
 };
 
 export class Anthropic extends ToolCallLLM<
@@ -504,20 +505,26 @@ export class Anthropic extends ToolCallLLM<
 
     let idx_counter: number = 0;
     for await (const part of stream) {
-      const content =
-        part.type === "content_block_delta"
-          ? part.delta.type === "text_delta"
-            ? part.delta.text
-            : part.delta
+      const textContent =
+        part.type === "content_block_delta" && part.delta.type === "text_delta"
+          ? part.delta.text
           : undefined;
 
-      if (typeof content !== "string") continue;
+      const thinking =
+        part.type === "content_block_delta" &&
+        part.delta.type === "thinking_delta"
+          ? part.delta.thinking
+          : undefined;
+
+      if (!textContent && !thinking) continue;
 
       idx_counter++;
       yield {
         raw: part,
-        delta: content,
-        options: {},
+        delta: textContent ?? "",
+        options: {
+          thinking: thinking,
+        },
       };
     }
     return;