Release 0.9.12 (#1744)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: marcusschiesser <17126+marcusschiesser@users.noreply.github.com>

.changeset/nervous-olives-guess.md

@@ -1,5 +0,0 @@
----
-"@llamaindex/cloud": patch
----
-
-chore: bump sdk openapi.json

.changeset/rude-gifts-leave.md

@@ -1,5 +0,0 @@
----
-"@llamaindex/azure": patch
----
-
-Add `fromConnectionString` method to azure storage libs to track the usage vCore.

.changeset/thirty-hats-act.md

@@ -1,8 +0,0 @@
----
-"@llamaindex/cloud": patch
-"@llamaindex/community": patch
-"@llamaindex/core": patch
-"@llamaindex/readers": patch
----
-
-fix: add retry handling logic to parser reader and fix lint issues

apps/next/CHANGELOG.md

@@ -1,5 +1,30 @@
 # @llamaindex/doc
 
+## 0.2.0
+
+### Minor Changes
+
+- f1db9b3: Adding an options parameter to vercel tool to tailor responses
+
+### Patch Changes
+
+- 21bebfc: Expose more content to fix the issue with unavailable documentation links, and adjust the documentation based on the latest code.
+- 2b39cef: Added documentation for structured output in openai and ollama
+- Updated dependencies [21bebfc]
+- Updated dependencies [93bc0ff]
+- Updated dependencies [91a18e7]
+- Updated dependencies [bf56fc0]
+- Updated dependencies [f8a86e4]
+- Updated dependencies [5189b44]
+- Updated dependencies [58a9446]
+  - @llamaindex/readers@3.0.0
+  - @llamaindex/core@0.6.0
+  - @llamaindex/openai@0.2.0
+  - @llamaindex/cloud@4.0.0
+  - @llamaindex/workflow@1.0.0
+  - llamaindex@0.9.12
+  - @llamaindex/node-parser@2.0.0
+
 ## 0.1.11
 
 ### Patch Changes

apps/next/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/doc",
-  "version": "0.1.11",
+  "version": "0.2.0",
   "private": true,
   "scripts": {
     "postinstall": "fumadocs-mdx",

apps/next/scripts/validate-links.mts

@@ -162,7 +162,12 @@ async function validateLinks(): Promise<LinkValidationResult[]> {
     const invalidLinks = links.filter(({ link }) => {
       // Check if the link exists in valid routes
       // First normalize the link (remove any query string or hash)
-      const normalizedLink = link.split("#")[0].split("?")[0];
+      const baseLink = link.split("?")[0].split("#")[0];
+      // Remove the trailing slash if present.
+      // This works with links like "api/interfaces/MetadataFilter#operator" and "api/interfaces/MetadataFilter/#operator".
+      const normalizedLink = baseLink.endsWith("/")
+        ? baseLink.slice(0, -1)
+        : baseLink;
 
       // Remove llamaindex/ prefix if it exists as it's the root of the docs
       let routePath = normalizedLink;
@@ -192,8 +197,7 @@ async function main() {
 
   try {
     // Check for invalid internal links
-    const validationResults: LinkValidationResult[] = [];
-    await validateLinks();
+    const validationResults: LinkValidationResult[] = await validateLinks();
     // Check for relative links
     const relativeLinksResults = await findRelativeLinks();
 

apps/next/src/content/docs/llamaindex/integration/vercel.mdx

@@ -84,6 +84,7 @@ const queryTool = llamaindex({
   model: openai("gpt-4"),
   index,
   description: "Search through the documents",
+  options: { fields: ["sourceNodes", "messages"]}
 });
 
 // Use the tool with Vercel's AI SDK

apps/next/src/content/docs/llamaindex/modules/data_loaders/index.mdx

@@ -35,7 +35,7 @@ Currently, the following readers are mapped to specific file types:
 
 - [TextFileReader](/docs/api/classes/TextFileReader): `.txt`
 - [PDFReader](/docs/api/classes/PDFReader): `.pdf`
-- [PapaCSVReader](/docs/api/classes/PapaCSVReader): `.csv`
+- [CSVReader](/docs/api/classes/CSVReader): `.csv`
 - [MarkdownReader](/docs/api/classes/MarkdownReader): `.md`
 - [DocxReader](/docs/api/classes/DocxReader): `.docx`
 - [HTMLReader](/docs/api/classes/HTMLReader): `.htm`, `.html`

apps/next/src/content/docs/llamaindex/modules/data_stores/chat_stores/index.mdx

@@ -12,5 +12,5 @@ Check the [LlamaIndexTS Github](https://github.com/run-llama/LlamaIndexTS) for t
 
 ## API Reference
 
-- [BaseChatStore](/docs/api/interfaces/BaseChatStore)
+- [BaseChatStore](/docs/api/classes/BaseChatStore)
 

apps/next/src/content/docs/llamaindex/modules/data_stores/vector_stores/qdrant.mdx

@@ -56,10 +56,10 @@ const vectorStore = new QdrantVectorStore({
 
 ```ts
 const document = new Document({ text: essay, id_: path });
-
-const index = await VectorStoreIndex.fromDocuments([document], {
-  vectorStore,
-});
+const storageContext = await storageContextFromDefaults({ vectorStore });
+  const index = await VectorStoreIndex.fromDocuments([document], {
+    storageContext,
+  });
 ```
 
 ## Query the index
@@ -91,11 +91,11 @@ async function main() {
   });
 
   const document = new Document({ text: essay, id_: path });
-
+  const storageContext = await storageContextFromDefaults({ vectorStore });
   const index = await VectorStoreIndex.fromDocuments([document], {
-    vectorStore,
+    storageContext,
   });
-
+  
   const queryEngine = index.asQueryEngine();
 
   const response = await queryEngine.query({

apps/next/src/content/docs/llamaindex/modules/evaluation/correctness.mdx

@@ -74,4 +74,4 @@ the response is not correct with a score of 2.5
 
 ## API Reference
 
-- [CorrectnessEvaluator](/docs/api/classes/CorrectnessEvaluator)
+- [CorrectnessEvaluator](/docs/api/classes/CorrectnessEvaluator)

apps/next/src/content/docs/llamaindex/modules/llms/ollama.mdx

@@ -55,6 +55,35 @@ const results = await queryEngine.query({
 });
 ```
 
+## Using JSON Response Format
+
+You can configure Ollama to return responses in JSON format:
+
+```ts
+import { Ollama } from "@llamaindex/llms/ollama";
+import { z } from "zod";
+
+// Simple JSON format
+const llm = new Ollama({ 
+  model: "llama2", 
+  temperature: 0,
+  responseFormat: { type: "json_object" }
+});
+
+// Using Zod schema for validation
+const responseSchema = z.object({
+  summary: z.string(),
+  topics: z.array(z.string()),
+  sentiment: z.enum(["positive", "negative", "neutral"])
+});
+
+const llm = new Ollama({ 
+  model: "llama2", 
+  temperature: 0,
+  responseFormat: responseSchema  
+});
+```
+
 ## Full Example
 
 ```ts

apps/next/src/content/docs/llamaindex/modules/llms/openai.mdx

@@ -46,6 +46,33 @@ or
 Settings.llm = new OpenAI({ model: "gpt-3.5-turbo", temperature: 0, apiKey: <YOUR_API_KEY>, baseURL: "https://api.scaleway.ai/v1" });
 ```
 
+## Using JSON Response Format
+
+You can configure OpenAI to return responses in JSON format:
+
+```ts
+Settings.llm = new OpenAI({ 
+  model: "gpt-4o", 
+  temperature: 0,
+  responseFormat: { type: "json_object" }  
+});
+
+// You can also use a Zod schema to validate the response structure
+import { z } from "zod";
+
+const responseSchema = z.object({
+  summary: z.string(),  
+  topics: z.array(z.string()),
+  sentiment: z.enum(["positive", "negative", "neutral"])
+});
+
+Settings.llm = new OpenAI({ 
+  model: "gpt-4o", 
+  temperature: 0,
+  responseFormat: responseSchema  
+});
+```
+
 ## Load and index documents
 
 For this example, we will use a single document. In a real-world scenario, you would have multiple documents to index.

apps/next/src/content/docs/llamaindex/modules/prompt/index.mdx

@@ -28,14 +28,21 @@ Answer:`;
 
 ### 1. Customizing the default prompt on initialization
 
-The first method is to create a new instance of `ResponseSynthesizer` (or the module you would like to update the prompt) and pass the custom prompt to the `responseBuilder` parameter. Then, pass the instance to the `asQueryEngine` method of the index.
+The first method is to create a new instance of a Response Synthesizer (or the module you would like to update the prompt) by using the getResponseSynthesizer function. Instead of passing the custom prompt to the deprecated responseBuilder parameter, call getResponseSynthesizer with the mode as the first argument and supply the new prompt via the options parameter.
 
 ```ts
-// Create an instance of response synthesizer
+// Create an instance of Response Synthesizer
+
+// Deprecated usage:
 const responseSynthesizer = new ResponseSynthesizer({
   responseBuilder: new CompactAndRefine(undefined, newTextQaPrompt),
 });
 
+// Current usage:
+const responseSynthesizer = getResponseSynthesizer('compact', {
+  textQATemplate: newTextQaPrompt
+})
+
 // Create index
 const index = await VectorStoreIndex.fromDocuments([document]);
 
@@ -75,5 +82,5 @@ const response = await queryEngine.query({
 
 ## API Reference
 
-- [ResponseSynthesizer](/docs/api/classes/ResponseSynthesizer)
+- [Response Synthesizer](/docs/llamaindex/modules/response_synthesizer)
 - [CompactAndRefine](/docs/api/classes/CompactAndRefine)

apps/next/src/content/docs/llamaindex/modules/response_synthesizer.mdx

@@ -1,5 +1,5 @@
 ---
-title: ResponseSynthesizer
+title: Response Synthesizer
 ---
 
 The ResponseSynthesizer is responsible for sending the query, nodes, and prompt templates to the LLM to generate a response. There are a few key modes for generating a response:
@@ -12,15 +12,17 @@ The ResponseSynthesizer is responsible for sending the query, nodes, and prompt
   multiple compact prompts. The same as `refine`, but should result in less LLM calls.
 - `TreeSummarize`: Given a set of text chunks and the query, recursively construct a tree
   and return the root node as the response. Good for summarization purposes.
-- `SimpleResponseBuilder`: Given a set of text chunks and the query, apply the query to each text
-  chunk while accumulating the responses into an array. Returns a concatenated string of all
-  responses. Good for when you need to run the same query separately against each text
-  chunk.
+- `MultiModal`: Combines textual inputs with additional modality-specific metadata to generate an integrated response. 
+  It leverages a text QA template to build a prompt that incorporates various input types and produces either streaming or complete responses.
+  This approach is ideal for use cases where enriching the answer with multi-modal context (such as images, audio, or other data) 
+  can enhance the output quality.
 
 ```typescript
-import { NodeWithScore, TextNode, ResponseSynthesizer } from "llamaindex";
+import { NodeWithScore, TextNode, getResponseSynthesizer, responseModeSchema } from "llamaindex";
 
-const responseSynthesizer = new ResponseSynthesizer();
+// you can also use responseModeSchema.Enum.refine, responseModeSchema.Enum.tree_summarize, responseModeSchema.Enum.multi_modal
+// or you can use the CompactAndRefine, Refine, TreeSummarize, or MultiModal classes directly
+const responseSynthesizer = getResponseSynthesizer(responseModeSchema.Enum.compact);
 
 const nodesWithScore: NodeWithScore[] = [
   {
@@ -55,8 +57,9 @@ for await (const chunk of stream) {
 
 ## API Reference
 
-- [ResponseSynthesizer](/docs/api/classes/ResponseSynthesizer)
+- [getResponseSynthesizer](/docs/api/functions/getResponseSynthesizer)
+- [responseModeSchema](/docs/api/variables/responseModeSchema)
 - [Refine](/docs/api/classes/Refine)
 - [CompactAndRefine](/docs/api/classes/CompactAndRefine)
 - [TreeSummarize](/docs/api/classes/TreeSummarize)
-- [SimpleResponseBuilder](/docs/api/classes/SimpleResponseBuilder)
+- [MultiModal](/docs/api/classes/MultiModal)

apps/next/typedoc.json

@@ -1,8 +1,13 @@
 {
   "plugin": ["typedoc-plugin-markdown", "typedoc-plugin-merge-modules"],
-  "entryPoints": ["../../packages/**/src/index.ts"],
+  "entryPoints": [
+    "../../packages/{,**/}index.ts",
+    "../../packages/readers/src/*.ts",
+    "../../packages/cloud/src/{reader,utils}.ts"
+  ],
   "exclude": [
     "../../packages/autotool/**/src/index.ts",
+    "../../packages/cloud/src/client/index.ts",
     "**/node_modules/**",
     "**/dist/**",
     "**/test/**",

e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/cloudflare-worker-agent-test
 
+## 0.0.146
+
+### Patch Changes
+
+- llamaindex@0.9.12
+
 ## 0.0.145
 
 ### Patch Changes

e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.145",
+  "version": "0.0.146",
   "type": "module",
   "private": true,
   "scripts": {

e2e/examples/llama-parse-browser/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @llamaindex/llama-parse-browser-test
 
+## 0.0.55
+
+### Patch Changes
+
+- Updated dependencies [bf56fc0]
+- Updated dependencies [5189b44]
+  - @llamaindex/cloud@4.0.0
+
 ## 0.0.54
 
 ### Patch Changes

e2e/examples/llama-parse-browser/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/llama-parse-browser-test",
   "private": true,
-  "version": "0.0.54",
+  "version": "0.0.55",
   "type": "module",
   "scripts": {
     "dev": "vite",

e2e/examples/nextjs-agent/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/next-agent-test
 
+## 0.1.146
+
+### Patch Changes
+
+- llamaindex@0.9.12
+
 ## 0.1.145
 
 ### Patch Changes

e2e/examples/nextjs-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-agent-test",
-  "version": "0.1.145",
+  "version": "0.1.146",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/nextjs-edge-runtime/CHANGELOG.md

@@ -1,5 +1,11 @@
 # test-edge-runtime
 
+## 0.1.145
+
+### Patch Changes
+
+- llamaindex@0.9.12
+
 ## 0.1.144
 
 ### Patch Changes

e2e/examples/nextjs-edge-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/nextjs-edge-runtime-test",
-  "version": "0.1.144",
+  "version": "0.1.145",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/nextjs-node-runtime/CHANGELOG.md

@@ -1,5 +1,16 @@
 # @llamaindex/next-node-runtime
 
+## 0.1.12
+
+### Patch Changes
+
+- Updated dependencies [21bebfc]
+- Updated dependencies [91a18e7]
+- Updated dependencies [5189b44]
+  - @llamaindex/readers@3.0.0
+  - @llamaindex/huggingface@0.1.0
+  - llamaindex@0.9.12
+
 ## 0.1.11
 
 ### Patch Changes

e2e/examples/nextjs-node-runtime/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/next-node-runtime-test",
-  "version": "0.1.11",
+  "version": "0.1.12",
   "private": true,
   "scripts": {
     "dev": "next dev",

e2e/examples/vite-import-llamaindex/CHANGELOG.md

@@ -1,5 +1,11 @@
 # vite-import-llamaindex
 
+## 0.0.12
+
+### Patch Changes
+
+- llamaindex@0.9.12
+
 ## 0.0.11
 
 ### Patch Changes

e2e/examples/vite-import-llamaindex/package.json

@@ -1,7 +1,7 @@
 {
   "name": "vite-import-llamaindex",
   "private": true,
-  "version": "0.0.11",
+  "version": "0.0.12",
   "type": "module",
   "scripts": {
     "build": "vite build",

e2e/examples/waku-query-engine/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/waku-query-engine-test
 
+## 0.0.146
+
+### Patch Changes
+
+- llamaindex@0.9.12
+
 ## 0.0.145
 
 ### Patch Changes

e2e/examples/waku-query-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/waku-query-engine-test",
-  "version": "0.0.145",
+  "version": "0.0.146",
   "type": "module",
   "private": true,
   "scripts": {

e2e/fixtures/llm/openai.ts

@@ -42,6 +42,7 @@ export class OpenAI implements LLM {
       contextWindow: 2048,
       tokenizer: undefined,
       isFunctionCallingModel: true,
+      structuredOutput: false,
     };
   }
 

examples/CHANGELOG.md

@@ -1,5 +1,77 @@
 # examples
 
+## 0.3.0
+
+### Minor Changes
+
+- 91a18e7: Added support for structured output in the chat api of openai and ollama
+  Added structured output parameter in the provider
+- d1c1f99: Added support for function calling in mistral provider
+  Update model list for mistral provider
+  Added example for the tool call in mistral
+
+### Patch Changes
+
+- 2509353: Added support for elastic search vector store
+- Updated dependencies [21bebfc]
+- Updated dependencies [77e24ce]
+- Updated dependencies [93bc0ff]
+- Updated dependencies [2509353]
+- Updated dependencies [da06e45]
+- Updated dependencies [2a0a899]
+- Updated dependencies [050cd53]
+- Updated dependencies [91a18e7]
+- Updated dependencies [bf56fc0]
+- Updated dependencies [f1db9b3]
+- Updated dependencies [da8068e]
+- Updated dependencies [c7ff323]
+- Updated dependencies [f8a86e4]
+- Updated dependencies [d1c1f99]
+- Updated dependencies [5189b44]
+- Updated dependencies [3fd4cc3]
+- Updated dependencies [04f8c96]
+- Updated dependencies [58a9446]
+  - @llamaindex/readers@3.0.0
+  - @llamaindex/core@0.6.0
+  - @llamaindex/tools@0.0.2
+  - @llamaindex/elastic-search@0.1.0
+  - @llamaindex/google@0.2.0
+  - @llamaindex/pinecone@0.1.0
+  - @llamaindex/huggingface@0.1.0
+  - @llamaindex/anthropic@0.3.0
+  - @llamaindex/mistral@0.1.0
+  - @llamaindex/ollama@0.1.0
+  - @llamaindex/openai@0.2.0
+  - @llamaindex/cloud@4.0.0
+  - @llamaindex/vercel@0.1.0
+  - @llamaindex/azure@0.1.9
+  - @llamaindex/workflow@1.0.0
+  - @llamaindex/mongodb@0.0.14
+  - llamaindex@0.9.12
+  - @llamaindex/node-parser@2.0.0
+  - @llamaindex/clip@0.0.46
+  - @llamaindex/cohere@0.0.14
+  - @llamaindex/deepinfra@0.0.46
+  - @llamaindex/jinaai@0.0.6
+  - @llamaindex/mixedbread@0.0.14
+  - @llamaindex/perplexity@0.0.3
+  - @llamaindex/portkey-ai@0.0.42
+  - @llamaindex/replicate@0.0.42
+  - @llamaindex/astra@0.0.14
+  - @llamaindex/chroma@0.0.14
+  - @llamaindex/firestore@1.0.7
+  - @llamaindex/milvus@0.1.9
+  - @llamaindex/postgres@0.0.42
+  - @llamaindex/qdrant@0.1.9
+  - @llamaindex/upstash@0.0.14
+  - @llamaindex/weaviate@0.0.14
+  - @llamaindex/voyage-ai@1.0.6
+  - @llamaindex/deepseek@0.0.6
+  - @llamaindex/fireworks@0.0.6
+  - @llamaindex/groq@0.0.61
+  - @llamaindex/together@0.0.6
+  - @llamaindex/vllm@0.0.32
+
 ## 0.2.10
 
 ### Patch Changes

examples/agent/wiki.ts

@@ -1,13 +1,12 @@
 import { OpenAI } from "@llamaindex/openai";
+import { wiki } from "@llamaindex/tools";
 import { AgentStream, agent } from "llamaindex";
-import { WikipediaTool } from "../wiki";
 
 async function main() {
   const llm = new OpenAI({ model: "gpt-4-turbo" });
-  const wikiTool = new WikipediaTool();
 
   const workflow = agent({
-    tools: [wikiTool],
+    tools: [wiki()],
     llm,
     verbose: false,
   });

examples/agentworkflow/blog-writer.ts

@@ -10,7 +10,7 @@ import {
 import os from "os";
 import { z } from "zod";
 
-import { WikipediaTool } from "../wiki";
+import { wiki } from "@llamaindex/tools";
 const llm = openai({
   model: "gpt-4o-mini",
 });
@@ -46,7 +46,7 @@ async function main() {
     description:
       "Responsible for gathering relevant information from the internet",
     systemPrompt: `You are a research agent. Your role is to gather information from the internet using the provided tools and then transfer this information to the report agent for content creation.`,
-    tools: [new WikipediaTool()],
+    tools: [wiki()],
     canHandoffTo: [reportAgent],
     llm,
   });

examples/anthropic/agent.ts

@@ -1,7 +1,7 @@
 import { anthropic } from "@llamaindex/anthropic";
+import { wiki } from "@llamaindex/tools";
 import { agent, tool } from "llamaindex";
 import { z } from "zod";
-import { WikipediaTool } from "../wiki";
 
 const workflow = agent({
   tools: [
@@ -13,7 +13,7 @@ const workflow = agent({
       }),
       execute: ({ location }) => `The weather in ${location} is sunny`,
     }),
-    new WikipediaTool(),
+    wiki(),
   ],
   llm: anthropic({
     apiKey: process.env.ANTHROPIC_API_KEY,

examples/jsonExtract.ts

@@ -1,4 +1,5 @@
 import { OpenAI } from "@llamaindex/openai";
+import { z } from "zod";
 
 // Example using OpenAI's chat API to extract JSON from a sales call transcript
 // using json_mode see https://platform.openai.com/docs/guides/text-generation/json-mode for more details
@@ -6,22 +7,47 @@ import { OpenAI } from "@llamaindex/openai";
 const transcript =
   "[Phone rings]\n\nJohn: Hello, this is John.\n\nSarah: Hi John, this is Sarah from XYZ Company. I'm calling to discuss our new product, the XYZ Widget, and see if it might be a good fit for your business.\n\nJohn: Hi Sarah, thanks for reaching out. I'm definitely interested in learning more about the XYZ Widget. Can you give me a quick overview of what it does?\n\nSarah: Of course! The XYZ Widget is a cutting-edge tool that helps businesses streamline their workflow and improve productivity. It's designed to automate repetitive tasks and provide real-time data analytics to help you make informed decisions.\n\nJohn: That sounds really interesting. I can see how that could benefit our team. Do you have any case studies or success stories from other companies who have used the XYZ Widget?\n\nSarah: Absolutely, we have several case studies that I can share with you. I'll send those over along with some additional information about the product. I'd also love to schedule a demo for you and your team to see the XYZ Widget in action.\n\nJohn: That would be great. I'll make sure to review the case studies and then we can set up a time for the demo. In the meantime, are there any specific action items or next steps we should take?\n\nSarah: Yes, I'll send over the information and then follow up with you to schedule the demo. In the meantime, feel free to reach out if you have any questions or need further information.\n\nJohn: Sounds good, I appreciate your help Sarah. I'm looking forward to learning more about the XYZ Widget and seeing how it can benefit our business.\n\nSarah: Thank you, John. I'll be in touch soon. Have a great day!\n\nJohn: You too, bye.";
 
+const exampleSchema = z.object({
+  summary: z.string(),
+  products: z.array(z.string()),
+  rep_name: z.string(),
+  prospect_name: z.string(),
+  action_items: z.array(z.string()),
+});
+
+const example = {
+  summary:
+    "High-level summary of the call transcript. Should not exceed 3 sentences.",
+  products: ["product 1", "product 2"],
+  rep_name: "Name of the sales rep",
+  prospect_name: "Name of the prospect",
+  action_items: ["action item 1", "action item 2"],
+};
+
 async function main() {
   const llm = new OpenAI({
-    model: "gpt-4-1106-preview",
-    additionalChatOptions: { response_format: { type: "json_object" } },
+    model: "gpt-4o",
   });
 
-  const example = {
-    summary:
-      "High-level summary of the call transcript. Should not exceed 3 sentences.",
-    products: ["product 1", "product 2"],
-    rep_name: "Name of the sales rep",
-    prospect_name: "Name of the prospect",
-    action_items: ["action item 1", "action item 2"],
-  };
-
+  //response format as zod schema
   const response = await llm.chat({
+    messages: [
+      {
+        role: "system",
+        content: `You are an expert assistant for summarizing and extracting insights from sales call transcripts.`,
+      },
+      {
+        role: "user",
+        content: `Here is the transcript: \n------\n${transcript}\n------`,
+      },
+    ],
+    responseFormat: exampleSchema,
+  });
+
+  console.log(response.message.content);
+
+  //response format as json_object
+  const response2 = await llm.chat({
     messages: [
       {
         role: "system",
@@ -34,9 +60,10 @@ async function main() {
         content: `Here is the transcript: \n------\n${transcript}\n------`,
       },
     ],
+    responseFormat: { type: "json_object" },
   });
 
-  console.log(response.message.content);
+  console.log(response2.message.content);
 }
 
 main().catch(console.error);

examples/mistral/agent.ts

@@ -0,0 +1,31 @@
+import { mistral } from "@llamaindex/mistral";
+import { wiki } from "@llamaindex/tools";
+import { agent, tool } from "llamaindex";
+import { z } from "zod";
+
+const workflow = agent({
+  tools: [
+    tool({
+      name: "weather",
+      description: "Get the weather",
+      parameters: z.object({
+        location: z.string().describe("The location to get the weather for"),
+      }),
+      execute: ({ location }) => `The weather in ${location} is sunny`,
+    }),
+    wiki(),
+  ],
+  llm: mistral({
+    apiKey: process.env.MISTRAL_API_KEY,
+    model: "mistral-small-latest",
+  }),
+});
+
+async function main() {
+  const result = await workflow.run(
+    "What is the weather in New York? What's the history of New York from Wikipedia in 3 sentences?",
+  );
+  console.log(result.data);
+}
+
+void main();

examples/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/examples",
-  "version": "0.2.10",
+  "version": "0.3.0",
   "private": true,
   "scripts": {
     "lint": "eslint .",
@@ -11,44 +11,46 @@
     "@azure/cosmos": "^4.1.1",
     "@azure/identity": "^4.4.1",
     "@azure/search-documents": "^12.1.0",
-    "@llamaindex/anthropic": "^0.2.6",
-    "@llamaindex/astra": "^0.0.13",
-    "@llamaindex/azure": "^0.1.8",
-    "@llamaindex/chroma": "^0.0.13",
-    "@llamaindex/clip": "^0.0.45",
-    "@llamaindex/cloud": "^3.0.9",
-    "@llamaindex/cohere": "^0.0.13",
-    "@llamaindex/core": "^0.5.8",
-    "@llamaindex/deepinfra": "^0.0.45",
+    "@llamaindex/anthropic": "^0.3.0",
+    "@llamaindex/astra": "^0.0.14",
+    "@llamaindex/azure": "^0.1.9",
+    "@llamaindex/chroma": "^0.0.14",
+    "@llamaindex/clip": "^0.0.46",
+    "@llamaindex/cloud": "^4.0.0",
+    "@llamaindex/cohere": "^0.0.14",
+    "@llamaindex/core": "^0.6.0",
+    "@llamaindex/deepinfra": "^0.0.46",
     "@llamaindex/env": "^0.1.29",
-    "@llamaindex/firestore": "^1.0.6",
-    "@llamaindex/google": "^0.1.2",
-    "@llamaindex/groq": "^0.0.60",
-    "@llamaindex/huggingface": "^0.0.45",
-    "@llamaindex/milvus": "^0.1.8",
-    "@llamaindex/mistral": "^0.0.14",
-    "@llamaindex/mixedbread": "^0.0.13",
-    "@llamaindex/mongodb": "^0.0.13",
-    "@llamaindex/node-parser": "^1.0.8",
-    "@llamaindex/ollama": "^0.0.48",
-    "@llamaindex/openai": "^0.1.61",
-    "@llamaindex/pinecone": "^0.0.13",
-    "@llamaindex/portkey-ai": "^0.0.41",
-    "@llamaindex/postgres": "^0.0.41",
-    "@llamaindex/qdrant": "^0.1.8",
-    "@llamaindex/readers": "^2.0.8",
-    "@llamaindex/replicate": "^0.0.41",
-    "@llamaindex/upstash": "^0.0.13",
-    "@llamaindex/vercel": "^0.0.19",
-    "@llamaindex/vllm": "^0.0.31",
-    "@llamaindex/voyage-ai": "^1.0.5",
-    "@llamaindex/weaviate": "^0.0.13",
-    "@llamaindex/workflow": "^0.0.16",
-    "@llamaindex/deepseek": "^0.0.5",
-    "@llamaindex/fireworks": "^0.0.5",
-    "@llamaindex/together": "^0.0.5",
-    "@llamaindex/jinaai": "^0.0.5",
-    "@llamaindex/perplexity": "^0.0.2",
+    "@llamaindex/firestore": "^1.0.7",
+    "@llamaindex/google": "^0.2.0",
+    "@llamaindex/groq": "^0.0.61",
+    "@llamaindex/huggingface": "^0.1.0",
+    "@llamaindex/milvus": "^0.1.9",
+    "@llamaindex/mistral": "^0.1.0",
+    "@llamaindex/mixedbread": "^0.0.14",
+    "@llamaindex/mongodb": "^0.0.14",
+    "@llamaindex/elastic-search": "^0.1.0",
+    "@llamaindex/node-parser": "^2.0.0",
+    "@llamaindex/ollama": "^0.1.0",
+    "@llamaindex/openai": "^0.2.0",
+    "@llamaindex/pinecone": "^0.1.0",
+    "@llamaindex/portkey-ai": "^0.0.42",
+    "@llamaindex/postgres": "^0.0.42",
+    "@llamaindex/qdrant": "^0.1.9",
+    "@llamaindex/readers": "^3.0.0",
+    "@llamaindex/replicate": "^0.0.42",
+    "@llamaindex/upstash": "^0.0.14",
+    "@llamaindex/vercel": "^0.1.0",
+    "@llamaindex/vllm": "^0.0.32",
+    "@llamaindex/voyage-ai": "^1.0.6",
+    "@llamaindex/weaviate": "^0.0.14",
+    "@llamaindex/workflow": "^1.0.0",
+    "@llamaindex/deepseek": "^0.0.6",
+    "@llamaindex/fireworks": "^0.0.6",
+    "@llamaindex/together": "^0.0.6",
+    "@llamaindex/jinaai": "^0.0.6",
+    "@llamaindex/perplexity": "^0.0.3",
+    "@llamaindex/tools": "^0.0.2",
     "@notionhq/client": "^2.2.15",
     "@pinecone-database/pinecone": "^4.0.0",
     "@vercel/postgres": "^0.10.0",
@@ -57,7 +59,7 @@
     "commander": "^12.1.0",
     "dotenv": "^16.4.5",
     "js-tiktoken": "^1.0.14",
-    "llamaindex": "^0.9.11",
+    "llamaindex": "^0.9.12",
     "mongodb": "6.7.0",
     "postgres": "^3.4.4",
     "wikipedia": "^2.1.2",

examples/vector-store/elastic-search/index.ts

@@ -0,0 +1,73 @@
+import { ElasticSearchVectorStore } from "@llamaindex/elastic-search";
+import {
+  gemini,
+  GEMINI_EMBEDDING_MODEL,
+  GEMINI_MODEL,
+  GeminiEmbedding,
+} from "@llamaindex/google";
+import {
+  Document,
+  Settings,
+  storageContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
+async function main() {
+  Settings.embedModel = new GeminiEmbedding({
+    model: GEMINI_EMBEDDING_MODEL.TEXT_EMBEDDING_004,
+  });
+  Settings.llm = gemini({
+    model: GEMINI_MODEL.GEMINI_PRO_1_5_FLASH,
+  });
+  // Create sample documents
+  const documents = [
+    new Document({
+      text: "Elastic search is a powerful search engine",
+      metadata: {
+        source: "tech_docs",
+        author: "John Doe",
+      },
+    }),
+    new Document({
+      text: "Vector search enables semantic similarity search",
+      metadata: {
+        source: "research_paper",
+        author: "Jane Smith",
+      },
+    }),
+    new Document({
+      text: "Elasticsearch supports various distance metrics for vector search",
+      metadata: {
+        source: "tech_docs",
+        author: "Bob Wilson",
+      },
+    }),
+  ];
+
+  // Initialize ElasticSearch Vector Store
+  const vectorStore = new ElasticSearchVectorStore({
+    indexName: "llamaindex-demo",
+    esCloudId: process.env.ES_CLOUD_ID,
+    esApiKey: process.env.ES_API_KEY,
+  });
+
+  // Create storage context with the vector store
+  const storageContext = await storageContextFromDefaults({
+    vectorStore,
+  });
+
+  // Create and store embeddings in ElasticSearch
+  const index = await VectorStoreIndex.fromDocuments(documents, {
+    storageContext,
+  });
+
+  // Query the index
+  const queryEngine = index.asQueryEngine();
+
+  // Simple query
+  const response = await queryEngine.query({
+    query: "What is vector search?",
+  });
+  console.log("Basic Query Response:", response.toString());
+}
+
+main().catch(console.error);

examples/vector-store/elastic-search/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "elastic-search-vector-store",
+  "type": "module",
+  "private": true,
+  "scripts": {
+    "start": "npx tsx index.ts"
+  }
+}

examples/vercel/llm.ts

@@ -1,7 +1,7 @@
 import { openai } from "@ai-sdk/openai";
+import { wiki } from "@llamaindex/tools";
 import { VercelLLM } from "@llamaindex/vercel";
 import { LLMAgent } from "llamaindex";
-import { WikipediaTool } from "../wiki";
 
 async function main() {
   // Create an instance of VercelLLM with the OpenAI model
@@ -33,7 +33,7 @@ async function main() {
   console.log("\n=== Test 3: Using LLMAgent with WikipediaTool ===");
   const agent = new LLMAgent({
     llm: vercelLLM,
-    tools: [new WikipediaTool()],
+    tools: [wiki()],
   });
 
   const { message } = await agent.chat({

examples/wiki.ts

@@ -1,62 +0,0 @@
-/** Example of a tool that uses Wikipedia */
-
-import type { JSONSchemaType } from "ajv";
-import type { BaseTool, ToolMetadata } from "llamaindex";
-import { default as wiki } from "wikipedia";
-
-type WikipediaParameter = {
-  query: string;
-  lang?: string;
-};
-
-type WikipediaToolParams = {
-  metadata?: ToolMetadata<JSONSchemaType<WikipediaParameter>>;
-};
-
-const DEFAULT_META_DATA: ToolMetadata<JSONSchemaType<WikipediaParameter>> = {
-  name: "wikipedia_search",
-  description: "A tool that uses a query engine to search Wikipedia.",
-  parameters: {
-    type: "object",
-    properties: {
-      query: {
-        type: "string",
-        description: "The query to search for",
-      },
-      lang: {
-        type: "string",
-        description: "The language to search in",
-        nullable: true,
-      },
-    },
-    required: ["query"],
-  },
-};
-
-export class WikipediaTool implements BaseTool<WikipediaParameter> {
-  private readonly DEFAULT_LANG = "en";
-  metadata: ToolMetadata<JSONSchemaType<WikipediaParameter>>;
-
-  constructor(params?: WikipediaToolParams) {
-    this.metadata = params?.metadata || DEFAULT_META_DATA;
-  }
-
-  async loadData(
-    page: string,
-    lang: string = this.DEFAULT_LANG,
-  ): Promise<string> {
-    wiki.setLang(lang);
-    const pageResult = await wiki.page(page, { autoSuggest: false });
-    const content = await pageResult.content();
-    return content;
-  }
-
-  async call({
-    query,
-    lang = this.DEFAULT_LANG,
-  }: WikipediaParameter): Promise<string> {
-    const searchResult = await wiki.search(query);
-    if (searchResult.results.length === 0) return "No search results.";
-    return await this.loadData(searchResult.results[0].title, lang);
-  }
-}

packages/autotool/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/autotool
 
+## 6.0.12
+
+### Patch Changes
+
+- llamaindex@0.9.12
+
 ## 6.0.11
 
 ### Patch Changes

packages/autotool/examples/01_node/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/autotool-01-node-example
 
+## 0.0.93
+
+### Patch Changes
+
+- llamaindex@0.9.12
+- @llamaindex/autotool@6.0.12
+
 ## 0.0.92
 
 ### Patch Changes

packages/autotool/examples/01_node/package.json

@@ -13,5 +13,5 @@
   "scripts": {
     "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
   },
-  "version": "0.0.92"
+  "version": "0.0.93"
 }

packages/autotool/package.json

@@ -6,7 +6,7 @@
     "url": "git+https://github.com/run-llama/LlamaIndexTS.git",
     "directory": "packages/autotool"
   },
-  "version": "6.0.11",
+  "version": "6.0.12",
   "description": "auto transpile your JS function to LLM Agent compatible",
   "files": [
     "dist",

packages/cloud/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @llamaindex/cloud
 
+## 4.0.0
+
+### Patch Changes
+
+- bf56fc0: chore: bump sdk openapi.json
+- 5189b44: fix: add retry handling logic to parser reader and fix lint issues
+- Updated dependencies [21bebfc]
+- Updated dependencies [93bc0ff]
+- Updated dependencies [91a18e7]
+- Updated dependencies [5189b44]
+  - @llamaindex/core@0.6.0
+
 ## 3.0.9
 
 ### Patch Changes

packages/cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "3.0.9",
+  "version": "4.0.0",
   "type": "module",
   "license": "MIT",
   "scripts": {

packages/community/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @llamaindex/community
 
+## 0.0.91
+
+### Patch Changes
+
+- 5189b44: fix: add retry handling logic to parser reader and fix lint issues
+- 3fd4cc3: feat: use google's new gen ai library to support multimodal output
+- Updated dependencies [21bebfc]
+- Updated dependencies [93bc0ff]
+- Updated dependencies [91a18e7]
+- Updated dependencies [5189b44]
+  - @llamaindex/core@0.6.0
+
 ## 0.0.90
 
 ### Patch Changes

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.90",
+  "version": "0.0.91",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/community/src/llm/bedrock/amazon/utils.ts

@@ -10,12 +10,10 @@ import type {
   MessageContentDetail,
   ToolCallLLMMessageOptions,
 } from "@llamaindex/core/llms";
-import {
-  extractDataUrlComponents,
-  mapMessageContentToMessageContentDetails,
-} from "../utils";
+import { extractDataUrlComponents } from "../utils";
 
 import type { JSONObject } from "@llamaindex/core/global";
+import { mapMessageContentToMessageContentDetails } from "../../utils";
 import type { AmazonMessage, AmazonMessages } from "./types";
 
 const ACCEPTED_IMAGE_MIME_TYPES = [

packages/community/src/llm/bedrock/anthropic/utils.ts

@@ -6,10 +6,8 @@ import type {
   MessageContentDetail,
   ToolCallLLMMessageOptions,
 } from "@llamaindex/core/llms";
-import {
-  extractDataUrlComponents,
-  mapMessageContentToMessageContentDetails,
-} from "../utils";
+import { mapMessageContentToMessageContentDetails } from "../../utils";
+import { extractDataUrlComponents } from "../utils";
 import type {
   AnthropicContent,
   AnthropicImageContent,

packages/community/src/llm/bedrock/index.ts

@@ -22,9 +22,9 @@ import {
   type BedrockChatStreamResponse,
   Provider,
 } from "./provider";
-import { mapMessageContentToMessageContentDetails } from "./utils";
 
 import { wrapLLMEvent } from "@llamaindex/core/decorator";
+import { mapMessageContentToMessageContentDetails } from "../utils";
 import { AmazonProvider } from "./amazon/provider";
 import { AnthropicProvider } from "./anthropic/provider";
 import { MetaProvider } from "./meta/provider";
@@ -381,6 +381,7 @@ export class Bedrock extends ToolCallLLM<BedrockAdditionalChatOptions> {
       maxTokens: this.maxTokens,
       contextWindow: BEDROCK_FOUNDATION_LLMS[this.model] ?? 128000,
       tokenizer: undefined,
+      structuredOutput: false,
     };
   }
 

packages/community/src/llm/bedrock/utils.ts

@@ -1,14 +1,3 @@
-import type {
-  MessageContent,
-  MessageContentDetail,
-} from "@llamaindex/core/llms";
-
-export const mapMessageContentToMessageContentDetails = (
-  content: MessageContent,
-): MessageContentDetail[] => {
-  return Array.isArray(content) ? content : [{ type: "text", text: content }];
-};
-
 export const toUtf8 = (input: Uint8Array): string =>
   new TextDecoder("utf-8").decode(input);
 

packages/community/src/llm/utils.ts

@@ -0,0 +1,10 @@
+import type {
+  MessageContent,
+  MessageContentDetail,
+} from "@llamaindex/core/llms";
+
+export const mapMessageContentToMessageContentDetails = (
+  content: MessageContent,
+): MessageContentDetail[] => {
+  return Array.isArray(content) ? content : [{ type: "text", text: content }];
+};

packages/core/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @llamaindex/core
 
+## 0.6.0
+
+### Minor Changes
+
+- 91a18e7: Added support for structured output in the chat api of openai and ollama
+  Added structured output parameter in the provider
+
+### Patch Changes
+
+- 21bebfc: Expose more content to fix the issue with unavailable documentation links, and adjust the documentation based on the latest code.
+- 93bc0ff: fix: include additional options for context chat engine
+- 5189b44: fix: add retry handling logic to parser reader and fix lint issues
+
 ## 0.5.8
 
 ### Patch Changes

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.5.8",
+  "version": "0.6.0",
   "description": "LlamaIndex Core Module",
   "exports": {
     "./agent": {

packages/core/src/chat-engine/context-chat-engine.ts

@@ -102,6 +102,7 @@ export class ContextChatEngine extends PromptMixin implements BaseChatEngine {
       const stream = await this.chatModel.chat({
         messages: requestMessages.messages,
         stream: true,
+        additionalChatOptions: params.chatOptions as object,
       });
       return streamConverter(
         streamReducer({
@@ -117,6 +118,7 @@ export class ContextChatEngine extends PromptMixin implements BaseChatEngine {
     }
     const response = await this.chatModel.chat({
       messages: requestMessages.messages,
+      additionalChatOptions: params.chatOptions as object,
     });
     chatHistory.put(response.message);
     return EngineResponse.fromChatResponse(response, requestMessages.nodes);

packages/core/src/llms/base.ts

@@ -28,11 +28,12 @@ export abstract class BaseLLM<
   async complete(
     params: LLMCompletionParamsStreaming | LLMCompletionParamsNonStreaming,
   ): Promise<CompletionResponse | AsyncIterable<CompletionResponse>> {
-    const { prompt, stream } = params;
+    const { prompt, stream, responseFormat } = params;
     if (stream) {
       const stream = await this.chat({
         messages: [{ content: prompt, role: "user" }],
         stream: true,
+        ...(responseFormat ? { responseFormat } : {}),
       });
       return streamConverter(stream, (chunk) => {
         return {
@@ -41,9 +42,12 @@ export abstract class BaseLLM<
         };
       });
     }
+
     const chatResponse = await this.chat({
       messages: [{ content: prompt, role: "user" }],
+      ...(responseFormat ? { responseFormat } : {}),
     });
+
     return {
       text: extractText(chatResponse.message.content),
       raw: chatResponse.raw,

packages/core/src/llms/type.ts

@@ -1,5 +1,6 @@
 import type { Tokenizers } from "@llamaindex/env/tokenizers";
 import type { JSONSchemaType } from "ajv";
+import { z } from "zod";
 import type { JSONObject, JSONValue } from "../global";
 
 /**
@@ -106,6 +107,7 @@ export type LLMMetadata = {
   maxTokens?: number | undefined;
   contextWindow: number;
   tokenizer: Tokenizers | undefined;
+  structuredOutput: boolean;
 };
 
 export interface LLMChatParamsBase<
@@ -115,6 +117,7 @@ export interface LLMChatParamsBase<
   messages: ChatMessage<AdditionalMessageOptions>[];
   additionalChatOptions?: AdditionalChatOptions;
   tools?: BaseTool[];
+  responseFormat?: z.ZodType | object;
 }
 
 export interface LLMChatParamsStreaming<
@@ -133,6 +136,7 @@ export interface LLMChatParamsNonStreaming<
 
 export interface LLMCompletionParamsBase {
   prompt: MessageContent;
+  responseFormat?: z.ZodType | object;
 }
 
 export interface LLMCompletionParamsStreaming extends LLMCompletionParamsBase {

packages/core/src/response-synthesizers/factory.ts

@@ -23,7 +23,7 @@ import {
 } from "./base-synthesizer";
 import { createMessageContent } from "./utils";
 
-const responseModeSchema = z.enum([
+export const responseModeSchema = z.enum([
   "refine",
   "compact",
   "tree_summarize",
@@ -35,7 +35,7 @@ export type ResponseMode = z.infer<typeof responseModeSchema>;
 /**
  * A response builder that uses the query to ask the LLM generate a better response using multiple text chunks.
  */
-class Refine extends BaseSynthesizer {
+export class Refine extends BaseSynthesizer {
   textQATemplate: TextQAPrompt;
   refineTemplate: RefinePrompt;
 
@@ -213,7 +213,7 @@ class Refine extends BaseSynthesizer {
 /**
  * CompactAndRefine is a slight variation of Refine that first compacts the text chunks into the smallest possible number of chunks.
  */
-class CompactAndRefine extends Refine {
+export class CompactAndRefine extends Refine {
   async getResponse(
     query: MessageContent,
     nodes: NodeWithScore[],
@@ -267,7 +267,7 @@ class CompactAndRefine extends Refine {
 /**
  * TreeSummarize repacks the text chunks into the smallest possible number of chunks and then summarizes them, then recursively does so until there's one chunk left.
  */
-class TreeSummarize extends BaseSynthesizer {
+export class TreeSummarize extends BaseSynthesizer {
   summaryTemplate: TreeSummarizePrompt;
 
   constructor(
@@ -370,7 +370,7 @@ class TreeSummarize extends BaseSynthesizer {
   }
 }
 
-class MultiModal extends BaseSynthesizer {
+export class MultiModal extends BaseSynthesizer {
   metadataMode: MetadataMode;
   textQATemplate: TextQAPrompt;
 

packages/core/src/response-synthesizers/index.ts

@@ -2,7 +2,15 @@ export {
   BaseSynthesizer,
   type BaseSynthesizerOptions,
 } from "./base-synthesizer";
-export { getResponseSynthesizer, type ResponseMode } from "./factory";
+export {
+  CompactAndRefine,
+  MultiModal,
+  Refine,
+  TreeSummarize,
+  getResponseSynthesizer,
+  responseModeSchema,
+  type ResponseMode,
+} from "./factory";
 export type {
   SynthesizeEndEvent,
   SynthesizeQuery,

packages/core/src/utils/mock.ts

@@ -35,6 +35,7 @@ export class MockLLM extends ToolCallLLM {
       topP: 0.5,
       contextWindow: 1024,
       tokenizer: undefined,
+      structuredOutput: false,
     };
   }
 

packages/core/tests/node-parser/text-splitter.test.ts

@@ -126,6 +126,7 @@ describe("sentence splitter", () => {
       id_: docId,
       text: "This is a test sentence. This is another test sentence.",
     });
+
     const nodes = sentenceSplitter.getNodesFromDocuments([doc]);
     nodes.forEach((node) => {
       // test node id should match uuid regex

packages/experimental/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/experimental
 
+## 0.0.162
+
+### Patch Changes
+
+- llamaindex@0.9.12
+
 ## 0.0.161
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.161",
+  "version": "0.0.162",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/llamaindex/CHANGELOG.md

@@ -1,5 +1,22 @@
 # llamaindex
 
+## 0.9.12
+
+### Patch Changes
+
+- Updated dependencies [21bebfc]
+- Updated dependencies [93bc0ff]
+- Updated dependencies [91a18e7]
+- Updated dependencies [bf56fc0]
+- Updated dependencies [f8a86e4]
+- Updated dependencies [5189b44]
+- Updated dependencies [58a9446]
+  - @llamaindex/core@0.6.0
+  - @llamaindex/openai@0.2.0
+  - @llamaindex/cloud@4.0.0
+  - @llamaindex/workflow@1.0.0
+  - @llamaindex/node-parser@2.0.0
+
 ## 0.9.11
 
 ### Patch Changes

packages/llamaindex/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llamaindex",
-  "version": "0.9.11",
+  "version": "0.9.12",
   "license": "MIT",
   "type": "module",
   "keywords": [

packages/node-parser/CHANGELOG.md

@@ -1,5 +1,15 @@
 # @llamaindex/node-parser
 
+## 2.0.0
+
+### Patch Changes
+
+- Updated dependencies [21bebfc]
+- Updated dependencies [93bc0ff]
+- Updated dependencies [91a18e7]
+- Updated dependencies [5189b44]
+  - @llamaindex/core@0.6.0
+
 ## 1.0.8
 
 ### Patch Changes

packages/node-parser/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/node-parser",
-  "version": "1.0.8",
+  "version": "2.0.0",
   "description": "Node parser for LlamaIndex",
   "type": "module",
   "exports": {

packages/providers/anthropic/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @llamaindex/anthropic
 
+## 0.3.0
+
+### Minor Changes
+
+- 91a18e7: Added support for structured output in the chat api of openai and ollama
+  Added structured output parameter in the provider
+
+### Patch Changes
+
+- Updated dependencies [21bebfc]
+- Updated dependencies [93bc0ff]
+- Updated dependencies [91a18e7]
+- Updated dependencies [5189b44]
+  - @llamaindex/core@0.6.0
+
 ## 0.2.6
 
 ### Patch Changes

packages/providers/anthropic/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/anthropic",
   "description": "Anthropic Adapter for LlamaIndex",
-  "version": "0.2.6",
+  "version": "0.3.0",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/anthropic/src/llm.ts

@@ -191,6 +191,7 @@ export class Anthropic extends ToolCallLLM<
             ].contextWindow
           : 200000,
       tokenizer: undefined,
+      structuredOutput: false,
     };
   }
 

packages/providers/clip/CHANGELOG.md

@@ -1,5 +1,16 @@
 # @llamaindex/clip
 
+## 0.0.46
+
+### Patch Changes
+
+- Updated dependencies [21bebfc]
+- Updated dependencies [93bc0ff]
+- Updated dependencies [91a18e7]
+- Updated dependencies [5189b44]
+  - @llamaindex/core@0.6.0
+  - @llamaindex/openai@0.2.0
+
 ## 0.0.45
 
 ### Patch Changes

packages/providers/clip/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/clip",
   "description": "Clip Embedding Adapter for LlamaIndex",
-  "version": "0.0.45",
+  "version": "0.0.46",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/cohere/CHANGELOG.md

@@ -1,5 +1,15 @@
 # @llamaindex/cohere
 
+## 0.0.14
+
+### Patch Changes
+
+- Updated dependencies [21bebfc]
+- Updated dependencies [93bc0ff]
+- Updated dependencies [91a18e7]
+- Updated dependencies [5189b44]
+  - @llamaindex/core@0.6.0
+
 ## 0.0.13
 
 ### Patch Changes

packages/providers/cohere/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/cohere",
   "description": "Cohere Adapter for LlamaIndex",
-  "version": "0.0.13",
+  "version": "0.0.14",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/deepinfra/CHANGELOG.md

@@ -1,5 +1,16 @@
 # @llamaindex/deepinfra
 
+## 0.0.46
+
+### Patch Changes
+
+- Updated dependencies [21bebfc]
+- Updated dependencies [93bc0ff]
+- Updated dependencies [91a18e7]
+- Updated dependencies [5189b44]
+  - @llamaindex/core@0.6.0
+  - @llamaindex/openai@0.2.0
+
 ## 0.0.45
 
 ### Patch Changes

packages/providers/deepinfra/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/deepinfra",
   "description": "Deepinfra Adapter for LlamaIndex",
-  "version": "0.0.45",
+  "version": "0.0.46",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/deepseek/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/deepseek
 
+## 0.0.6
+
+### Patch Changes
+
+- Updated dependencies [91a18e7]
+  - @llamaindex/openai@0.2.0
+
 ## 0.0.5
 
 ### Patch Changes

packages/providers/deepseek/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/deepseek",
   "description": "DeepSeek Adapter for LlamaIndex",
-  "version": "0.0.5",
+  "version": "0.0.6",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/fireworks/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/fireworks
 
+## 0.0.6
+
+### Patch Changes
+
+- Updated dependencies [91a18e7]
+  - @llamaindex/openai@0.2.0
+
 ## 0.0.5
 
 ### Patch Changes

packages/providers/fireworks/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/fireworks",
   "description": "Fireworks Adapter for LlamaIndex",
-  "version": "0.0.5",
+  "version": "0.0.6",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/google/CHANGELOG.md

@@ -1,5 +1,23 @@
 # @llamaindex/google
 
+## 0.2.0
+
+### Minor Changes
+
+- 91a18e7: Added support for structured output in the chat api of openai and ollama
+  Added structured output parameter in the provider
+
+### Patch Changes
+
+- da06e45: fix: don't ignore parts that only have inline data for google studio
+- 2a0a899: Added saftey setting parameter for gemini
+- 3fd4cc3: feat: use google's new gen ai library to support multimodal output
+- Updated dependencies [21bebfc]
+- Updated dependencies [93bc0ff]
+- Updated dependencies [91a18e7]
+- Updated dependencies [5189b44]
+  - @llamaindex/core@0.6.0
+
 ## 0.1.2
 
 ### Patch Changes

packages/providers/google/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/google",
   "description": "Google Adapter for LlamaIndex",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",
@@ -34,6 +34,7 @@
   },
   "dependencies": {
     "@google-cloud/vertexai": "1.9.0",
+    "@google/genai": "^0.4.0",
     "@google/generative-ai": "0.21.0",
     "@llamaindex/core": "workspace:*",
     "@llamaindex/env": "workspace:*"

packages/providers/google/src/base.ts

@@ -6,6 +6,7 @@ import {
   type ModelParams as GoogleModelParams,
   type RequestOptions as GoogleRequestOptions,
   type GenerateContentStreamResult as GoogleStreamGenerateContentResult,
+  type SafetySetting,
 } from "@google/generative-ai";
 
 import { wrapLLMEvent } from "@llamaindex/core/decorator";
@@ -62,7 +63,7 @@ export const GEMINI_MODEL_INFO_MAP: Record<GEMINI_MODEL, GeminiModelInfo> = {
   [GEMINI_MODEL.GEMINI_2_0_PRO_EXPERIMENTAL]: { contextWindow: 2 * 10 ** 6 },
 };
 
-const SUPPORT_TOOL_CALL_MODELS: GEMINI_MODEL[] = [
+export const SUPPORT_TOOL_CALL_MODELS: GEMINI_MODEL[] = [
   GEMINI_MODEL.GEMINI_PRO,
   GEMINI_MODEL.GEMINI_PRO_VISION,
   GEMINI_MODEL.GEMINI_PRO_1_5_PRO_PREVIEW,
@@ -78,7 +79,7 @@ const SUPPORT_TOOL_CALL_MODELS: GEMINI_MODEL[] = [
   GEMINI_MODEL.GEMINI_2_0_PRO_EXPERIMENTAL,
 ];
 
-const DEFAULT_GEMINI_PARAMS = {
+export const DEFAULT_GEMINI_PARAMS = {
   model: GEMINI_MODEL.GEMINI_PRO,
   temperature: 0.1,
   topP: 1,
@@ -88,6 +89,7 @@ const DEFAULT_GEMINI_PARAMS = {
 export type GeminiConfig = Partial<typeof DEFAULT_GEMINI_PARAMS> & {
   session?: IGeminiSession;
   requestOptions?: GoogleRequestOptions;
+  safetySettings?: SafetySetting[];
 };
 
 /**
@@ -112,7 +114,7 @@ export class GeminiSession implements IGeminiSession {
   ): GoogleGenerativeModel {
     return this.gemini.getGenerativeModel(
       {
-        safetySettings: DEFAULT_SAFETY_SETTINGS,
+        safetySettings: metadata.safetySettings ?? DEFAULT_SAFETY_SETTINGS,
         ...metadata,
       },
       requestOpts,
@@ -218,6 +220,7 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
   maxTokens?: number | undefined;
   #requestOptions?: GoogleRequestOptions | undefined;
   session: IGeminiSession;
+  safetySettings: SafetySetting[];
 
   constructor(init?: GeminiConfig) {
     super();
@@ -227,13 +230,14 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
     this.maxTokens = init?.maxTokens ?? undefined;
     this.session = init?.session ?? GeminiSessionStore.get();
     this.#requestOptions = init?.requestOptions ?? undefined;
+    this.safetySettings = init?.safetySettings ?? DEFAULT_SAFETY_SETTINGS;
   }
 
   get supportToolCall(): boolean {
     return SUPPORT_TOOL_CALL_MODELS.includes(this.model);
   }
 
-  get metadata(): LLMMetadata {
+  get metadata(): LLMMetadata & { safetySettings: SafetySetting[] } {
     return {
       model: this.model,
       temperature: this.temperature,
@@ -241,6 +245,8 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
       maxTokens: this.maxTokens,
       contextWindow: GEMINI_MODEL_INFO_MAP[this.model].contextWindow,
       tokenizer: undefined,
+      structuredOutput: false,
+      safetySettings: this.safetySettings,
     };
   }
 
@@ -250,7 +256,7 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
     const context = getChatContext(params);
     const common = {
       history: context.history,
-      safetySettings: DEFAULT_SAFETY_SETTINGS,
+      safetySettings: this.safetySettings,
     };
 
     return params.tools?.length
@@ -264,7 +270,7 @@ export class Gemini extends ToolCallLLM<GeminiAdditionalChatOptions> {
               ),
             },
           ],
-          safetySettings: DEFAULT_SAFETY_SETTINGS,
+          safetySettings: this.safetySettings,
         }
       : common;
   }

packages/providers/google/src/index.ts

@@ -3,4 +3,10 @@ export * from "./types";
 export * from "./utils";
 export * from "./vertex";
 
+export {
+  GoogleStudio,
+  Modality,
+  getGoogleStudioInlineData,
+} from "./studio/index.js";
+
 export * from "./GeminiEmbedding";

packages/providers/google/src/studio/index.ts

@@ -0,0 +1,262 @@
+import {
+  GenerateContentResponse,
+  GoogleGenAI,
+  Modality,
+  type Blob,
+  type GenerateContentConfig,
+  type GoogleGenAIOptions,
+} from "@google/genai";
+import {
+  ToolCallLLM,
+  type ChatResponse,
+  type ChatResponseChunk,
+  type CompletionResponse,
+  type LLMChatParamsNonStreaming,
+  type LLMChatParamsStreaming,
+  type LLMCompletionParamsNonStreaming,
+  type LLMCompletionParamsStreaming,
+  type LLMMetadata,
+  type ToolCall,
+  type ToolCallLLMMessageOptions,
+} from "@llamaindex/core/llms";
+export { Modality };
+
+import { streamConverter } from "@llamaindex/core/utils";
+
+import { wrapLLMEvent } from "@llamaindex/core/decorator";
+import type { JSONObject } from "@llamaindex/core/global";
+import { DEFAULT_GEMINI_PARAMS, SUPPORT_TOOL_CALL_MODELS } from "../base";
+import type { GEMINI_MODEL } from "../types";
+import {
+  mapChatMessagesToGoogleFunctions,
+  mapChatMessagesToGoogleMessages,
+  mapMessageContentDetailToGooglePart,
+  mapMessageContentToMessageContentDetails,
+} from "./utils";
+
+export type GoogleAdditionalChatOptions = { config: GenerateContentConfig };
+
+export type GoogleChatStreamResponse = AsyncIterable<
+  ChatResponseChunk<
+    ToolCallLLMMessageOptions & {
+      data?: Blob[];
+    }
+  >
+>;
+
+export type GoogleChatParamsStreaming = LLMChatParamsStreaming<
+  GoogleAdditionalChatOptions,
+  ToolCallLLMMessageOptions
+>;
+
+export type GoogleChatParamsNonStreaming = LLMChatParamsNonStreaming<
+  GoogleAdditionalChatOptions,
+  ToolCallLLMMessageOptions
+>;
+
+export type GoogleChatNonStreamResponse =
+  ChatResponse<ToolCallLLMMessageOptions>;
+
+export const getGoogleStudioInlineData = (
+  response: GenerateContentResponse,
+): Blob[] => {
+  return response.candidates
+    ?.flatMap((candidate) => candidate.content?.parts)
+    .map((part) => part?.inlineData)
+    .filter((data) => data) as Blob[];
+};
+
+export type GoogleModelParams = {
+  model: GEMINI_MODEL;
+  temperature?: number;
+  topP?: number;
+  maxTokens?: number;
+};
+
+export type GoogleParams = GoogleGenAIOptions & GoogleModelParams;
+
+export class GoogleStudio extends ToolCallLLM<GoogleAdditionalChatOptions> {
+  client: GoogleGenAI;
+  model: GEMINI_MODEL;
+  temperature: number;
+  topP: number;
+  maxTokens?: number | undefined;
+  topK?: number;
+
+  constructor({
+    temperature,
+    topP,
+    maxTokens,
+    model,
+    ...params
+  }: GoogleParams) {
+    super();
+    this.model = model;
+    this.maxTokens = maxTokens ?? DEFAULT_GEMINI_PARAMS.maxTokens;
+    this.temperature = temperature ?? DEFAULT_GEMINI_PARAMS.temperature;
+    this.topP = topP ?? DEFAULT_GEMINI_PARAMS.topP;
+    this.client = new GoogleGenAI(params);
+  }
+
+  get supportToolCall(): boolean {
+    return SUPPORT_TOOL_CALL_MODELS.includes(this.model);
+  }
+
+  get metadata(): LLMMetadata {
+    return {
+      model: this.model,
+      temperature: this.temperature,
+      topP: this.topP,
+      maxTokens: this.maxTokens,
+      contextWindow: 128000,
+      tokenizer: undefined,
+      structuredOutput: false,
+    };
+  }
+
+  getToolCallsFromResponse(response: GenerateContentResponse): ToolCall[] {
+    if (!response.functionCalls) return [];
+    return response.functionCalls.map((call) => ({
+      id: call.id ?? "",
+      name: call.name ?? "",
+      input: call.args as JSONObject,
+    }));
+  }
+
+  protected async nonStreamChat(
+    params: GoogleChatParamsNonStreaming,
+  ): Promise<GoogleChatNonStreamResponse> {
+    if (!this.supportToolCall && params.tools?.length) {
+      console.warn(`The model "${this.model}" doesn't support ToolCall`);
+    }
+
+    const config: GenerateContentConfig =
+      params.additionalChatOptions?.config ?? {};
+    if (params.tools?.length) {
+      if (config.responseModalities?.includes(Modality.IMAGE)) {
+        console.warn("Tools are currently not supported with Modality.IMAGE");
+      } else {
+        config.tools = mapChatMessagesToGoogleFunctions(params.tools);
+      }
+    }
+    const response = await this.client.models.generateContent({
+      model: this.model,
+      contents: mapChatMessagesToGoogleMessages(params.messages),
+      config,
+    });
+
+    if (this.supportToolCall) {
+      const tools = this.getToolCallsFromResponse(response);
+      if (tools.length) {
+        return {
+          raw: response,
+          message: {
+            role: "assistant",
+            content: "",
+            options: { toolCall: tools },
+          },
+        };
+      }
+    }
+    return {
+      raw: response,
+      message: {
+        role: "assistant",
+        content: response.text ?? "",
+        options: {
+          inlineData: getGoogleStudioInlineData(response),
+        },
+      },
+    };
+  }
+
+  protected async *streamChat(
+    params: GoogleChatParamsStreaming,
+  ): GoogleChatStreamResponse {
+    if (!this.supportToolCall && params.tools?.length) {
+      console.warn(`The model "${this.model}" doesn't support ToolCall`);
+    }
+    const config: GenerateContentConfig =
+      params.additionalChatOptions?.config ?? {};
+    if (params.tools?.length) {
+      if (config.responseModalities?.includes(Modality.IMAGE)) {
+        console.warn("Tools are currently not supported with Modality.IMAGE");
+      } else {
+        config.tools = mapChatMessagesToGoogleFunctions(params.tools);
+      }
+    }
+    const response = await this.client.models.generateContentStream({
+      model: this.model,
+      contents: mapChatMessagesToGoogleMessages(params.messages),
+      config,
+    });
+    yield* streamConverter(response, (response) => {
+      if (response.functionCalls?.length) {
+        return {
+          delta: "",
+          raw: response,
+          options: {
+            toolCall: this.getToolCallsFromResponse(response),
+          },
+        };
+      }
+
+      return {
+        delta: response.text ?? "",
+        raw: response,
+        options: {
+          inlineData: getGoogleStudioInlineData(response),
+        },
+      };
+    });
+  }
+
+  chat(params: GoogleChatParamsStreaming): Promise<GoogleChatStreamResponse>;
+  chat(
+    params: GoogleChatParamsNonStreaming,
+  ): Promise<GoogleChatNonStreamResponse>;
+  @wrapLLMEvent
+  async chat(
+    params: GoogleChatParamsStreaming | GoogleChatParamsNonStreaming,
+  ): Promise<GoogleChatStreamResponse | GoogleChatNonStreamResponse> {
+    if (params.stream) {
+      return this.streamChat(params);
+    }
+    return this.nonStreamChat(params);
+  }
+
+  complete(
+    params: LLMCompletionParamsStreaming,
+  ): Promise<AsyncIterable<CompletionResponse>>;
+  complete(
+    params: LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse>;
+  async complete(
+    params: LLMCompletionParamsStreaming | LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse | AsyncIterable<CompletionResponse>> {
+    const contents = mapMessageContentToMessageContentDetails(
+      params.prompt,
+    ).map(mapMessageContentDetailToGooglePart);
+    if (params.stream) {
+      const response = await this.client.models.generateContentStream({
+        model: this.model,
+        contents,
+      });
+      return streamConverter(response, (response) => {
+        return {
+          text: response.text ?? "",
+          raw: response,
+        };
+      });
+    }
+    const response = await this.client.models.generateContent({
+      model: this.model,
+      contents,
+    });
+
+    return {
+      text: response.text || "",
+      raw: response,
+    };
+  }
+}

packages/providers/google/src/studio/utils.ts

@@ -0,0 +1,136 @@
+import type {
+  ContentListUnion,
+  ContentUnion,
+  Part,
+  Schema,
+  ToolListUnion,
+} from "@google/genai";
+import type {
+  BaseTool,
+  ChatMessage,
+  MessageContentDetail,
+  ToolCallLLMMessageOptions,
+} from "@llamaindex/core/llms";
+import { extractDataUrlComponents } from "@llamaindex/core/utils";
+
+import type { MessageContent } from "@llamaindex/core/llms";
+
+export const mapMessageContentToMessageContentDetails = (
+  content: MessageContent,
+): MessageContentDetail[] => {
+  return Array.isArray(content) ? content : [{ type: "text", text: content }];
+};
+
+const ACCEPTED_IMAGE_MIME_TYPES = ["image/jpeg", "image/png"];
+
+export const mapTextPart = (text: string): Part => {
+  return { text };
+};
+
+export const mapImagePart = (imageUrl: string): Part => {
+  if (!imageUrl.startsWith("data:"))
+    throw new Error(
+      "For Google please only use base64 data url, e.g.: data:image/jpeg;base64,SGVsbG8sIFdvcmxkIQ==",
+    );
+  const { mimeType, base64: data } = extractDataUrlComponents(imageUrl);
+  if (!ACCEPTED_IMAGE_MIME_TYPES.includes(mimeType))
+    throw new Error(
+      `Anthropic only accepts the following mimeTypes: ${ACCEPTED_IMAGE_MIME_TYPES.join("\n")}`,
+    );
+
+  return {
+    inlineData: {
+      mimeType,
+      data,
+    },
+  };
+};
+
+export const mapMessageContentDetailToGooglePart = <
+  T extends MessageContentDetail,
+>(
+  detail: T,
+): Part => {
+  let part: Part;
+
+  if (detail.type === "text") {
+    part = mapTextPart(detail.text);
+  } else if (detail.type === "image_url") {
+    part = mapImagePart(detail.image_url.url);
+  } else {
+    throw new Error("Unsupported content detail type");
+  }
+  return part;
+};
+export const mapChatMessagesToGoogleFunctions = (
+  tools: BaseTool[],
+): ToolListUnion => {
+  return [
+    {
+      functionDeclarations: tools.map((tool) => ({
+        response: tool.metadata.parameters as Schema,
+        description: tool.metadata.description,
+        name: tool.metadata.name,
+      })),
+    },
+  ];
+};
+
+export const mapChatMessagesToGoogleMessages = <
+  T extends ChatMessage<ToolCallLLMMessageOptions>,
+>(
+  messages: T[],
+): ContentListUnion => {
+  const functionNames: Record<string, string> = {};
+  messages.forEach((msg: T) => {
+    if (msg.options && "toolCall" in msg.options) {
+      const mapped = msg.options.toolCall.reduce(
+        (result, item) => {
+          result[item.id] = item.name;
+          return result;
+        },
+        {} as Record<string, string>,
+      );
+
+      Object.assign(functionNames, mapped);
+    }
+  });
+  return messages.flatMap((msg: T): ContentListUnion => {
+    if (msg.options && "toolResult" in msg.options) {
+      return {
+        role: "user",
+        parts: [
+          {
+            functionResponse: {
+              name: functionNames[msg.options.toolResult.id] ?? "",
+              response: msg.options.toolResult,
+            },
+          },
+        ],
+      };
+    }
+
+    if (msg.options && "toolCall" in msg.options) {
+      return {
+        role: "model",
+        parts: msg.options.toolCall.map((call) => ({
+          functionCall: {
+            name: call.name,
+            args: call.input as Record<string, unknown>,
+          },
+        })),
+      };
+    }
+    return mapMessageContentToMessageContentDetails(msg.content)
+      .map((detail: MessageContentDetail): ContentUnion | null => {
+        const part = mapMessageContentDetailToGooglePart(detail);
+        if (!part.text && !part.inlineData) return null;
+
+        return {
+          role: msg.role === "assistant" ? "model" : "user",
+          parts: [part],
+        };
+      })
+      .filter((content) => content) as ContentUnion;
+  });
+};

packages/providers/google/src/vertex.ts

@@ -59,14 +59,15 @@ export class GeminiVertexSession implements IGeminiSession {
   getGenerativeModel(
     metadata: VertexModelParams,
   ): VertexGenerativeModelPreview | VertexGenerativeModel {
+    const safetySettings = metadata.safetySettings ?? DEFAULT_SAFETY_SETTINGS;
     if (this.preview) {
       return this.vertex.preview.getGenerativeModel({
-        safetySettings: DEFAULT_SAFETY_SETTINGS,
+        safetySettings,
         ...metadata,
       });
     }
     return this.vertex.getGenerativeModel({
-      safetySettings: DEFAULT_SAFETY_SETTINGS,
+      safetySettings,
       ...metadata,
     });
   }

packages/providers/groq/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @llamaindex/groq
 
+## 0.0.61
+
+### Patch Changes
+
+- Updated dependencies [91a18e7]
+  - @llamaindex/openai@0.2.0
+
 ## 0.0.60
 
 ### Patch Changes

packages/providers/groq/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/groq",
   "description": "Groq Adapter for LlamaIndex",
-  "version": "0.0.60",
+  "version": "0.0.61",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/huggingface/CHANGELOG.md

@@ -1,5 +1,21 @@
 # @llamaindex/huggingface
 
+## 0.1.0
+
+### Minor Changes
+
+- 91a18e7: Added support for structured output in the chat api of openai and ollama
+  Added structured output parameter in the provider
+
+### Patch Changes
+
+- Updated dependencies [21bebfc]
+- Updated dependencies [93bc0ff]
+- Updated dependencies [91a18e7]
+- Updated dependencies [5189b44]
+  - @llamaindex/core@0.6.0
+  - @llamaindex/openai@0.2.0
+
 ## 0.0.45
 
 ### Patch Changes

packages/providers/huggingface/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/huggingface",
   "description": "Huggingface Adapter for LlamaIndex",
-  "version": "0.0.45",
+  "version": "0.1.0",
   "type": "module",
   "types": "dist/index.d.ts",
   "main": "dist/index.cjs",

packages/providers/huggingface/src/llm.ts

@@ -57,6 +57,7 @@ export class HuggingFaceLLM extends BaseLLM {
       maxTokens: this.maxTokens,
       contextWindow: this.contextWindow,
       tokenizer: undefined,
+      structuredOutput: false,
     };
   }
 

packages/providers/huggingface/src/shared.ts

@@ -123,6 +123,7 @@ export class HuggingFaceInferenceAPI extends BaseLLM {
       maxTokens: this.maxTokens,
       contextWindow: this.contextWindow,
       tokenizer: undefined,
+      structuredOutput: false,
     };
   }
 

packages/providers/jinaai/CHANGELOG.md

@@ -1,5 +1,16 @@
 # @llamaindex/jinaai
 
+## 0.0.6
+
+### Patch Changes
+
+- Updated dependencies [21bebfc]
+- Updated dependencies [93bc0ff]
+- Updated dependencies [91a18e7]
+- Updated dependencies [5189b44]
+  - @llamaindex/core@0.6.0
+  - @llamaindex/openai@0.2.0
+
 ## 0.0.5
 
 ### Patch Changes

packages/providers/jinaai/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/jinaai",
   "description": "JinaAI Adapter for LlamaIndex",
-  "version": "0.0.5",
+  "version": "0.0.6",
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

packages/providers/mistral/CHANGELOG.md

@@ -1,5 +1,23 @@
 # @llamaindex/mistral
 
+## 0.1.0
+
+### Minor Changes
+
+- 91a18e7: Added support for structured output in the chat api of openai and ollama
+  Added structured output parameter in the provider
+- d1c1f99: Added support for function calling in mistral provider
+  Update model list for mistral provider
+  Added example for the tool call in mistral
+
+### Patch Changes
+
+- Updated dependencies [21bebfc]
+- Updated dependencies [93bc0ff]
+- Updated dependencies [91a18e7]
+- Updated dependencies [5189b44]
+  - @llamaindex/core@0.6.0
+
 ## 0.0.14
 
 ### Patch Changes