Release 0.2.19 (#365)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.github/workflows/e2e.yml

@@ -19,7 +19,7 @@ jobs:
         python-version: ["3.11"]
         os: [macos-latest, windows-latest, ubuntu-22.04]
         frameworks: ["fastapi"]
-        datasources: ["--no-files", "--example-file"]
+        datasources: ["--no-files", "--example-file", "--llamacloud"]
     defaults:
       run:
         shell: bash

.github/workflows/release.yml

@@ -17,6 +17,9 @@ jobs:
 
       - uses: pnpm/action-setup@v3
 
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:

.husky/pre-commit

@@ -1,2 +1,3 @@
 pnpm format
 pnpm lint
+uvx ruff format --check templates/

CHANGELOG.md

@@ -1,5 +1,36 @@
 # create-llama
 
+## 0.2.19
+
+### Patch Changes
+
+- 3d41488: feat: use selected llamacloud for multiagent
+
+## 0.2.18
+
+### Patch Changes
+
+- 75e1f61: Fix cannot query public document from llamacloud
+- 88220f1: fix workflow doesn't stop when user presses stop generation button
+- 75e1f61: Fix typescript templates cannot upload file to llamacloud
+- 88220f1: Bump llama_index@0.11.17
+
+## 0.2.17
+
+### Patch Changes
+
+- cd3fcd0: bump: use LlamaIndexTS 0.6.18
+- 6335de1: Fix using LlamaCloud selector does not use the configured values in the environment (Python)
+
+## 0.2.16
+
+### Patch Changes
+
+- 0e78ba4: Fix: programmatically ensure index for LlamaCloud
+- 0e78ba4: Fix .env not loaded on poetry run generate
+- 7f4ac22: Don't need to run generate script for LlamaCloud
+- 5263bde: Use selected LlamaCloud index in multi-agent template
+
 ## 0.2.15
 
 ### Patch Changes

e2e/python/resolve_dependencies.spec.ts

@@ -15,6 +15,8 @@ const dataSource: string = process.env.DATASOURCE
   ? process.env.DATASOURCE
   : "--example-file";
 
+// TODO: add support for other templates
+
 if (
   dataSource === "--example-file" // XXX: this test provides its own data source - only trigger it on one data source (usually the CI matrix will trigger multiple data sources)
 ) {
@@ -45,14 +47,86 @@ if (
 
   const observabilityOptions = ["llamatrace", "traceloop"];
 
-  // Run separate tests for each observability option to reduce CI runtime
-  test.describe("Test resolve python dependencies with observability", () => {
-    // Testing with streaming template, vectorDb: none, tools: none, and dataSource: --example-file
-    for (const observability of observabilityOptions) {
-      test(`observability: ${observability}`, async () => {
-        const cwd = await createTestDir();
+  test.describe("Mypy check", () => {
+    test.describe.configure({ retries: 0 });
 
-        await createAndCheckLlamaProject({
+    // Test vector databases
+    for (const vectorDb of vectorDbs) {
+      test(`Mypy check for vectorDB: ${vectorDb}`, async () => {
+        const cwd = await createTestDir();
+        const { pyprojectPath } = await createAndCheckLlamaProject({
+          options: {
+            cwd,
+            templateType: "streaming",
+            templateFramework,
+            dataSource: "--example-file",
+            vectorDb,
+            tools: "none",
+            port: 3000,
+            externalPort: 8000,
+            postInstallAction: "none",
+            templateUI: undefined,
+            appType: "--no-frontend",
+            llamaCloudProjectName: undefined,
+            llamaCloudIndexName: undefined,
+            observability: undefined,
+          },
+        });
+
+        const pyprojectContent = fs.readFileSync(pyprojectPath, "utf-8");
+        if (vectorDb !== "none") {
+          if (vectorDb === "pg") {
+            expect(pyprojectContent).toContain(
+              "llama-index-vector-stores-postgres",
+            );
+          } else {
+            expect(pyprojectContent).toContain(
+              `llama-index-vector-stores-${vectorDb}`,
+            );
+          }
+        }
+      });
+    }
+
+    // Test tools
+    for (const tool of toolOptions) {
+      test(`Mypy check for tool: ${tool}`, async () => {
+        const cwd = await createTestDir();
+        const { pyprojectPath } = await createAndCheckLlamaProject({
+          options: {
+            cwd,
+            templateType: "streaming",
+            templateFramework,
+            dataSource: "--example-file",
+            vectorDb: "none",
+            tools: tool,
+            port: 3000,
+            externalPort: 8000,
+            postInstallAction: "none",
+            templateUI: undefined,
+            appType: "--no-frontend",
+            llamaCloudProjectName: undefined,
+            llamaCloudIndexName: undefined,
+            observability: undefined,
+          },
+        });
+
+        const pyprojectContent = fs.readFileSync(pyprojectPath, "utf-8");
+        if (tool === "wikipedia.WikipediaToolSpec") {
+          expect(pyprojectContent).toContain("wikipedia");
+        }
+        if (tool === "google.GoogleSearchToolSpec") {
+          expect(pyprojectContent).toContain("google");
+        }
+      });
+    }
+
+    // Test data sources
+    for (const dataSource of dataSources) {
+      const dataSourceType = dataSource.split(" ")[0];
+      test(`Mypy check for data source: ${dataSourceType}`, async () => {
+        const cwd = await createTestDir();
+        const { pyprojectPath } = await createAndCheckLlamaProject({
           options: {
             cwd,
             templateType: "streaming",
@@ -60,89 +134,53 @@ if (
             dataSource,
             vectorDb: "none",
             tools: "none",
-            port: 3000, // port, not used
-            externalPort: 8000, // externalPort, not used
-            postInstallAction: "none", // postInstallAction
-            templateUI: undefined, // ui
-            appType: "--no-frontend", // appType
-            llamaCloudProjectName: undefined, // llamaCloudProjectName
-            llamaCloudIndexName: undefined, // llamaCloudIndexName
+            port: 3000,
+            externalPort: 8000,
+            postInstallAction: "none",
+            templateUI: undefined,
+            appType: "--no-frontend",
+            llamaCloudProjectName: undefined,
+            llamaCloudIndexName: undefined,
+            observability: undefined,
+          },
+        });
+
+        const pyprojectContent = fs.readFileSync(pyprojectPath, "utf-8");
+        if (dataSource.includes("--web-source")) {
+          expect(pyprojectContent).toContain("llama-index-readers-web");
+        }
+        if (dataSource.includes("--db-source")) {
+          expect(pyprojectContent).toContain("llama-index-readers-database");
+        }
+      });
+    }
+
+    // Test observability options
+    for (const observability of observabilityOptions) {
+      test(`Mypy check for observability: ${observability}`, async () => {
+        const cwd = await createTestDir();
+
+        const { pyprojectPath } = await createAndCheckLlamaProject({
+          options: {
+            cwd,
+            templateType: "streaming",
+            templateFramework,
+            dataSource: "--example-file",
+            vectorDb: "none",
+            tools: "none",
+            port: 3000,
+            externalPort: 8000,
+            postInstallAction: "none",
+            templateUI: undefined,
+            appType: "--no-frontend",
+            llamaCloudProjectName: undefined,
+            llamaCloudIndexName: undefined,
             observability,
           },
         });
       });
     }
   });
-
-  test.describe("Test resolve python dependencies", () => {
-    for (const vectorDb of vectorDbs) {
-      for (const tool of toolOptions) {
-        for (const dataSource of dataSources) {
-          const dataSourceType = dataSource.split(" ")[0];
-          const toolDescription = tool === "none" ? "no tools" : tool;
-          const optionDescription = `vectorDb: ${vectorDb}, ${toolDescription}, dataSource: ${dataSourceType}`;
-
-          test(`options: ${optionDescription}`, async () => {
-            const cwd = await createTestDir();
-
-            const { pyprojectPath, projectPath } =
-              await createAndCheckLlamaProject({
-                options: {
-                  cwd,
-                  templateType: "streaming",
-                  templateFramework,
-                  dataSource,
-                  vectorDb,
-                  tools: tool,
-                  port: 3000, // port, not used
-                  externalPort: 8000, // externalPort, not used
-                  postInstallAction: "none", // postInstallAction
-                  templateUI: undefined, // ui
-                  appType: "--no-frontend", // appType
-                  llamaCloudProjectName: undefined, // llamaCloudProjectName
-                  llamaCloudIndexName: undefined, // llamaCloudIndexName
-                  observability: undefined, // observability
-                },
-              });
-
-            // Additional checks for specific dependencies
-
-            // Verify that specific dependencies are in pyproject.toml
-            const pyprojectContent = fs.readFileSync(pyprojectPath, "utf-8");
-            if (vectorDb !== "none") {
-              if (vectorDb === "pg") {
-                expect(pyprojectContent).toContain(
-                  "llama-index-vector-stores-postgres",
-                );
-              } else {
-                expect(pyprojectContent).toContain(
-                  `llama-index-vector-stores-${vectorDb}`,
-                );
-              }
-            }
-            if (tool !== "none") {
-              if (tool === "wikipedia.WikipediaToolSpec") {
-                expect(pyprojectContent).toContain("wikipedia");
-              }
-              if (tool === "google.GoogleSearchToolSpec") {
-                expect(pyprojectContent).toContain("google");
-              }
-            }
-
-            // Check for data source specific dependencies
-            if (dataSource.includes("--web-source")) {
-              expect(pyprojectContent).toContain("llama-index-readers-web");
-            }
-            if (dataSource.includes("--db-source")) {
-              expect(pyprojectContent).toContain(
-                "llama-index-readers-database ",
-              );
-            }
-          });
-        }
-      }
-    }
-  });
 }
 
 async function createAndCheckLlamaProject({
@@ -161,21 +199,39 @@ async function createAndCheckLlamaProject({
   const pyprojectPath = path.join(projectPath, "pyproject.toml");
   expect(fs.existsSync(pyprojectPath)).toBeTruthy();
 
-  // Run poetry lock
+  const env = {
+    ...process.env,
+    POETRY_VIRTUALENVS_IN_PROJECT: "true",
+  };
+
+  // Run poetry install
   try {
-    const { stdout, stderr } = await execAsync(
-      "poetry config virtualenvs.in-project true && poetry lock --no-update",
-      { cwd: projectPath },
+    const { stdout: installStdout, stderr: installStderr } = await execAsync(
+      "poetry install",
+      { cwd: projectPath, env },
     );
-    console.log("poetry lock stdout:", stdout);
-    console.error("poetry lock stderr:", stderr);
+    console.log("poetry install stdout:", installStdout);
+    console.error("poetry install stderr:", installStderr);
   } catch (error) {
-    console.error("Error running poetry lock:", error);
+    console.error("Error running poetry install:", error);
     throw error;
   }
 
-  // Check if poetry.lock file was created
-  expect(fs.existsSync(path.join(projectPath, "poetry.lock"))).toBeTruthy();
+  // Run poetry run mypy
+  try {
+    const { stdout: mypyStdout, stderr: mypyStderr } = await execAsync(
+      "poetry run mypy .",
+      { cwd: projectPath, env },
+    );
+    console.log("poetry run mypy stdout:", mypyStdout);
+    console.error("poetry run mypy stderr:", mypyStderr);
+  } catch (error) {
+    console.error("Error running mypy:", error);
+    throw error;
+  }
+
+  // If we reach this point without throwing an error, the test passes
+  expect(true).toBeTruthy();
 
   return { pyprojectPath, projectPath };
 }

e2e/shared/streaming_template.spec.ts

@@ -27,6 +27,13 @@ const userMessage =
   dataSource !== "--no-files" ? "Physical standard for letters" : "Hello";
 
 test.describe(`Test streaming template ${templateFramework} ${dataSource} ${templateUI} ${appType} ${templatePostInstallAction}`, async () => {
+  const isNode18 = process.version.startsWith("v18");
+  const isLlamaCloud = dataSource === "--llamacloud";
+  // llamacloud is using File API which is not supported on node 18
+  if (isNode18 && isLlamaCloud) {
+    test.skip(true, "Skipping tests for Node 18 and LlamaCloud data source");
+  }
+
   let port: number;
   let externalPort: number;
   let cwd: string;

helpers/env-variables.ts

@@ -65,7 +65,7 @@ const getVectorDBEnvs = (
         {
           name: "PG_CONNECTION_STRING",
           description:
-            "For generating a connection URI, see https://docs.timescale.com/use-timescale/latest/services/create-a-service\nThe PostgreSQL connection string.",
+            "For generating a connection URI, see https://supabase.com/vector\nThe PostgreSQL connection string.",
         },
       ];
 

helpers/python.ts

@@ -123,7 +123,7 @@ const getAdditionalDependencies = (
             extras: ["rsa"],
           });
           dependencies.push({
-            name: "psycopg2",
+            name: "psycopg2-binary",
             version: "^2.9.9",
           });
           break;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-llama",
-  "version": "0.2.15",
+  "version": "0.2.19",
   "description": "Create LlamaIndex-powered apps with one command",
   "keywords": [
     "rag",

templates/components/engines/python/agent/engine.py

@@ -1,17 +1,19 @@
 import os
+from typing import List
 
 from app.engine.index import IndexConfig, get_index
 from app.engine.tools import ToolFactory
 from llama_index.core.agent import AgentRunner
 from llama_index.core.callbacks import CallbackManager
 from llama_index.core.settings import Settings
+from llama_index.core.tools import BaseTool
 from llama_index.core.tools.query_engine import QueryEngineTool
 
 
 def get_chat_engine(filters=None, params=None, event_handlers=None, **kwargs):
     system_prompt = os.getenv("SYSTEM_PROMPT")
     top_k = int(os.getenv("TOP_K", 0))
-    tools = []
+    tools: List[BaseTool] = []
     callback_manager = CallbackManager(handlers=event_handlers or [])
 
     # Add query tool if index exists
@@ -25,7 +27,8 @@ def get_chat_engine(filters=None, params=None, event_handlers=None, **kwargs):
         tools.append(query_engine_tool)
 
     # Add additional tools
-    tools += ToolFactory.from_env()
+    configured_tools: List[BaseTool] = ToolFactory.from_env()
+    tools.extend(configured_tools)
 
     return AgentRunner.from_llm(
         llm=Settings.llm,

templates/components/engines/python/agent/tools/__init__.py

@@ -1,7 +1,8 @@
 import importlib
 import os
+from typing import Dict, List, Union
 
-import yaml
+import yaml  # type: ignore
 from llama_index.core.tools.function_tool import FunctionTool
 from llama_index.core.tools.tool_spec.base import BaseToolSpec
 
@@ -17,7 +18,8 @@ class ToolFactory:
         ToolType.LOCAL: "app.engine.tools",
     }
 
-    def load_tools(tool_type: str, tool_name: str, config: dict) -> list[FunctionTool]:
+    @staticmethod
+    def load_tools(tool_type: str, tool_name: str, config: dict) -> List[FunctionTool]:
         source_package = ToolFactory.TOOL_SOURCE_PACKAGE_MAP[tool_type]
         try:
             if "ToolSpec" in tool_name:
@@ -43,24 +45,32 @@ class ToolFactory:
     @staticmethod
     def from_env(
         map_result: bool = False,
-    ) -> list[FunctionTool] | dict[str, FunctionTool]:
+    ) -> Union[Dict[str, List[FunctionTool]], List[FunctionTool]]:
         """
         Load tools from the configured file.
-        Params:
-            - use_map: if True, return map of tool name and the tool itself
+
+        Args:
+            map_result: If True, return a map of tool names to their corresponding tools.
+
+        Returns:
+            A dictionary of tool names to lists of FunctionTools if map_result is True,
+            otherwise a list of FunctionTools.
         """
-        if map_result:
-            tools = {}
-        else:
-            tools = []
+        tools: Union[Dict[str, List[FunctionTool]], List[FunctionTool]] = (
+            {} if map_result else []
+        )
+
         if os.path.exists("config/tools.yaml"):
             with open("config/tools.yaml", "r") as f:
                 tool_configs = yaml.safe_load(f)
                 for tool_type, config_entries in tool_configs.items():
                     for tool_name, config in config_entries.items():
-                        tool = ToolFactory.load_tools(tool_type, tool_name, config)
+                        loaded_tools = ToolFactory.load_tools(
+                            tool_type, tool_name, config
+                        )
                         if map_result:
-                            tools[tool_name] = tool
+                            tools[tool_name] = loaded_tools  # type: ignore
                         else:
-                            tools.extend(tool)
+                            tools.extend(loaded_tools)  # type: ignore
+
         return tools

templates/components/engines/python/agent/tools/artifact.py

@@ -87,7 +87,7 @@ class CodeGeneratorTool:
             ChatMessage(role="user", content=user_message),
         ]
         try:
-            sllm = Settings.llm.as_structured_llm(output_cls=CodeArtifact)
+            sllm = Settings.llm.as_structured_llm(output_cls=CodeArtifact)  # type: ignore
             response = sllm.chat(messages)
             data: CodeArtifact = response.raw
             return data.model_dump()

templates/components/engines/python/agent/tools/duckduckgo.py

@@ -21,14 +21,15 @@ def duckduckgo_search(
             "Please install it by running: `poetry add duckduckgo_search` or `pip install duckduckgo_search`"
         )
 
-    params = {
-        "keywords": query,
-        "region": region,
-        "max_results": max_results,
-    }
     results = []
     with DDGS() as ddg:
-        results = list(ddg.text(**params))
+        results = list(
+            ddg.text(
+                keywords=query,
+                region=region,
+                max_results=max_results,
+            )
+        )
     return results
 
 
@@ -51,13 +52,14 @@ def duckduckgo_image_search(
             "duckduckgo_search package is required to use this function."
             "Please install it by running: `poetry add duckduckgo_search` or `pip install duckduckgo_search`"
         )
-    params = {
-        "keywords": query,
-        "region": region,
-        "max_results": max_results,
-    }
     with DDGS() as ddg:
-        results = list(ddg.images(**params))
+        results = list(
+            ddg.images(
+                keywords=query,
+                region=region,
+                max_results=max_results,
+            )
+        )
     return results
 
 

templates/components/engines/python/chat/engine.py

@@ -43,6 +43,6 @@ def get_chat_engine(filters=None, params=None, event_handlers=None, **kwargs):
         memory=memory,
         system_prompt=system_prompt,
         retriever=retriever,
-        node_postprocessors=node_postprocessors,
+        node_postprocessors=node_postprocessors,  # type: ignore
         callback_manager=callback_manager,
     )

templates/components/engines/typescript/agent/chat.ts

@@ -1,6 +1,6 @@
 import {
+  BaseChatEngine,
   BaseToolWithCall,
-  ChatEngine,
   OpenAIAgent,
   QueryEngineTool,
 } from "llamaindex";
@@ -45,7 +45,7 @@ export async function createChatEngine(documentIds?: string[], params?: any) {
   const agent = new OpenAIAgent({
     tools,
     systemPrompt: process.env.SYSTEM_PROMPT,
-  }) as unknown as ChatEngine;
+  }) as unknown as BaseChatEngine;
 
   return agent;
 }

templates/components/llamaindex/typescript/documents/upload.ts

@@ -16,14 +16,26 @@ export async function uploadDocument(
     // trigger LlamaCloudIndex API to upload the file and run the pipeline
     const projectId = await index.getProjectId();
     const pipelineId = await index.getPipelineId();
-    return [
-      await LLamaCloudFileService.addFileToPipeline(
-        projectId,
-        pipelineId,
-        new File([fileBuffer], filename, { type: mimeType }),
-        { private: "true" },
-      ),
-    ];
+    try {
+      return [
+        await LLamaCloudFileService.addFileToPipeline(
+          projectId,
+          pipelineId,
+          new File([fileBuffer], filename, { type: mimeType }),
+          { private: "true" },
+        ),
+      ];
+    } catch (error) {
+      if (
+        error instanceof ReferenceError &&
+        error.message.includes("File is not defined")
+      ) {
+        throw new Error(
+          "File class is not supported in the current Node.js version. Please use Node.js 20 or higher.",
+        );
+      }
+      throw error;
+    }
   }
 
   // run the pipeline for other vector store indexes

templates/components/llamaindex/typescript/streaming/annotations.ts

@@ -172,3 +172,26 @@ function getValidAnnotation(annotation: JSONValue): Annotation {
   }
   return { type: annotation.type, data: annotation.data };
 }
+
+// validate and get all annotations of a specific type or role from the frontend messages
+export function getAnnotations<
+  T extends Annotation["data"] = Annotation["data"],
+>(
+  messages: Message[],
+  options?: {
+    role?: Message["role"]; // message role
+    type?: Annotation["type"]; // annotation type
+  },
+): {
+  type: string;
+  data: T;
+}[] {
+  const messagesByRole = options?.role
+    ? messages.filter((msg) => msg.role === options?.role)
+    : messages;
+  const annotations = getAllAnnotations(messagesByRole);
+  const annotationsByType = options?.type
+    ? annotations.filter((a) => a.type === options.type)
+    : annotations;
+  return annotationsByType as { type: string; data: T }[];
+}

templates/components/llamaindex/typescript/streaming/events.ts

@@ -75,7 +75,7 @@ export function createCallbackManager(stream: StreamData) {
   callbackManager.on("retrieve-end", (data) => {
     const { nodes, query } = data.detail;
     appendSourceData(stream, nodes);
-    appendEventData(stream, `Retrieving context for query: '${query}'`);
+    appendEventData(stream, `Retrieving context for query: '${query.query}'`);
     appendEventData(
       stream,
       `Retrieved ${nodes.length} sources to use as context for the query`,

templates/components/loaders/python/__init__.py

@@ -1,20 +1,22 @@
 import logging
+from typing import Any, Dict, List
 
-import yaml
+import yaml  # type: ignore
 from app.engine.loaders.db import DBLoaderConfig, get_db_documents
 from app.engine.loaders.file import FileLoaderConfig, get_file_documents
 from app.engine.loaders.web import WebLoaderConfig, get_web_documents
+from llama_index.core import Document
 
 logger = logging.getLogger(__name__)
 
 
-def load_configs():
+def load_configs() -> Dict[str, Any]:
     with open("config/loaders.yaml") as f:
         configs = yaml.safe_load(f)
     return configs
 
 
-def get_documents():
+def get_documents() -> List[Document]:
     documents = []
     config = load_configs()
     for loader_type, loader_config in config.items():

templates/components/loaders/python/db.py

@@ -1,5 +1,6 @@
 import logging
 from typing import List
+
 from pydantic import BaseModel
 
 logger = logging.getLogger(__name__)
@@ -11,7 +12,13 @@ class DBLoaderConfig(BaseModel):
 
 
 def get_db_documents(configs: list[DBLoaderConfig]):
-    from llama_index.readers.database import DatabaseReader
+    try:
+        from llama_index.readers.database import DatabaseReader
+    except ImportError:
+        logger.error(
+            "Failed to import DatabaseReader. Make sure llama_index is installed."
+        )
+        raise
 
     docs = []
     for entry in configs:

templates/components/loaders/python/web.py

@@ -1,3 +1,5 @@
+from typing import List, Optional
+
 from pydantic import BaseModel, Field
 
 
@@ -8,8 +10,8 @@ class CrawlUrl(BaseModel):
 
 
 class WebLoaderConfig(BaseModel):
-    driver_arguments: list[str] = Field(default=None)
-    urls: list[CrawlUrl]
+    driver_arguments: Optional[List[str]] = Field(default_factory=list)
+    urls: List[CrawlUrl]
 
 
 def get_web_documents(config: WebLoaderConfig):

templates/components/multiagent/python/app/api/routers/chat.py

@@ -1,11 +1,10 @@
 import logging
 
-from app.api.routers.events import EventCallbackHandler
 from app.api.routers.models import (
     ChatData,
 )
 from app.api.routers.vercel_response import VercelStreamResponse
-from app.engine import get_chat_engine
+from app.engine.engine import get_chat_engine
 from fastapi import APIRouter, BackgroundTasks, HTTPException, Request, status
 
 chat_router = r = APIRouter()
@@ -23,13 +22,12 @@ async def chat(
         last_message_content = data.get_last_message_content()
         messages = data.get_history_messages(include_agent_messages=True)
 
-        event_handler = EventCallbackHandler()
         # The chat API supports passing private document filters and chat params
         # but agent workflow does not support them yet
         # ignore chat params and use all documents for now
         # TODO: generate filters based on doc_ids
-        # TODO: use chat params
-        engine = get_chat_engine(chat_history=messages)
+        params = data.data or {}
+        engine = get_chat_engine(chat_history=messages, params=params)
 
         event_handler = engine.run(input=last_message_content, streaming=True)
         return VercelStreamResponse(

templates/components/multiagent/python/app/api/routers/vercel_response.py

@@ -1,6 +1,6 @@
+import asyncio
 import json
 import logging
-from abc import ABC
 from typing import AsyncGenerator, List
 
 from aiostream import stream
@@ -13,7 +13,7 @@ from fastapi.responses import StreamingResponse
 logger = logging.getLogger("uvicorn")
 
 
-class VercelStreamResponse(StreamingResponse, ABC):
+class VercelStreamResponse(StreamingResponse):
     """
     Base class to convert the response from the chat engine to the streaming format expected by Vercel
     """
@@ -23,26 +23,34 @@ class VercelStreamResponse(StreamingResponse, ABC):
 
     def __init__(self, request: Request, chat_data: ChatData, *args, **kwargs):
         self.request = request
-
-        stream = self._create_stream(request, chat_data, *args, **kwargs)
-        content = self.content_generator(stream)
-
+        self.chat_data = chat_data
+        content = self.content_generator(*args, **kwargs)
         super().__init__(content=content)
 
-    async def content_generator(self, stream):
+    async def content_generator(self, event_handler, events):
+        logger.info("Starting content_generator")
+        stream = self._create_stream(
+            self.request, self.chat_data, event_handler, events
+        )
         is_stream_started = False
+        try:
+            async with stream.stream() as streamer:
+                async for output in streamer:
+                    if not is_stream_started:
+                        is_stream_started = True
+                        # Stream a blank message to start the stream
+                        yield self.convert_text("")
 
-        async with stream.stream() as streamer:
-            async for output in streamer:
-                if not is_stream_started:
-                    is_stream_started = True
-                    # Stream a blank message to start the stream
-                    yield self.convert_text("")
-
-                yield output
-
-                if await self.request.is_disconnected():
-                    break
+                    yield output
+        except asyncio.CancelledError:
+            logger.info("Stopping workflow")
+            await event_handler.cancel_run()
+        except Exception as e:
+            logger.error(
+                f"Unexpected error in content_generator: {str(e)}", exc_info=True
+            )
+        finally:
+            logger.info("The stream has been stopped!")
 
     def _create_stream(
         self,

templates/components/multiagent/python/app/engine/engine.py

@@ -18,11 +18,11 @@ def get_chat_engine(
     agent_type = os.getenv("EXAMPLE_TYPE", "").lower()
     match agent_type:
         case "choreography":
-            agent = create_choreography(chat_history)
+            agent = create_choreography(chat_history, **kwargs)
         case "orchestrator":
-            agent = create_orchestrator(chat_history)
+            agent = create_orchestrator(chat_history, **kwargs)
         case _:
-            agent = create_workflow(chat_history)
+            agent = create_workflow(chat_history, **kwargs)
 
     logger.info(f"Using agent pattern: {agent_type}")
 

templates/components/multiagent/python/app/examples/choreography.py

@@ -8,8 +8,8 @@ from app.examples.researcher import create_researcher
 from llama_index.core.chat_engine.types import ChatMessage
 
 
-def create_choreography(chat_history: Optional[List[ChatMessage]] = None):
-    researcher = create_researcher(chat_history)
+def create_choreography(chat_history: Optional[List[ChatMessage]] = None, **kwargs):
+    researcher = create_researcher(chat_history, **kwargs)
     publisher = create_publisher(chat_history)
     reviewer = FunctionCallingAgent(
         name="reviewer",
@@ -21,12 +21,14 @@ def create_choreography(chat_history: Optional[List[ChatMessage]] = None):
         name="writer",
         agents=[researcher, reviewer, publisher],
         description="expert in writing blog posts, needs researched information and images to write a blog post",
-        system_prompt=dedent("""
+        system_prompt=dedent(
+            """
             You are an expert in writing blog posts. You are given a task to write a blog post. Before starting to write the post, consult the researcher agent to get the information you need. Don't make up any information yourself.
             After creating a draft for the post, send it to the reviewer agent to receive feedback and make sure to incorporate the feedback from the reviewer.
             You can consult the reviewer and researcher a maximum of two times. Your output should contain only the blog post.
             Finally, always request the publisher to create a document (PDF, HTML) and publish the blog post.
-        """),
+        """
+        ),
         # TODO: add chat_history support to AgentCallingAgent
         # chat_history=chat_history,
     )

templates/components/multiagent/python/app/examples/orchestrator.py

@@ -8,28 +8,32 @@ from app.examples.researcher import create_researcher
 from llama_index.core.chat_engine.types import ChatMessage
 
 
-def create_orchestrator(chat_history: Optional[List[ChatMessage]] = None):
-    researcher = create_researcher(chat_history)
+def create_orchestrator(chat_history: Optional[List[ChatMessage]] = None, **kwargs):
+    researcher = create_researcher(chat_history, **kwargs)
     writer = FunctionCallingAgent(
         name="writer",
         description="expert in writing blog posts, need information and images to write a post",
-        system_prompt=dedent("""
+        system_prompt=dedent(
+            """
             You are an expert in writing blog posts.
             You are given a task to write a blog post. Do not make up any information yourself.
             If you don't have the necessary information to write a blog post, reply "I need information about the topic to write the blog post".
             If you need to use images, reply "I need images about the topic to write the blog post". Do not use any dummy images made up by you.
             If you have all the information needed, write the blog post.
-        """),
+        """
+        ),
         chat_history=chat_history,
     )
     reviewer = FunctionCallingAgent(
         name="reviewer",
         description="expert in reviewing blog posts, needs a written blog post to review",
-        system_prompt=dedent("""
+        system_prompt=dedent(
+            """
             You are an expert in reviewing blog posts. You are given a task to review a blog post. Review the post and fix any issues found yourself. You must output a final blog post.
             A post must include at least one valid image. If not, reply "I need images about the topic to write the blog post". An image URL starting with "example" or "your website" is not valid.
             Especially check for logical inconsistencies and proofread the post for grammar and spelling errors.
-        """),
+        """
+        ),
         chat_history=chat_history,
     )
     publisher = create_publisher(chat_history)

templates/components/multiagent/python/app/examples/researcher.py

@@ -3,17 +3,19 @@ from textwrap import dedent
 from typing import List
 
 from app.agents.single import FunctionCallingAgent
-from app.engine.index import get_index
+from app.engine.index import IndexConfig, get_index
 from app.engine.tools import ToolFactory
 from llama_index.core.chat_engine.types import ChatMessage
 from llama_index.core.tools import QueryEngineTool, ToolMetadata
 
 
-def _create_query_engine_tool() -> QueryEngineTool:
+def _create_query_engine_tool(params=None) -> QueryEngineTool:
     """
     Provide an agent worker that can be used to query the index.
     """
-    index = get_index()
+    # Add query tool if index exists
+    index_config = IndexConfig(**(params or {}))
+    index = get_index(index_config)
     if index is None:
         return None
     top_k = int(os.getenv("TOP_K", 0))
@@ -31,13 +33,13 @@ def _create_query_engine_tool() -> QueryEngineTool:
     )
 
 
-def _get_research_tools() -> QueryEngineTool:
+def _get_research_tools(**kwargs) -> QueryEngineTool:
     """
     Researcher take responsibility for retrieving information.
     Try init wikipedia or duckduckgo tool if available.
     """
     tools = []
-    query_engine_tool = _create_query_engine_tool()
+    query_engine_tool = _create_query_engine_tool(**kwargs)
     if query_engine_tool is not None:
         tools.append(query_engine_tool)
     researcher_tool_names = ["duckduckgo", "wikipedia.WikipediaToolSpec"]
@@ -48,16 +50,17 @@ def _get_research_tools() -> QueryEngineTool:
     return tools
 
 
-def create_researcher(chat_history: List[ChatMessage]):
+def create_researcher(chat_history: List[ChatMessage], **kwargs):
     """
     Researcher is an agent that take responsibility for using tools to complete a given task.
     """
-    tools = _get_research_tools()
+    tools = _get_research_tools(**kwargs)
     return FunctionCallingAgent(
         name="researcher",
         tools=tools,
         description="expert in retrieving any unknown content or searching for images from the internet",
-        system_prompt=dedent("""
+        system_prompt=dedent(
+            """
             You are a researcher agent. You are given a research task.
             
             If the conversation already includes the information and there is no new request for additional information from the user, you should return the appropriate content to the writer.
@@ -77,6 +80,7 @@ def create_researcher(chat_history: List[ChatMessage]):
 
             If you use the tools but don't find any related information, please return "I didn't find any new information for {the topic}." along with the content you found. Don't try to make up information yourself.
             If the request doesn't need any new information because it was in the conversation history, please return "The task doesn't need any new information. Please reuse the existing content in the conversation history."
-        """),
+        """
+        ),
         chat_history=chat_history,
     )

templates/components/multiagent/python/app/examples/workflow.py

@@ -17,9 +17,10 @@ from llama_index.core.workflow import (
 )
 
 
-def create_workflow(chat_history: Optional[List[ChatMessage]] = None):
+def create_workflow(chat_history: Optional[List[ChatMessage]] = None, **kwargs):
     researcher = create_researcher(
         chat_history=chat_history,
+        **kwargs,
     )
     publisher = create_publisher(
         chat_history=chat_history,
@@ -127,7 +128,8 @@ class BlogPostWorkflow(Workflow):
         self, input: str, chat_history: List[ChatMessage]
     ) -> str:
         prompt_template = PromptTemplate(
-            dedent("""
+            dedent(
+                """
                 You are an expert in decision-making, helping people write and publish blog posts.
                 If the user is asking for a file or to publish content, respond with 'publish'.
                 If the user requests to write or update a blog post, respond with 'not_publish'.
@@ -140,7 +142,8 @@ class BlogPostWorkflow(Workflow):
 
                 Given the chat history and the new user request, decide whether to publish based on existing information.
                 Decision (respond with either 'not_publish' or 'publish'):
-            """)
+            """
+            )
         )
 
         chat_history_str = "\n".join(

templates/components/multiagent/typescript/express/chat.controller.ts

@@ -1,13 +1,13 @@
 import { StopEvent } from "@llamaindex/core/workflow";
 import { Message, streamToResponse } from "ai";
 import { Request, Response } from "express";
-import { ChatMessage, ChatResponseChunk } from "llamaindex";
+import { ChatResponseChunk } from "llamaindex";
 import { createWorkflow } from "./workflow/factory";
 import { toDataStream, workflowEventsToStreamData } from "./workflow/stream";
 
 export const chat = async (req: Request, res: Response) => {
   try {
-    const { messages }: { messages: Message[] } = req.body;
+    const { messages, data }: { messages: Message[]; data?: any } = req.body;
     const userMessage = messages.pop();
     if (!messages || !userMessage || userMessage.role !== "user") {
       return res.status(400).json({
@@ -16,8 +16,7 @@ export const chat = async (req: Request, res: Response) => {
       });
     }
 
-    const chatHistory = messages as ChatMessage[];
-    const agent = createWorkflow(chatHistory);
+    const agent = createWorkflow(messages, data);
     const result = agent.run<AsyncGenerator<ChatResponseChunk>>(
       userMessage.content,
     ) as unknown as Promise<StopEvent<AsyncGenerator<ChatResponseChunk>>>;

templates/components/multiagent/typescript/nextjs/route.ts

@@ -1,7 +1,7 @@
 import { initObservability } from "@/app/observability";
 import { StopEvent } from "@llamaindex/core/workflow";
 import { Message, StreamingTextResponse } from "ai";
-import { ChatMessage, ChatResponseChunk } from "llamaindex";
+import { ChatResponseChunk } from "llamaindex";
 import { NextRequest, NextResponse } from "next/server";
 import { initSettings } from "./engine/settings";
 import { createWorkflow } from "./workflow/factory";
@@ -16,7 +16,7 @@ export const dynamic = "force-dynamic";
 export async function POST(request: NextRequest) {
   try {
     const body = await request.json();
-    const { messages }: { messages: Message[] } = body;
+    const { messages, data }: { messages: Message[]; data?: any } = body;
     const userMessage = messages.pop();
     if (!messages || !userMessage || userMessage.role !== "user") {
       return NextResponse.json(
@@ -28,8 +28,7 @@ export async function POST(request: NextRequest) {
       );
     }
 
-    const chatHistory = messages as ChatMessage[];
-    const agent = createWorkflow(chatHistory);
+    const agent = createWorkflow(messages, data);
     // TODO: fix type in agent.run in LITS
     const result = agent.run<AsyncGenerator<ChatResponseChunk>>(
       userMessage.content,

templates/components/multiagent/typescript/workflow/agents.ts

@@ -1,14 +1,19 @@
 import { ChatMessage } from "llamaindex";
 import { FunctionCallingAgent } from "./single-agent";
-import { lookupTools } from "./tools";
+import { getQueryEngineTool, lookupTools } from "./tools";
 
-export const createResearcher = async (chatHistory: ChatMessage[]) => {
-  const tools = await lookupTools([
-    "query_index",
-    "wikipedia_tool",
-    "duckduckgo_search",
-    "image_generator",
-  ]);
+export const createResearcher = async (
+  chatHistory: ChatMessage[],
+  params?: any,
+) => {
+  const queryEngineTool = await getQueryEngineTool(params);
+  const tools = (
+    await lookupTools([
+      "wikipedia_tool",
+      "duckduckgo_search",
+      "image_generator",
+    ])
+  ).concat(queryEngineTool ? [queryEngineTool] : []);
 
   return new FunctionCallingAgent({
     name: "researcher",

templates/components/multiagent/typescript/workflow/factory.ts

@@ -5,7 +5,9 @@ import {
   Workflow,
   WorkflowEvent,
 } from "@llamaindex/core/workflow";
+import { Message } from "ai";
 import { ChatMessage, ChatResponseChunk, Settings } from "llamaindex";
+import { getAnnotations } from "../llamaindex/streaming/annotations";
 import {
   createPublisher,
   createResearcher,
@@ -25,19 +27,15 @@ class WriteEvent extends WorkflowEvent<{
 class ReviewEvent extends WorkflowEvent<{ input: string }> {}
 class PublishEvent extends WorkflowEvent<{ input: string }> {}
 
-const prepareChatHistory = (chatHistory: ChatMessage[]) => {
+const prepareChatHistory = (chatHistory: Message[]): ChatMessage[] => {
   // By default, the chat history only contains the assistant and user messages
   // all the agents messages are stored in annotation data which is not visible to the LLM
 
   const MAX_AGENT_MESSAGES = 10;
-
-  // Construct a new agent message from agent messages
-  // Get annotations from assistant messages
-  const agentAnnotations = chatHistory
-    .filter((msg) => msg.role === "assistant")
-    .flatMap((msg) => msg.annotations || [])
-    .filter((annotation) => annotation.type === "agent")
-    .slice(-MAX_AGENT_MESSAGES);
+  const agentAnnotations = getAnnotations<{ agent: string; text: string }>(
+    chatHistory,
+    { role: "assistant", type: "agent" },
+  ).slice(-MAX_AGENT_MESSAGES);
 
   const agentMessages = agentAnnotations
     .map(
@@ -59,13 +57,13 @@ const prepareChatHistory = (chatHistory: ChatMessage[]) => {
       ...chatHistory.slice(0, -1),
       agentMessage,
       chatHistory.slice(-1)[0],
-    ];
+    ] as ChatMessage[];
   }
-  return chatHistory;
+  return chatHistory as ChatMessage[];
 };
 
-export const createWorkflow = (chatHistory: ChatMessage[]) => {
-  const chatHistoryWithAgentMessages = prepareChatHistory(chatHistory);
+export const createWorkflow = (messages: Message[], params?: any) => {
+  const chatHistoryWithAgentMessages = prepareChatHistory(messages);
   const runAgent = async (
     context: Context,
     agent: Workflow,
@@ -123,7 +121,10 @@ Decision (respond with either 'not_publish' or 'publish'):`;
   };
 
   const research = async (context: Context, ev: ResearchEvent) => {
-    const researcher = await createResearcher(chatHistoryWithAgentMessages);
+    const researcher = await createResearcher(
+      chatHistoryWithAgentMessages,
+      params,
+    );
     const researchRes = await runAgent(context, researcher, {
       message: ev.data.input,
     });

templates/components/multiagent/typescript/workflow/single-agent.ts

@@ -143,7 +143,7 @@ export class FunctionCallingAgent extends Workflow {
         fullResponse = chunk;
       }
 
-      if (fullResponse) {
+      if (fullResponse?.options && Object.keys(fullResponse.options).length) {
         memory.put({
           role: "assistant",
           content: "",

templates/components/multiagent/typescript/workflow/tools.ts

@@ -4,8 +4,10 @@ import path from "path";
 import { getDataSource } from "../engine";
 import { createTools } from "../engine/tools/index";
 
-const getQueryEngineTool = async (): Promise<QueryEngineTool | null> => {
-  const index = await getDataSource();
+export const getQueryEngineTool = async (
+  params?: any,
+): Promise<QueryEngineTool | null> => {
+  const index = await getDataSource(params);
   if (!index) {
     return null;
   }

templates/components/settings/python/llmhub.py

@@ -1,7 +1,11 @@
-from llama_index.embeddings.openai import OpenAIEmbedding
-from llama_index.core.settings import Settings
-from typing import Dict
+import logging
 import os
+from typing import Dict
+
+from llama_index.core.settings import Settings
+from llama_index.embeddings.openai import OpenAIEmbedding
+
+logger = logging.getLogger(__name__)
 
 DEFAULT_MODEL = "gpt-3.5-turbo"
 DEFAULT_EMBEDDING_MODEL = "text-embedding-3-large"
@@ -50,7 +54,11 @@ def embedding_config_from_env() -> Dict:
 
 
 def init_llmhub():
-    from llama_index.llms.openai_like import OpenAILike
+    try:
+        from llama_index.llms.openai_like import OpenAILike
+    except ImportError:
+        logger.error("Failed to import OpenAILike. Make sure llama_index is installed.")
+        raise
 
     llm_configs = llm_config_from_env()
     embedding_configs = embedding_config_from_env()

templates/components/settings/python/settings.py

@@ -33,8 +33,13 @@ def init_settings():
 
 
 def init_ollama():
-    from llama_index.embeddings.ollama import OllamaEmbedding
-    from llama_index.llms.ollama.base import DEFAULT_REQUEST_TIMEOUT, Ollama
+    try:
+        from llama_index.embeddings.ollama import OllamaEmbedding
+        from llama_index.llms.ollama.base import DEFAULT_REQUEST_TIMEOUT, Ollama
+    except ImportError:
+        raise ImportError(
+            "Ollama support is not installed. Please install it with `poetry add llama-index-llms-ollama` and `poetry add llama-index-embeddings-ollama`"
+        )
 
     base_url = os.getenv("OLLAMA_BASE_URL") or "http://127.0.0.1:11434"
     request_timeout = float(
@@ -55,25 +60,29 @@ def init_openai():
     from llama_index.llms.openai import OpenAI
 
     max_tokens = os.getenv("LLM_MAX_TOKENS")
-    config = {
-        "model": os.getenv("MODEL"),
-        "temperature": float(os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)),
-        "max_tokens": int(max_tokens) if max_tokens is not None else None,
-    }
-    Settings.llm = OpenAI(**config)
+    Settings.llm = OpenAI(
+        model=os.getenv("MODEL", "gpt-4o-mini"),
+        temperature=float(os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)),
+        max_tokens=int(max_tokens) if max_tokens is not None else None,
+    )
 
     dimensions = os.getenv("EMBEDDING_DIM")
-    config = {
-        "model": os.getenv("EMBEDDING_MODEL"),
-        "dimensions": int(dimensions) if dimensions is not None else None,
-    }
-    Settings.embed_model = OpenAIEmbedding(**config)
+    Settings.embed_model = OpenAIEmbedding(
+        model=os.getenv("EMBEDDING_MODEL", "text-embedding-3-small"),
+        dimensions=int(dimensions) if dimensions is not None else None,
+    )
 
 
 def init_azure_openai():
     from llama_index.core.constants import DEFAULT_TEMPERATURE
-    from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding
-    from llama_index.llms.azure_openai import AzureOpenAI
+
+    try:
+        from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding
+        from llama_index.llms.azure_openai import AzureOpenAI
+    except ImportError:
+        raise ImportError(
+            "Azure OpenAI support is not installed. Please install it with `poetry add llama-index-llms-azure-openai` and `poetry add llama-index-embeddings-azure-openai`"
+        )
 
     llm_deployment = os.environ["AZURE_OPENAI_LLM_DEPLOYMENT"]
     embedding_deployment = os.environ["AZURE_OPENAI_EMBEDDING_DEPLOYMENT"]
@@ -105,26 +114,37 @@ def init_azure_openai():
 
 
 def init_fastembed():
-    """
-    Use Qdrant Fastembed as the local embedding provider.
-    """
-    from llama_index.embeddings.fastembed import FastEmbedEmbedding
+    try:
+        from llama_index.embeddings.fastembed import FastEmbedEmbedding
+    except ImportError:
+        raise ImportError(
+            "FastEmbed support is not installed. Please install it with `poetry add llama-index-embeddings-fastembed`"
+        )
 
     embed_model_map: Dict[str, str] = {
         # Small and multilingual
         "all-MiniLM-L6-v2": "sentence-transformers/all-MiniLM-L6-v2",
         # Large and multilingual
-        "paraphrase-multilingual-mpnet-base-v2": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2",  # noqa: E501
+        "paraphrase-multilingual-mpnet-base-v2": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2",
     }
 
+    embedding_model = os.getenv("EMBEDDING_MODEL")
+    if embedding_model is None:
+        raise ValueError("EMBEDDING_MODEL environment variable is not set")
+
     # This will download the model automatically if it is not already downloaded
     Settings.embed_model = FastEmbedEmbedding(
-        model_name=embed_model_map[os.getenv("EMBEDDING_MODEL")]
+        model_name=embed_model_map[embedding_model]
     )
 
 
 def init_groq():
-    from llama_index.llms.groq import Groq
+    try:
+        from llama_index.llms.groq import Groq
+    except ImportError:
+        raise ImportError(
+            "Groq support is not installed. Please install it with `poetry add llama-index-llms-groq`"
+        )
 
     Settings.llm = Groq(model=os.getenv("MODEL"))
     # Groq does not provide embeddings, so we use FastEmbed instead
@@ -132,7 +152,12 @@ def init_groq():
 
 
 def init_anthropic():
-    from llama_index.llms.anthropic import Anthropic
+    try:
+        from llama_index.llms.anthropic import Anthropic
+    except ImportError:
+        raise ImportError(
+            "Anthropic support is not installed. Please install it with `poetry add llama-index-llms-anthropic`"
+        )
 
     model_map: Dict[str, str] = {
         "claude-3-opus": "claude-3-opus-20240229",
@@ -148,8 +173,13 @@ def init_anthropic():
 
 
 def init_gemini():
-    from llama_index.embeddings.gemini import GeminiEmbedding
-    from llama_index.llms.gemini import Gemini
+    try:
+        from llama_index.embeddings.gemini import GeminiEmbedding
+        from llama_index.llms.gemini import Gemini
+    except ImportError:
+        raise ImportError(
+            "Gemini support is not installed. Please install it with `poetry add llama-index-llms-gemini` and `poetry add llama-index-embeddings-gemini`"
+        )
 
     model_name = f"models/{os.getenv('MODEL')}"
     embed_model_name = f"models/{os.getenv('EMBEDDING_MODEL')}"

templates/components/vectordbs/python/astra/vectordb.py

@@ -15,6 +15,6 @@ def get_vector_store():
         token=token,
         api_endpoint=endpoint,
         collection_name=collection,
-        embedding_dimension=int(os.getenv("EMBEDDING_DIM")),
+        embedding_dimension=int(os.getenv("EMBEDDING_DIM", 768)),
     )
     return store

templates/components/vectordbs/python/chroma/vectordb.py

@@ -1,4 +1,5 @@
 import os
+
 from llama_index.vector_stores.chroma import ChromaVectorStore
 
 
@@ -18,7 +19,7 @@ def get_vector_store():
             )
         store = ChromaVectorStore.from_params(
             host=os.getenv("CHROMA_HOST"),
-            port=int(os.getenv("CHROMA_PORT")),
+            port=os.getenv("CHROMA_PORT", "8001"),
             collection_name=collection_name,
         )
     return store

templates/components/vectordbs/python/llamacloud/generate.py

@@ -1,22 +1,66 @@
 # flake8: noqa: E402
-from dotenv import load_dotenv
+import os
 
-from app.engine.index import get_index
+from dotenv import load_dotenv
 
 load_dotenv()
 
 import logging
-from llama_index.core.readers import SimpleDirectoryReader
+
+from app.engine.index import get_client, get_index
 from app.engine.service import LLamaCloudFileService
+from app.settings import init_settings
+from llama_cloud import PipelineType
+from llama_index.core.readers import SimpleDirectoryReader
+from llama_index.core.settings import Settings
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger()
 
 
+def ensure_index(index):
+    project_id = index._get_project_id()
+    client = get_client()
+    pipelines = client.pipelines.search_pipelines(
+        project_id=project_id,
+        pipeline_name=index.name,
+        pipeline_type=PipelineType.MANAGED.value,
+    )
+    if len(pipelines) == 0:
+        from llama_index.embeddings.openai import OpenAIEmbedding
+
+        if not isinstance(Settings.embed_model, OpenAIEmbedding):
+            raise ValueError(
+                "Creating a new pipeline with a non-OpenAI embedding model is not supported."
+            )
+        client.pipelines.upsert_pipeline(
+            project_id=project_id,
+            request={
+                "name": index.name,
+                "embedding_config": {
+                    "type": "OPENAI_EMBEDDING",
+                    "component": {
+                        "api_key": os.getenv("OPENAI_API_KEY"),  # editable
+                        "model_name": os.getenv("EMBEDDING_MODEL"),
+                    },
+                },
+                "transform_config": {
+                    "mode": "auto",
+                    "config": {
+                        "chunk_size": Settings.chunk_size,  # editable
+                        "chunk_overlap": Settings.chunk_overlap,  # editable
+                    },
+                },
+            },
+        )
+
+
 def generate_datasource():
+    init_settings()
     logger.info("Generate index for the provided data")
 
     index = get_index()
+    ensure_index(index)
     project_id = index._get_project_id()
     pipeline_id = index._get_pipeline_id()
 
@@ -34,13 +78,7 @@ def generate_datasource():
                 f"Adding file {input_file} to pipeline {index.name} in project {index.project_name}"
             )
             LLamaCloudFileService.add_file_to_pipeline(
-                project_id,
-                pipeline_id,
-                f,
-                custom_metadata={
-                    # Set private=false to mark the document as public (required for filtering)
-                    "private": "false",
-                },
+                project_id, pipeline_id, f, custom_metadata={}
             )
 
     logger.info("Finished generating the index")

templates/components/vectordbs/python/llamacloud/index.py

@@ -7,7 +7,7 @@ from llama_index.core.ingestion.api_utils import (
     get_client as llama_cloud_get_client,
 )
 from llama_index.indices.managed.llama_cloud import LlamaCloudIndex
-from pydantic import BaseModel, Field, validator
+from pydantic import BaseModel, Field, field_validator
 
 logger = logging.getLogger("uvicorn")
 
@@ -15,31 +15,39 @@ logger = logging.getLogger("uvicorn")
 class LlamaCloudConfig(BaseModel):
     # Private attributes
     api_key: str = Field(
-        default=os.getenv("LLAMA_CLOUD_API_KEY"),
         exclude=True,  # Exclude from the model representation
     )
     base_url: Optional[str] = Field(
-        default=os.getenv("LLAMA_CLOUD_BASE_URL"),
         exclude=True,
     )
     organization_id: Optional[str] = Field(
-        default=os.getenv("LLAMA_CLOUD_ORGANIZATION_ID"),
         exclude=True,
     )
     # Configuration attributes, can be set by the user
     pipeline: str = Field(
         description="The name of the pipeline to use",
-        default=os.getenv("LLAMA_CLOUD_INDEX_NAME"),
     )
     project: str = Field(
         description="The name of the LlamaCloud project",
-        default=os.getenv("LLAMA_CLOUD_PROJECT_NAME"),
     )
 
+    def __init__(self, **kwargs):
+        if "api_key" not in kwargs:
+            kwargs["api_key"] = os.getenv("LLAMA_CLOUD_API_KEY")
+        if "base_url" not in kwargs:
+            kwargs["base_url"] = os.getenv("LLAMA_CLOUD_BASE_URL")
+        if "organization_id" not in kwargs:
+            kwargs["organization_id"] = os.getenv("LLAMA_CLOUD_ORGANIZATION_ID")
+        if "pipeline" not in kwargs:
+            kwargs["pipeline"] = os.getenv("LLAMA_CLOUD_INDEX_NAME")
+        if "project" not in kwargs:
+            kwargs["project"] = os.getenv("LLAMA_CLOUD_PROJECT_NAME")
+        super().__init__(**kwargs)
+
     # Validate and throw error if the env variables are not set before starting the app
-    @validator("pipeline", "project", "api_key", pre=True, always=True)
+    @field_validator("pipeline", "project", "api_key", mode="before")
     @classmethod
-    def validate_env_vars(cls, value):
+    def validate_fields(cls, value):
         if value is None:
             raise ValueError(
                 "Please set LLAMA_CLOUD_INDEX_NAME, LLAMA_CLOUD_PROJECT_NAME and LLAMA_CLOUD_API_KEY"
@@ -56,7 +64,7 @@ class LlamaCloudConfig(BaseModel):
 
 class IndexConfig(BaseModel):
     llama_cloud_pipeline_config: LlamaCloudConfig = Field(
-        default=LlamaCloudConfig(),
+        default_factory=lambda: LlamaCloudConfig(),
         alias="llamaCloudPipeline",
     )
     callback_manager: Optional[CallbackManager] = Field(

templates/components/vectordbs/python/llamacloud/query_filter.py

@@ -5,7 +5,7 @@ def generate_filters(doc_ids):
     """
     Generate public/private document filters based on the doc_ids and the vector store.
     """
-    # Using "is_empty" filter to include the documents don't have the "private" key because they're uploaded in LlamaCloud UI
+    # public documents (ingested by "poetry run generate" or in the LlamaCloud UI) don't have the "private" field
     public_doc_filter = MetadataFilter(
         key="private",
         value=None,

templates/components/vectordbs/python/milvus/vectordb.py

@@ -1,4 +1,5 @@
 import os
+
 from llama_index.vector_stores.milvus import MilvusVectorStore
 
 
@@ -15,6 +16,6 @@ def get_vector_store():
         user=os.getenv("MILVUS_USERNAME"),
         password=os.getenv("MILVUS_PASSWORD"),
         collection_name=collection,
-        dim=int(os.getenv("EMBEDDING_DIM")),
+        dim=int(os.getenv("EMBEDDING_DIM", 768)),
     )
     return store

templates/components/vectordbs/python/none/index.py

@@ -3,7 +3,7 @@ import os
 from datetime import timedelta
 from typing import Optional
 
-from cachetools import TTLCache, cached
+from cachetools import TTLCache, cached  # type: ignore
 from llama_index.core.callbacks import CallbackManager
 from llama_index.core.indices import load_index_from_storage
 from llama_index.core.storage import StorageContext

templates/components/vectordbs/typescript/llamacloud/generate.ts

@@ -25,6 +25,8 @@ async function* walk(dir: string): AsyncGenerator<string> {
 
 async function loadAndIndex() {
   const index = await getDataSource();
+  // ensure the index is available or create a new one
+  await index.ensureIndex();
   const projectId = await index.getProjectId();
   const pipelineId = await index.getPipelineId();
 
@@ -32,10 +34,23 @@ async function loadAndIndex() {
   for await (const filePath of walk(DATA_DIR)) {
     const buffer = await fs.readFile(filePath);
     const filename = path.basename(filePath);
-    const file = new File([buffer], filename);
-    await LLamaCloudFileService.addFileToPipeline(projectId, pipelineId, file, {
-      private: "false",
-    });
+    try {
+      await LLamaCloudFileService.addFileToPipeline(
+        projectId,
+        pipelineId,
+        new File([buffer], filename),
+      );
+    } catch (error) {
+      if (
+        error instanceof ReferenceError &&
+        error.message.includes("File is not defined")
+      ) {
+        throw new Error(
+          "File class is not supported in the current Node.js version. Please use Node.js 20 or higher.",
+        );
+      }
+      throw error;
+    }
   }
 
   console.log(`Successfully uploaded documents to LlamaCloud!`);

templates/components/vectordbs/typescript/llamacloud/queryFilter.ts

@@ -1,14 +1,17 @@
-import { MetadataFilter, MetadataFilters } from "llamaindex";
+import { CloudRetrieveParams, MetadataFilter } from "llamaindex";
 
-export function generateFilters(documentIds: string[]): MetadataFilters {
-  // public documents don't have the "private" field or it's set to "false"
+export function generateFilters(documentIds: string[]) {
+  // public documents (ingested by "npm run generate" or in the LlamaCloud UI) don't have the "private" field
   const publicDocumentsFilter: MetadataFilter = {
     key: "private",
     operator: "is_empty",
   };
 
   // if no documentIds are provided, only retrieve information from public documents
-  if (!documentIds.length) return { filters: [publicDocumentsFilter] };
+  if (!documentIds.length)
+    return {
+      filters: [publicDocumentsFilter],
+    } as CloudRetrieveParams["filters"];
 
   const privateDocumentsFilter: MetadataFilter = {
     key: "file_id", // Note: LLamaCloud uses "file_id" to reference private document ids as "doc_id" is a restricted field in LlamaCloud
@@ -20,5 +23,5 @@ export function generateFilters(documentIds: string[]): MetadataFilters {
   return {
     filters: [publicDocumentsFilter, privateDocumentsFilter],
     condition: "or",
-  };
+  } as CloudRetrieveParams["filters"];
 }

templates/components/vectordbs/typescript/pg/generate.ts

@@ -18,7 +18,9 @@ async function loadAndIndex() {
 
   // create postgres vector store
   const vectorStore = new PGVectorStore({
-    connectionString: process.env.PG_CONNECTION_STRING,
+    clientConfig: {
+      connectionString: process.env.PG_CONNECTION_STRING,
+    },
     schemaName: PGVECTOR_SCHEMA,
     tableName: PGVECTOR_TABLE,
   });

templates/components/vectordbs/typescript/pg/index.ts

@@ -9,7 +9,9 @@ import {
 export async function getDataSource(params?: any) {
   checkRequiredEnvVars();
   const pgvs = new PGVectorStore({
-    connectionString: process.env.PG_CONNECTION_STRING,
+    clientConfig: {
+      connectionString: process.env.PG_CONNECTION_STRING,
+    },
     schemaName: PGVECTOR_SCHEMA,
     tableName: PGVECTOR_TABLE,
   });

templates/types/extractor/fastapi/app/engine/generate.py

@@ -6,7 +6,7 @@ load_dotenv()
 import logging
 import os
 
-from llama_index.core.ingestion import IngestionPipeline
+from llama_index.core.ingestion import DocstoreStrategy, IngestionPipeline
 from llama_index.core.node_parser import SentenceSplitter
 from llama_index.core.settings import Settings
 from llama_index.core.storage import StorageContext
@@ -41,7 +41,7 @@ def run_pipeline(docstore, vector_store, documents):
             Settings.embed_model,
         ],
         docstore=docstore,
-        docstore_strategy="upserts_and_delete",
+        docstore_strategy=DocstoreStrategy.UPSERTS_AND_DELETE,  # type: ignore
         vector_store=vector_store,
     )
 

templates/types/extractor/fastapi/app/engine/index.py

@@ -16,7 +16,7 @@ class IndexConfig(BaseModel):
     )
 
 
-def get_index(config: IndexConfig = None):
+def get_index(config: Optional[IndexConfig] = None) -> VectorStoreIndex:
     if config is None:
         config = IndexConfig()
     logger.info("Connecting vector store...")

templates/types/streaming/express/package.json

@@ -21,7 +21,7 @@
     "dotenv": "^16.3.1",
     "duck-duck-scrape": "^2.2.5",
     "express": "^4.18.2",
-    "llamaindex": "0.6.2",
+    "llamaindex": "0.6.19",
     "pdf2json": "3.0.5",
     "ajv": "^8.12.0",
     "@e2b/code-interpreter": "0.0.9-beta.3",

templates/types/streaming/fastapi/app/api/routers/chat.py

@@ -13,7 +13,7 @@ from app.api.routers.models import (
     SourceNodes,
 )
 from app.api.routers.vercel_response import VercelStreamResponse
-from app.engine import get_chat_engine
+from app.engine.engine import get_chat_engine
 from app.engine.query_filter import generate_filters
 
 chat_router = r = APIRouter()

templates/types/streaming/fastapi/app/api/routers/chat_config.py

@@ -1,11 +1,10 @@
 import logging
 import os
 
-from fastapi import APIRouter
+from fastapi import APIRouter, HTTPException
 
 from app.api.routers.models import ChatConfig
 
-
 config_router = r = APIRouter()
 
 logger = logging.getLogger("uvicorn")
@@ -23,10 +22,14 @@ async def chat_config() -> ChatConfig:
 try:
     from app.engine.service import LLamaCloudFileService
 
-    logger.info("LlamaCloud is configured. Adding /config/llamacloud route.")
+    print("LlamaCloud is configured. Adding /config/llamacloud route.")
 
     @r.get("/llamacloud")
     async def chat_llama_cloud_config():
+        if not os.getenv("LLAMA_CLOUD_API_KEY"):
+            raise HTTPException(
+                status_code=500, detail="LlamaCloud API KEY is not configured"
+            )
         projects = LLamaCloudFileService.get_all_projects_with_pipelines()
         pipeline = os.getenv("LLAMA_CLOUD_INDEX_NAME")
         project = os.getenv("LLAMA_CLOUD_PROJECT_NAME")
@@ -42,7 +45,5 @@ try:
         }
 
 except ImportError:
-    logger.debug(
-        "LlamaCloud is not configured. Skipping adding /config/llamacloud route."
-    )
+    print("LlamaCloud is not configured. Skipping adding /config/llamacloud route.")
     pass

templates/types/streaming/fastapi/app/api/routers/events.py

@@ -1,13 +1,13 @@
-import json
 import asyncio
+import json
 import logging
-from typing import AsyncGenerator, Dict, Any, List, Optional
+from typing import Any, AsyncGenerator, Dict, List, Optional
+
 from llama_index.core.callbacks.base import BaseCallbackHandler
 from llama_index.core.callbacks.schema import CBEventType
 from llama_index.core.tools.types import ToolOutput
 from pydantic import BaseModel
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -31,15 +31,20 @@ class CallbackEvent(BaseModel):
             return None
 
     def get_tool_message(self) -> dict | None:
+        if self.payload is None:
+            return None
         func_call_args = self.payload.get("function_call")
         if func_call_args is not None and "tool" in self.payload:
             tool = self.payload.get("tool")
+            if tool is None:
+                return None
             return {
                 "type": "events",
                 "data": {
                     "title": f"Calling tool: {tool.name} with inputs: {func_call_args}",
                 },
             }
+        return None
 
     def _is_output_serializable(self, output: Any) -> bool:
         try:
@@ -49,6 +54,8 @@ class CallbackEvent(BaseModel):
             return False
 
     def get_agent_tool_response(self) -> dict | None:
+        if self.payload is None:
+            return None
         response = self.payload.get("response")
         if response is not None:
             sources = response.sources
@@ -74,6 +81,7 @@ class CallbackEvent(BaseModel):
                             },
                         },
                     }
+        return None
 
     def to_response(self):
         try:
@@ -114,11 +122,13 @@ class EventCallbackHandler(BaseCallbackHandler):
         event_type: CBEventType,
         payload: Optional[Dict[str, Any]] = None,
         event_id: str = "",
+        parent_id: str = "",
         **kwargs: Any,
     ) -> str:
         event = CallbackEvent(event_id=event_id, event_type=event_type, payload=payload)
         if event.to_response() is not None:
             self._aqueue.put_nowait(event)
+        return event_id
 
     def on_event_end(
         self,

templates/types/streaming/fastapi/app/api/routers/models.py

@@ -1,6 +1,6 @@
 import logging
 import os
-from typing import Any, Dict, List, Literal, Optional
+from typing import Any, Dict, List, Literal, Optional, Union
 
 from llama_index.core.llms import ChatMessage, MessageRole
 from llama_index.core.schema import NodeWithScore
@@ -62,15 +62,23 @@ class ArtifactAnnotation(BaseModel):
 
 class Annotation(BaseModel):
     type: str
-    data: AnnotationFileData | List[str] | AgentAnnotation | ArtifactAnnotation
+    data: Union[AnnotationFileData, List[str], AgentAnnotation, ArtifactAnnotation]
 
-    def to_content(self) -> str | None:
+    def to_content(self) -> Optional[str]:
         if self.type == "document_file":
-            # We only support generating context content for CSV files for now
-            csv_files = [file for file in self.data.files if file.filetype == "csv"]
-            if len(csv_files) > 0:
-                return "Use data from following CSV raw content\n" + "\n".join(
-                    [f"```csv\n{csv_file.content.value}\n```" for csv_file in csv_files]
+            if isinstance(self.data, AnnotationFileData):
+                # We only support generating context content for CSV files for now
+                csv_files = [file for file in self.data.files if file.filetype == "csv"]
+                if len(csv_files) > 0:
+                    return "Use data from following CSV raw content\n" + "\n".join(
+                        [
+                            f"```csv\n{csv_file.content.value}\n```"
+                            for csv_file in csv_files
+                        ]
+                    )
+            else:
+                logger.warning(
+                    f"Unexpected data type for document_file annotation: {type(self.data)}"
                 )
         else:
             logger.warning(
@@ -213,6 +221,7 @@ class ChatData(BaseModel):
                 for annotation in message.annotations:
                     if (
                         annotation.type == "document_file"
+                        and isinstance(annotation.data, AnnotationFileData)
                         and annotation.data.files is not None
                     ):
                         for fi in annotation.data.files:
@@ -242,7 +251,7 @@ class SourceNodes(BaseModel):
         )
 
     @classmethod
-    def get_url_from_metadata(cls, metadata: Dict[str, Any]) -> str:
+    def get_url_from_metadata(cls, metadata: Dict[str, Any]) -> Optional[str]:
         url_prefix = os.getenv("FILESERVER_URL_PREFIX")
         if not url_prefix:
             logger.warning(

templates/types/streaming/fastapi/app/engine/__init__.py

@@ -1 +0,0 @@
-from .engine import get_chat_engine as get_chat_engine

templates/types/streaming/fastapi/app/engine/generate.py

@@ -6,15 +6,16 @@ load_dotenv()
 import logging
 import os
 
-from app.engine.loaders import get_documents
-from app.engine.vectordb import get_vector_store
-from app.settings import init_settings
-from llama_index.core.ingestion import IngestionPipeline
+from llama_index.core.ingestion import DocstoreStrategy, IngestionPipeline
 from llama_index.core.node_parser import SentenceSplitter
 from llama_index.core.settings import Settings
 from llama_index.core.storage import StorageContext
 from llama_index.core.storage.docstore import SimpleDocumentStore
 
+from app.engine.loaders import get_documents
+from app.engine.vectordb import get_vector_store
+from app.settings import init_settings
+
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger()
 
@@ -40,7 +41,7 @@ def run_pipeline(docstore, vector_store, documents):
             Settings.embed_model,
         ],
         docstore=docstore,
-        docstore_strategy="upserts_and_delete",
+        docstore_strategy=DocstoreStrategy.UPSERTS_AND_DELETE,  # type: ignore
         vector_store=vector_store,
     )
 

templates/types/streaming/fastapi/pyproject.toml

@@ -15,8 +15,25 @@ uvicorn = { extras = ["standard"], version = "^0.23.2" }
 python-dotenv = "^1.0.0"
 aiostream = "^0.5.2"
 cachetools = "^5.3.3"
-llama-index = "0.11.6"
+llama-index = "^0.11.17"
+
+[tool.poetry.group.dev.dependencies]
+mypy = "^1.8.0"
 
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
+
+[tool.mypy]
+python_version = "3.11"
+plugins = "pydantic.mypy"
+exclude = [ "tests", "venv", ".venv", "output", "config" ]
+check_untyped_defs = true
+warn_unused_ignores = false
+show_error_codes = true
+namespace_packages = true
+ignore_missing_imports = true
+follow_imports = "silent"
+implicit_optional = true
+strict_optional = false
+disable_error_code = ["return-value", "import-untyped", "assignment"]

templates/types/streaming/nextjs/app/components/ui/chat/widgets/LlamaCloudSelector.tsx

@@ -45,7 +45,7 @@ export function LlamaCloudSelector({
   setRequestData,
   onSelect,
   defaultPipeline,
-  shouldCheckValid = true,
+  shouldCheckValid = false,
 }: LlamaCloudSelectorProps) {
   const { backend } = useClientConfig();
   const [config, setConfig] = useState<LlamaCloudConfig>();
@@ -66,7 +66,16 @@ export function LlamaCloudSelector({
   useEffect(() => {
     if (process.env.NEXT_PUBLIC_USE_LLAMACLOUD === "true" && !config) {
       fetch(`${backend}/api/chat/config/llamacloud`)
-        .then((response) => response.json())
+        .then((response) => {
+          if (!response.ok) {
+            return response.json().then((errorData) => {
+              window.alert(
+                `Error: ${JSON.stringify(errorData) || "Unknown error occurred"}`,
+              );
+            });
+          }
+          return response.json();
+        })
         .then((data) => {
           const pipeline = defaultPipeline ?? data.pipeline; // defaultPipeline will override pipeline in .env
           setConfig({ ...data, pipeline });
@@ -95,7 +104,8 @@ export function LlamaCloudSelector({
       </div>
     );
   }
-  if (!isValid(config) && shouldCheckValid) {
+
+  if (shouldCheckValid && !isValid(config.projects, config.pipeline)) {
     return (
       <p className="text-red-500">
         Invalid LlamaCloud configuration. Check console logs.
@@ -107,7 +117,11 @@ export function LlamaCloudSelector({
   return (
     <Select
       onValueChange={handlePipelineSelect}
-      defaultValue={JSON.stringify(pipeline)}
+      defaultValue={
+        isValid(projects, pipeline, false)
+          ? JSON.stringify(pipeline)
+          : undefined
+      }
     >
       <SelectTrigger className="w-[200px]">
         <SelectValue placeholder="Select a pipeline" />
@@ -137,26 +151,33 @@ export function LlamaCloudSelector({
   );
 }
 
-function isValid(config: LlamaCloudConfig): boolean {
-  const { projects, pipeline } = config;
+function isValid(
+  projects: LLamaCloudProject[] | undefined,
+  pipeline: PipelineConfig | undefined,
+  logErrors: boolean = true,
+): boolean {
   if (!projects?.length) return false;
   if (!pipeline) return false;
   const matchedProject = projects.find(
     (project: LLamaCloudProject) => project.name === pipeline.project,
   );
   if (!matchedProject) {
-    console.error(
-      `LlamaCloud project ${pipeline.project} not found. Check LLAMA_CLOUD_PROJECT_NAME variable`,
-    );
+    if (logErrors) {
+      console.error(
+        `LlamaCloud project ${pipeline.project} not found. Check LLAMA_CLOUD_PROJECT_NAME variable`,
+      );
+    }
     return false;
   }
   const pipelineExists = matchedProject.pipelines.some(
     (p) => p.name === pipeline.pipeline,
   );
   if (!pipelineExists) {
-    console.error(
-      `LlamaCloud pipeline ${pipeline.pipeline} not found. Check LLAMA_CLOUD_INDEX_NAME variable`,
-    );
+    if (logErrors) {
+      console.error(
+        `LlamaCloud pipeline ${pipeline.pipeline} not found. Check LLAMA_CLOUD_INDEX_NAME variable`,
+      );
+    }
     return false;
   }
   return true;

templates/types/streaming/nextjs/package.json

@@ -27,7 +27,7 @@
     "duck-duck-scrape": "^2.2.5",
     "formdata-node": "^6.0.3",
     "got": "^14.4.1",
-    "llamaindex": "0.6.2",
+    "llamaindex": "0.6.19",
     "lucide-react": "^0.294.0",
     "next": "^14.2.4",
     "react": "^18.2.0",