Update questions.ts

.changeset/yellow-jokes-protect.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Add multi agents template for Express

helpers/typescript.ts

@@ -33,7 +33,11 @@ export const installTSTemplate = async ({
    * Copy the template files to the target directory.
    */
   console.log("\nInitializing project with template:", template, "\n");
-  const type = template === "multiagent" ? "streaming" : template; // use nextjs streaming template for multiagent
+  let type = "streaming";
+  if (template === "multiagent" && framework === "express") {
+    // use nextjs streaming template as frontend for express and fastapi
+    type = "multiagent";
+  }
   const templatePath = path.join(templatesDir, "types", type, framework);
   const copySource = ["**"];
 
@@ -124,6 +128,24 @@ export const installTSTemplate = async ({
     cwd: path.join(compPath, "vectordbs", "typescript", vectorDb ?? "none"),
   });
 
+  if (template === "multiagent") {
+    const multiagentPath = path.join(compPath, "multiagent", "typescript");
+
+    // copy workflow code for multiagent template
+    await copy("**", path.join(root, relativeEngineDestPath, "workflow"), {
+      parents: true,
+      cwd: path.join(multiagentPath, "workflow"),
+    });
+
+    if (framework === "nextjs") {
+      // patch route.ts file
+      await copy("**", path.join(root, relativeEngineDestPath), {
+        parents: true,
+        cwd: path.join(multiagentPath, "nextjs"),
+      });
+    }
+  }
+
   // copy loader component (TS only supports llama_parse and file for now)
   const loaderFolder = useLlamaParse ? "llama_parse" : "file";
   await copy("**", enginePath, {
@@ -145,6 +167,11 @@ export const installTSTemplate = async ({
     cwd: path.join(compPath, "engines", "typescript", engine),
   });
 
+  // copy settings to engine folder
+  await copy("**", enginePath, {
+    cwd: path.join(compPath, "settings", "typescript"),
+  });
+
   /**
    * Copy the selected UI files to the target directory and reference it.
    */

questions.ts

@@ -410,10 +410,7 @@ export const askQuestions = async (
     return; // early return - no further questions needed for llamapack projects
   }
 
-  if (program.template === "multiagent") {
-    // TODO: multi-agents currently only supports FastAPI
-    program.framework = preferences.framework = "fastapi";
-  } else if (program.template === "extractor") {
+  if (program.template === "extractor") {
     // Extractor template only supports FastAPI, empty data sources, and llamacloud
     // So we just use example file for extractor template, this allows user to choose vector database later
     program.dataSources = [EXAMPLE_FILE];
@@ -424,7 +421,7 @@ export const askQuestions = async (
       program.framework = getPrefOrDefault("framework");
     } else {
       const choices = [
-        { title: "NextJS", value: "nextjs" },
+        { title: "NextJS", value: "nextjs" }, 
         { title: "Express", value: "express" },
         { title: "FastAPI (Python)", value: "fastapi" },
       ];

templates/components/multiagent/typescript/nextjs/route.ts

@@ -0,0 +1,53 @@
+import { initObservability } from "@/app/observability";
+import { Message, StreamData, StreamingTextResponse } from "ai";
+import { ChatMessage } from "llamaindex";
+import { NextRequest, NextResponse } from "next/server";
+import { initSettings } from "./engine/settings";
+import { createStreamTimeout } from "./llamaindex/streaming/events";
+import { createWorkflow } from "./workflow";
+import { toDataStream } from "./workflow/stream";
+
+initObservability();
+initSettings();
+
+export const runtime = "nodejs";
+export const dynamic = "force-dynamic";
+
+export async function POST(request: NextRequest) {
+  // Init Vercel AI StreamData and timeout
+  const vercelStreamData = new StreamData();
+  const streamTimeout = createStreamTimeout(vercelStreamData);
+
+  try {
+    const body = await request.json();
+    const { messages, data }: { messages: Message[]; data?: any } = body;
+    const userMessage = messages.pop();
+    if (!messages || !userMessage || userMessage.role !== "user") {
+      return NextResponse.json(
+        {
+          error:
+            "messages are required in the request body and the last message must be from the user",
+        },
+        { status: 400 },
+      );
+    }
+
+    const chatHistory = messages as ChatMessage[];
+    const agent = await createWorkflow(chatHistory, vercelStreamData);
+    agent.run(userMessage.content);
+    const stream = toDataStream(agent.streamEvents(), vercelStreamData);
+    return new StreamingTextResponse(stream, {}, vercelStreamData);
+  } catch (error) {
+    console.error("[LlamaIndex]", error);
+    return NextResponse.json(
+      {
+        detail: (error as Error).message,
+      },
+      {
+        status: 500,
+      },
+    );
+  } finally {
+    clearTimeout(streamTimeout);
+  }
+}

templates/components/multiagent/typescript/workflow/agents.ts

@@ -0,0 +1,62 @@
+import { StreamData } from "ai";
+import { ChatMessage, QueryEngineTool } from "llamaindex";
+import { getDataSource } from "../engine";
+import { FunctionCallingAgent } from "./factory";
+
+const getQueryEngineTool = async () => {
+  const index = await getDataSource();
+  if (!index) {
+    throw new Error("Index not found. Please create an index first.");
+  }
+
+  const topK = process.env.TOP_K ? parseInt(process.env.TOP_K) : undefined;
+  return new QueryEngineTool({
+    queryEngine: index.asQueryEngine({
+      similarityTopK: topK,
+    }),
+    metadata: {
+      name: "query_index",
+      description: `Use this tool to retrieve information about the text corpus from the index.`,
+    },
+  });
+};
+
+export const createResearcher = async (
+  chatHistory: ChatMessage[],
+  stream: StreamData,
+) => {
+  return new FunctionCallingAgent({
+    name: "researcher",
+    tools: [await getQueryEngineTool()],
+    systemPrompt:
+      "You are a researcher agent. You are given a researching task. You must use your tools to complete the research.",
+    chatHistory,
+    stream,
+  });
+};
+
+export const createWriter = (
+  chatHistory: ChatMessage[],
+  stream: StreamData,
+) => {
+  return new FunctionCallingAgent({
+    name: "writer",
+    systemPrompt:
+      "You are an expert in writing blog posts. You are given a task to write a blog post. Don't make up any information yourself.",
+    chatHistory,
+    stream,
+  });
+};
+
+export const createReviewer = (
+  chatHistory: ChatMessage[],
+  stream: StreamData,
+) => {
+  return new FunctionCallingAgent({
+    name: "reviewer",
+    systemPrompt:
+      "You are an expert in reviewing blog posts. You are given a task to review a blog post. Review the post for logical inconsistencies, ask critical questions, and provide suggestions for improvement. Furthermore, proofread the post for grammar and spelling errors. Only if the post is good enough for publishing, then you MUST return 'The post is good.'. In all other cases return your review.",
+    chatHistory,
+    stream,
+  });
+};

templates/components/multiagent/typescript/workflow/factory.ts

@@ -0,0 +1,152 @@
+import {
+  Context,
+  StartEvent,
+  StopEvent,
+  Workflow,
+  WorkflowEvent,
+} from "@llamaindex/core/workflow";
+import { StreamData } from "ai";
+import {
+  BaseToolWithCall,
+  CallbackManager,
+  ChatMemoryBuffer,
+  ChatMessage,
+  EngineResponse,
+  LLM,
+  OpenAIAgent,
+  Settings,
+} from "llamaindex";
+import { AgentInput, FunctionCallingStreamResult } from "./type";
+
+class InputEvent extends WorkflowEvent<{
+  input: ChatMessage[];
+}> {}
+
+export class FunctionCallingAgent extends Workflow {
+  name: string;
+  llm: LLM;
+  memory: ChatMemoryBuffer;
+  tools: BaseToolWithCall[];
+  systemPrompt?: string;
+  writeEvents: boolean;
+  role?: string;
+  callbackManager: CallbackManager;
+  stream: StreamData;
+
+  constructor(options: {
+    name: string;
+    llm?: LLM;
+    chatHistory?: ChatMessage[];
+    tools?: BaseToolWithCall[];
+    systemPrompt?: string;
+    writeEvents?: boolean;
+    role?: string;
+    verbose?: boolean;
+    timeout?: number;
+    stream: StreamData;
+  }) {
+    super({
+      verbose: options?.verbose ?? false,
+      timeout: options?.timeout ?? 360,
+    });
+    this.name = options?.name;
+    this.llm = options.llm ?? Settings.llm;
+    this.memory = new ChatMemoryBuffer({
+      llm: this.llm,
+      chatHistory: options.chatHistory,
+    });
+    this.tools = options?.tools ?? [];
+    this.systemPrompt = options.systemPrompt;
+    this.writeEvents = options?.writeEvents ?? true;
+    this.role = options?.role;
+    this.callbackManager = this.createCallbackManager();
+    this.stream = options.stream;
+
+    // add steps
+    this.addStep(StartEvent<AgentInput>, this.prepareChatHistory, {
+      outputs: InputEvent,
+    });
+    this.addStep(InputEvent, this.handleLLMInput, {
+      outputs: StopEvent,
+    });
+  }
+
+  private get chatHistory() {
+    return this.memory.getAllMessages();
+  }
+
+  private async prepareChatHistory(
+    ctx: Context,
+    ev: StartEvent<AgentInput>,
+  ): Promise<InputEvent> {
+    const { message, streaming } = ev.data.input;
+    ctx.set("streaming", streaming);
+    this.writeEvent(`Start to work on: ${message}`);
+    if (this.systemPrompt) {
+      this.memory.put({ role: "system", content: this.systemPrompt });
+    }
+    this.memory.put({ role: "user", content: message });
+    return new InputEvent({ input: this.chatHistory });
+  }
+
+  private async handleLLMInput(
+    ctx: Context,
+    ev: InputEvent,
+  ): Promise<StopEvent<string | ReadableStream<EngineResponse>>> {
+    const chatEngine = new OpenAIAgent({
+      tools: this.tools,
+      systemPrompt: this.systemPrompt,
+      chatHistory: this.chatHistory,
+    });
+
+    if (!ctx.get("streaming")) {
+      const response = await Settings.withCallbackManager(
+        this.callbackManager,
+        () => {
+          return chatEngine.chat({
+            message: ev.data.input.pop()!.content,
+          });
+        },
+      );
+      this.writeEvent("Finished task");
+      return new StopEvent({ result: response.message.content.toString() });
+    }
+
+    const response = await Settings.withCallbackManager(
+      this.callbackManager,
+      () => {
+        return chatEngine.chat({
+          message: ev.data.input.pop()!.content,
+          stream: true,
+        });
+      },
+    );
+    ctx.writeEventToStream({ data: new FunctionCallingStreamResult(response) });
+    return new StopEvent({ result: response });
+  }
+
+  private createCallbackManager() {
+    const callbackManager = new CallbackManager();
+    callbackManager.on("llm-tool-call", (event) => {
+      const { toolCall } = event.detail;
+      this.writeEvent(
+        `Calling tool "${toolCall.name}" with input: ${JSON.stringify(toolCall.input)}`,
+      );
+    });
+    callbackManager.on("llm-tool-result", (event) => {
+      const { toolCall, toolResult } = event.detail;
+      this.writeEvent(
+        `Getting result from tool "${toolCall.name}": \n${JSON.stringify(toolResult.output)}`,
+      );
+    });
+    return callbackManager;
+  }
+
+  private writeEvent(msg: string) {
+    if (!this.writeEvents) return;
+    this.stream.appendMessageAnnotation({
+      type: "agent",
+      data: { agent: this.name, text: msg },
+    });
+  }
+}

templates/components/multiagent/typescript/workflow/index.ts

@@ -0,0 +1,129 @@
+import {
+  Context,
+  StartEvent,
+  StopEvent,
+  Workflow,
+  WorkflowEvent,
+} from "@llamaindex/core/workflow";
+import { StreamData } from "ai";
+import { ChatMessage } from "llamaindex";
+import { createResearcher, createReviewer, createWriter } from "./agents";
+import { AgentInput, AgentRunResult } from "./type";
+
+const TIMEOUT = 360 * 1000;
+const MAX_ATTEMPTS = 2;
+
+class ResearchEvent extends WorkflowEvent<{ input: string }> {}
+class WriteEvent extends WorkflowEvent<{
+  input: string;
+  isGood: boolean;
+}> {}
+class ReviewEvent extends WorkflowEvent<{ input: string }> {}
+
+export const createWorkflow = async (
+  chatHistory: ChatMessage[],
+  stream: StreamData,
+) => {
+  const appendStream = (agent: string, text: string) => {
+    stream.appendMessageAnnotation({
+      type: "agent",
+      data: { agent, text },
+    });
+  };
+
+  const start = async (context: Context, ev: StartEvent) => {
+    context.set("task", ev.data.input);
+    return new ResearchEvent({
+      input: `Research for this task: ${ev.data.input}`,
+    });
+  };
+
+  const research = async (context: Context, ev: ResearchEvent) => {
+    const researcher = await createResearcher(chatHistory, stream);
+    const researchRes = await researcher.run(
+      new StartEvent<AgentInput>({ input: { message: ev.data.input } }),
+    );
+    const researchResult = researchRes.data.result;
+    return new WriteEvent({
+      input: `Write a blog post given this task: ${context.get("task")} using this research content: ${researchResult}`,
+      isGood: false,
+    });
+  };
+
+  const write = async (context: Context, ev: WriteEvent) => {
+    context.set("attempts", context.get("attempts", 0) + 1);
+    const tooManyAttempts = context.get("attempts") > MAX_ATTEMPTS;
+    if (tooManyAttempts) {
+      appendStream(
+        "writer",
+        `Too many attempts (${MAX_ATTEMPTS}) to write the blog post. Proceeding with the current version.`,
+      );
+    }
+
+    if (ev.data.isGood || tooManyAttempts) {
+      const writer = createWriter(chatHistory, stream);
+      writer.run(
+        new StartEvent<AgentInput>({
+          input: { message: ev.data.input, streaming: true },
+        }),
+      );
+      const finalResultStream = writer.streamEvents();
+      context.writeEventToStream({
+        data: new AgentRunResult(finalResultStream),
+      });
+      return new StopEvent({ result: finalResultStream }); // stop the workflow
+    }
+
+    const writer = createWriter(chatHistory, stream);
+    const writeRes = await writer.run(
+      new StartEvent<AgentInput>({ input: { message: ev.data.input } }),
+    );
+    const writeResult = writeRes.data.result;
+    context.set("result", writeResult); // store the last result
+    return new ReviewEvent({ input: writeResult });
+  };
+
+  const review = async (context: Context, ev: ReviewEvent) => {
+    const reviewer = createReviewer(chatHistory, stream);
+    const reviewRes = await reviewer.run(
+      new StartEvent<AgentInput>({ input: { message: ev.data.input } }),
+    );
+    const reviewResult = reviewRes.data.result;
+    const oldContent = context.get("result");
+    const postIsGood = reviewResult.toLowerCase().includes("post is good");
+    appendStream(
+      "reviewer",
+      `The post is ${postIsGood ? "" : "not "}good enough for publishing. Sending back to the writer${
+        postIsGood ? " for publication." : "."
+      }`,
+    );
+    if (postIsGood) {
+      return new WriteEvent({
+        input: `You're blog post is ready for publication. Please respond with just the blog post. Blog post: \`\`\`${oldContent}\`\`\``,
+        isGood: true,
+      });
+    }
+
+    return new WriteEvent({
+      input: `Improve the writing of a given blog post by using a given review.
+            Blog post:
+            \`\`\`
+            ${oldContent}
+            \`\`\`
+
+            Review:
+            \`\`\`
+            ${reviewResult}
+            \`\`\``,
+      isGood: false,
+    });
+  };
+
+  const workflow = new Workflow({ timeout: TIMEOUT, validate: true });
+  workflow.addStep(StartEvent, start, { outputs: ResearchEvent });
+  workflow.addStep(ResearchEvent, research, { outputs: WriteEvent });
+  workflow.addStep(WriteEvent, write, { outputs: [ReviewEvent, StopEvent] });
+  workflow.addStep(ReviewEvent, review, { outputs: WriteEvent });
+
+  return workflow;
+};

templates/components/multiagent/typescript/workflow/stream.ts

@@ -0,0 +1,57 @@
+import { WorkflowEvent } from "@llamaindex/core/workflow";
+import {
+  createCallbacksTransformer,
+  createStreamDataTransformer,
+  StreamData,
+  trimStartOfStreamHelper,
+  type AIStreamCallbacksAndOptions,
+} from "ai";
+import { AgentRunResult, FunctionCallingStreamResult } from "./type";
+
+export function toDataStream(
+  generator: AsyncGenerator<WorkflowEvent, void>,
+  data: StreamData,
+  callbacks?: AIStreamCallbacksAndOptions,
+) {
+  return toReadableStream(generator, data)
+    .pipeThrough(createCallbacksTransformer(callbacks))
+    .pipeThrough(createStreamDataTransformer());
+}
+
+function toReadableStream(
+  generator: AsyncGenerator<WorkflowEvent, void>,
+  data: StreamData,
+) {
+  const trimStartOfStream = trimStartOfStreamHelper();
+  return new ReadableStream<string>({
+    start(controller) {
+      controller.enqueue(""); // Kickstart the stream
+    },
+    async pull(controller): Promise<void> {
+      const { value, done } = await generator.next();
+      if (done) return;
+      if (value.data instanceof AgentRunResult) {
+        const finalResultStream = value.data.response;
+        for await (const event of finalResultStream) {
+          if (event.data instanceof FunctionCallingStreamResult) {
+            const readableStream = event.data.response;
+            readableStream.pipeTo(
+              new WritableStream({
+                write(chunk) {
+                  const text = trimStartOfStream(chunk.delta ?? "");
+                  if (text) {
+                    controller.enqueue(text);
+                  }
+                },
+                close() {
+                  controller.close();
+                  data.close();
+                },
+              }),
+            );
+          }
+        }
+      }
+    },
+  });
+}

templates/components/multiagent/typescript/workflow/type.ts

@@ -0,0 +1,20 @@
+import { WorkflowEvent } from "@llamaindex/core/workflow";
+import { EngineResponse } from "llamaindex";
+
+export type AgentInput = {
+  message: string;
+  streaming?: boolean;
+};
+
+export class AgentRunEvent extends WorkflowEvent<{
+  name: string;
+  msg: string;
+}> {}
+
+export class AgentRunResult {
+  constructor(public response: AsyncGenerator<WorkflowEvent>) {}
+}
+
+export class FunctionCallingStreamResult {
+  constructor(public response: ReadableStream<EngineResponse>) {}
+}

templates/types/multiagent/express/README-template.md

@@ -0,0 +1,103 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) project using [Express](https://expressjs.com/) bootstrapped with [`create-llama`](https://github.com/run-llama/LlamaIndexTS/tree/main/packages/create-llama).
+
+## Getting Started
+
+First, install the dependencies:
+
+```
+npm install
+```
+
+Second, generate the embeddings of the documents in the `./data` directory (if this folder exists - otherwise, skip this step):
+
+```
+npm run generate
+```
+
+Third, run the development server:
+
+```
+npm run dev
+```
+
+The example provides two different API endpoints:
+
+1. `/api/chat` - a streaming chat endpoint (found in `src/controllers/chat.controller.ts`)
+2. `/api/chat/request` - a non-streaming chat endpoint (found in `src/controllers/chat-request.controller.ts`)
+
+You can test the streaming endpoint with the following curl request:
+
+```
+curl --location 'localhost:8000/api/chat' \
+--header 'Content-Type: application/json' \
+--data '{ "messages": [{ "role": "user", "content": "Hello" }] }'
+```
+
+And for the non-streaming endpoint run:
+
+```
+curl --location 'localhost:8000/api/chat/request' \
+--header 'Content-Type: application/json' \
+--data '{ "messages": [{ "role": "user", "content": "Hello" }] }'
+```
+
+You can start editing the API by modifying `src/controllers/chat.controller.ts` or `src/controllers/chat-request.controller.ts`. The endpoint auto-updates as you save the file.
+You can delete the endpoint that you're not using.
+
+## Production
+
+First, build the project:
+
+```
+npm run build
+```
+
+You can then run the production server:
+
+```
+NODE_ENV=production npm run start
+```
+
+> Note that the `NODE_ENV` environment variable is set to `production`. This disables CORS for all origins.
+
+## Using Docker
+
+1. Build an image for the Express API:
+
+```
+docker build -t <your_backend_image_name> .
+```
+
+2. Generate embeddings:
+
+Parse the data and generate the vector embeddings if the `./data` folder exists - otherwise, skip this step:
+
+```
+docker run --rm \
+  -v $(pwd)/.env:/app/.env \ # Use ENV variables and configuration from your file-system
+  -v $(pwd)/config:/app/config \
+  -v $(pwd)/data:/app/data \
+  -v $(pwd)/cache:/app/cache \ # Use your file system to store the vector database
+  <your_backend_image_name>
+  npm run generate
+```
+
+3. Start the API:
+
+```
+docker run \
+  -v $(pwd)/.env:/app/.env \ # Use ENV variables and configuration from your file-system
+  -v $(pwd)/config:/app/config \
+  -v $(pwd)/cache:/app/cache \ # Use your file system to store the vector database
+  -p 8000:8000 \
+  <your_backend_image_name>
+```
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex (Python features).
+- [LlamaIndexTS Documentation](https://ts.llamaindex.ai) - learn about LlamaIndex (Typescript features).
+
+You can check out [the LlamaIndexTS GitHub repository](https://github.com/run-llama/LlamaIndexTS) - your feedback and contributions are welcome!

templates/types/multiagent/express/eslintrc.json

@@ -0,0 +1,10 @@
+{
+  "extends": ["eslint:recommended", "prettier"],
+  "rules": {
+    "max-params": ["error", 4],
+    "prefer-const": "error"
+  },
+  "parserOptions": {
+    "sourceType": "module"
+  }
+}

templates/types/multiagent/express/gitignore

@@ -0,0 +1,5 @@
+# local env files
+.env
+node_modules/
+
+output/

templates/types/multiagent/express/index.ts

@@ -0,0 +1,46 @@
+/* eslint-disable turbo/no-undeclared-env-vars */
+import cors from "cors";
+import "dotenv/config";
+import express, { Express, Request, Response } from "express";
+import { initObservability } from "./src/observability";
+import chatRouter from "./src/routes/chat.route";
+
+const app: Express = express();
+const port = parseInt(process.env.PORT || "8000");
+
+const env = process.env["NODE_ENV"];
+const isDevelopment = !env || env === "development";
+const prodCorsOrigin = process.env["PROD_CORS_ORIGIN"];
+
+initObservability();
+
+app.use(express.json({ limit: "50mb" }));
+
+if (isDevelopment) {
+  console.warn("Running in development mode - allowing CORS for all origins");
+  app.use(cors());
+} else if (prodCorsOrigin) {
+  console.log(
+    `Running in production mode - allowing CORS for domain: ${prodCorsOrigin}`,
+  );
+  const corsOptions = {
+    origin: prodCorsOrigin, // Restrict to production domain
+  };
+  app.use(cors(corsOptions));
+} else {
+  console.warn("Production CORS origin not set, defaulting to no CORS.");
+}
+
+app.use("/api/files/data", express.static("data"));
+app.use("/api/files/output", express.static("output"));
+app.use(express.text());
+
+app.get("/", (req: Request, res: Response) => {
+  res.send("LlamaIndex Express Server");
+});
+
+app.use("/api/chat", chatRouter);
+
+app.listen(port, () => {
+  console.log(`⚡️[server]: Server is running at http://localhost:${port}`);
+});

templates/types/multiagent/express/npmrc

@@ -0,0 +1 @@
+node-linker=hoisted

templates/types/multiagent/express/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "llama-index-express-multiagent",
+  "version": "1.0.0",
+  "exports": "./index.js",
+  "types": "./index.d.ts",
+  "type": "module",
+  "engines": {
+    "node": ">=18"
+  },
+  "scripts": {
+    "format": "prettier --ignore-unknown --cache --check .",
+    "format:write": "prettier --ignore-unknown --write .",
+    "build": "tsup index.ts --format esm --dts",
+    "start": "node dist/index.js",
+    "dev": "concurrently \"tsup index.ts --format esm --dts --watch\" \"nodemon --watch dist/index.js\""
+  },
+  "dependencies": {
+    "ai": "3.3.38",
+    "cors": "^2.8.5",
+    "dotenv": "^16.3.1",
+    "duck-duck-scrape": "^2.2.5",
+    "express": "^4.18.2",
+    "llamaindex": "0.6.2",
+    "pdf2json": "3.0.5",
+    "ajv": "^8.12.0",
+    "@e2b/code-interpreter": "^0.0.5",
+    "got": "^14.4.1",
+    "@apidevtools/swagger-parser": "^10.1.0",
+    "formdata-node": "^6.0.3"
+  },
+  "devDependencies": {
+    "@types/cors": "^2.8.16",
+    "@types/express": "^4.17.21",
+    "@types/node": "^20.9.5",
+    "concurrently": "^8.2.2",
+    "eslint": "^8.54.0",
+    "eslint-config-prettier": "^8.10.0",
+    "nodemon": "^3.0.1",
+    "prettier": "^3.2.5",
+    "prettier-plugin-organize-imports": "^3.2.4",
+    "tsx": "^4.7.2",
+    "tsup": "8.1.0",
+    "typescript": "^5.3.2"
+  }
+}

templates/types/multiagent/express/prettier.config.cjs

@@ -0,0 +1,3 @@
+module.exports = {
+  plugins: ["prettier-plugin-organize-imports"],
+};

templates/types/multiagent/express/src/controllers/chat-config.controller.ts

@@ -0,0 +1,31 @@
+import { Request, Response } from "express";
+import { LLamaCloudFileService } from "llamaindex";
+
+export const chatConfig = async (_req: Request, res: Response) => {
+  let starterQuestions = undefined;
+  if (
+    process.env.CONVERSATION_STARTERS &&
+    process.env.CONVERSATION_STARTERS.trim()
+  ) {
+    starterQuestions = process.env.CONVERSATION_STARTERS.trim().split("\n");
+  }
+  return res.status(200).json({
+    starterQuestions,
+  });
+};
+
+export const chatLlamaCloudConfig = async (_req: Request, res: Response) => {
+  if (!process.env.LLAMA_CLOUD_API_KEY) {
+    return res.status(500).json({
+      error: "env variable LLAMA_CLOUD_API_KEY is required to use LlamaCloud",
+    });
+  }
+  const config = {
+    projects: await LLamaCloudFileService.getAllProjectsWithPipelines(),
+    pipeline: {
+      pipeline: process.env.LLAMA_CLOUD_INDEX_NAME,
+      project: process.env.LLAMA_CLOUD_PROJECT_NAME,
+    },
+  };
+  return res.status(200).json(config);
+};

templates/types/multiagent/express/src/controllers/chat.controller.ts

@@ -0,0 +1,34 @@
+import { Message, StreamData, streamToResponse } from "ai";
+import { Request, Response } from "express";
+import { ChatMessage } from "llamaindex";
+import { createStreamTimeout } from "./llamaindex/streaming/events";
+import { createWorkflow } from "./workflow";
+import { toDataStream } from "./workflow/stream";
+
+export const chat = async (req: Request, res: Response) => {
+  const vercelStreamData = new StreamData();
+  const streamTimeout = createStreamTimeout(vercelStreamData);
+  try {
+    const { messages }: { messages: Message[] } = req.body;
+    const userMessage = messages.pop();
+    if (!messages || !userMessage || userMessage.role !== "user") {
+      return res.status(400).json({
+        error:
+          "messages are required in the request body and the last message must be from the user",
+      });
+    }
+
+    const chatHistory = messages as ChatMessage[];
+    const agent = await createWorkflow(chatHistory, vercelStreamData);
+    agent.run(userMessage.content);
+    const stream = toDataStream(agent.streamEvents(), vercelStreamData);
+    return streamToResponse(stream, res, {}, vercelStreamData);
+  } catch (error) {
+    console.error("[LlamaIndex]", error);
+    return res.status(500).json({
+      detail: (error as Error).message,
+    });
+  } finally {
+    clearTimeout(streamTimeout);
+  }
+};

templates/types/multiagent/express/src/observability/index.ts

@@ -0,0 +1 @@
+export const initObservability = () => {};

templates/types/multiagent/express/src/routes/chat.route.ts

@@ -0,0 +1,12 @@
+import express, { Router } from "express";
+import { chatConfig } from "../controllers/chat-config.controller";
+import { chat } from "../controllers/chat.controller";
+import { initSettings } from "../controllers/engine/settings";
+
+const llmRouter: Router = express.Router();
+
+initSettings();
+llmRouter.route("/").post(chat);
+llmRouter.route("/config").get(chatConfig);
+
+export default llmRouter;

templates/types/multiagent/express/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "target": "es2016",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "paths": {
+      "@/*": ["./*"]
+    }
+  }
+}

templates/types/streaming/nextjs/app/api/chat/engine/settings.ts

@@ -1,179 +0,0 @@
-import {
-  ALL_AVAILABLE_MISTRAL_MODELS,
-  Anthropic,
-  GEMINI_EMBEDDING_MODEL,
-  GEMINI_MODEL,
-  Gemini,
-  GeminiEmbedding,
-  Groq,
-  MistralAI,
-  MistralAIEmbedding,
-  MistralAIEmbeddingModelType,
-  OpenAI,
-  OpenAIEmbedding,
-  Settings,
-} from "llamaindex";
-import { HuggingFaceEmbedding } from "llamaindex/embeddings/HuggingFaceEmbedding";
-import { OllamaEmbedding } from "llamaindex/embeddings/OllamaEmbedding";
-import { ALL_AVAILABLE_ANTHROPIC_MODELS } from "llamaindex/llm/anthropic";
-import { Ollama } from "llamaindex/llm/ollama";
-
-const CHUNK_SIZE = 512;
-const CHUNK_OVERLAP = 20;
-
-export const initSettings = async () => {
-  // HINT: you can delete the initialization code for unused model providers
-  console.log(`Using '${process.env.MODEL_PROVIDER}' model provider`);
-
-  if (!process.env.MODEL || !process.env.EMBEDDING_MODEL) {
-    throw new Error("'MODEL' and 'EMBEDDING_MODEL' env variables must be set.");
-  }
-
-  switch (process.env.MODEL_PROVIDER) {
-    case "ollama":
-      initOllama();
-      break;
-    case "groq":
-      initGroq();
-      break;
-    case "anthropic":
-      initAnthropic();
-      break;
-    case "gemini":
-      initGemini();
-      break;
-    case "mistral":
-      initMistralAI();
-      break;
-    case "azure-openai":
-      initAzureOpenAI();
-      break;
-    default:
-      initOpenAI();
-      break;
-  }
-  Settings.chunkSize = CHUNK_SIZE;
-  Settings.chunkOverlap = CHUNK_OVERLAP;
-};
-
-function initOpenAI() {
-  Settings.llm = new OpenAI({
-    model: process.env.MODEL ?? "gpt-4o-mini",
-    maxTokens: process.env.LLM_MAX_TOKENS
-      ? Number(process.env.LLM_MAX_TOKENS)
-      : undefined,
-  });
-  Settings.embedModel = new OpenAIEmbedding({
-    model: process.env.EMBEDDING_MODEL,
-    dimensions: process.env.EMBEDDING_DIM
-      ? parseInt(process.env.EMBEDDING_DIM)
-      : undefined,
-  });
-}
-
-function initAzureOpenAI() {
-  // Map Azure OpenAI model names to OpenAI model names (only for TS)
-  const AZURE_OPENAI_MODEL_MAP: Record<string, string> = {
-    "gpt-35-turbo": "gpt-3.5-turbo",
-    "gpt-35-turbo-16k": "gpt-3.5-turbo-16k",
-    "gpt-4o": "gpt-4o",
-    "gpt-4": "gpt-4",
-    "gpt-4-32k": "gpt-4-32k",
-    "gpt-4-turbo": "gpt-4-turbo",
-    "gpt-4-turbo-2024-04-09": "gpt-4-turbo",
-    "gpt-4-vision-preview": "gpt-4-vision-preview",
-    "gpt-4-1106-preview": "gpt-4-1106-preview",
-    "gpt-4o-2024-05-13": "gpt-4o-2024-05-13",
-  };
-
-  const azureConfig = {
-    apiKey: process.env.AZURE_OPENAI_KEY,
-    endpoint: process.env.AZURE_OPENAI_ENDPOINT,
-    apiVersion:
-      process.env.AZURE_OPENAI_API_VERSION || process.env.OPENAI_API_VERSION,
-  };
-
-  Settings.llm = new OpenAI({
-    model:
-      AZURE_OPENAI_MODEL_MAP[process.env.MODEL ?? "gpt-35-turbo"] ??
-      "gpt-3.5-turbo",
-    maxTokens: process.env.LLM_MAX_TOKENS
-      ? Number(process.env.LLM_MAX_TOKENS)
-      : undefined,
-    azure: {
-      ...azureConfig,
-      deployment: process.env.AZURE_OPENAI_LLM_DEPLOYMENT,
-    },
-  });
-
-  Settings.embedModel = new OpenAIEmbedding({
-    model: process.env.EMBEDDING_MODEL,
-    dimensions: process.env.EMBEDDING_DIM
-      ? parseInt(process.env.EMBEDDING_DIM)
-      : undefined,
-    azure: {
-      ...azureConfig,
-      deployment: process.env.AZURE_OPENAI_EMBEDDING_DEPLOYMENT,
-    },
-  });
-}
-
-function initOllama() {
-  const config = {
-    host: process.env.OLLAMA_BASE_URL ?? "http://127.0.0.1:11434",
-  };
-  Settings.llm = new Ollama({
-    model: process.env.MODEL ?? "",
-    config,
-  });
-  Settings.embedModel = new OllamaEmbedding({
-    model: process.env.EMBEDDING_MODEL ?? "",
-    config,
-  });
-}
-
-function initGroq() {
-  const embedModelMap: Record<string, string> = {
-    "all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
-    "all-mpnet-base-v2": "Xenova/all-mpnet-base-v2",
-  };
-
-  Settings.llm = new Groq({
-    model: process.env.MODEL!,
-  });
-
-  Settings.embedModel = new HuggingFaceEmbedding({
-    modelType: embedModelMap[process.env.EMBEDDING_MODEL!],
-  });
-}
-
-function initAnthropic() {
-  const embedModelMap: Record<string, string> = {
-    "all-MiniLM-L6-v2": "Xenova/all-MiniLM-L6-v2",
-    "all-mpnet-base-v2": "Xenova/all-mpnet-base-v2",
-  };
-  Settings.llm = new Anthropic({
-    model: process.env.MODEL as keyof typeof ALL_AVAILABLE_ANTHROPIC_MODELS,
-  });
-  Settings.embedModel = new HuggingFaceEmbedding({
-    modelType: embedModelMap[process.env.EMBEDDING_MODEL!],
-  });
-}
-
-function initGemini() {
-  Settings.llm = new Gemini({
-    model: process.env.MODEL as GEMINI_MODEL,
-  });
-  Settings.embedModel = new GeminiEmbedding({
-    model: process.env.EMBEDDING_MODEL as GEMINI_EMBEDDING_MODEL,
-  });
-}
-
-function initMistralAI() {
-  Settings.llm = new MistralAI({
-    model: process.env.MODEL as keyof typeof ALL_AVAILABLE_MISTRAL_MODELS,
-  });
-  Settings.embedModel = new MistralAIEmbedding({
-    model: process.env.EMBEDDING_MODEL as MistralAIEmbeddingModelType,
-  });
-}