fix: commit

Releases:
  llamaindex@0.0.46
  create-llama@0.0.15

[skip ci]

.changeset/silly-bugs-fold.md

@@ -0,0 +1,5 @@
+---
+"create-llama": patch
+---
+
+Added option to automatically install dependencies (for Python and TS)

.github/workflows/e2e.yml

@@ -8,6 +8,9 @@ on:
       - ".github/workflows/e2e.yml"
     branches: [main]
 
+env:
+  POETRY_VERSION: "1.6.1"
+
 jobs:
   e2e:
     name: create-llama
@@ -16,10 +19,19 @@ jobs:
       fail-fast: true
       matrix:
         node-version: [18, 20]
+        python-version: ["3.11"]
         os: [macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
+      - name: Set up python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
       - uses: pnpm/action-setup@v2
       - name: Setup Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4

.github/workflows/test.yml

@@ -18,3 +18,21 @@ jobs:
         run: pnpm install
       - name: Run tests
         run: pnpm run test
+  typecheck:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v2
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: ".nvmrc"
+          cache: "pnpm"
+      - name: Install dependencies
+        run: pnpm install
+      - name: Build
+        run: pnpm run build
+        working-directory: ./packages/core
+      - name: Run Type Check
+        run: pnpm run type-check

.gitignore

@@ -37,6 +37,7 @@ yarn-error.log*
 .vercel
 
 dist/
+lib/
 
 # vs code
 .vscode/launch.json

.prettierignore

@@ -1,3 +1,4 @@
 apps/docs/i18n
 pnpm-lock.yaml
-
+lib/
+dist/

README.md

@@ -1,5 +1,10 @@
 # LlamaIndex.TS
 
+[![NPM Version](https://img.shields.io/npm/v/llamaindex)](https://www.npmjs.com/package/llamaindex)
+[![NPM License](https://img.shields.io/npm/l/llamaindex)](https://www.npmjs.com/package/llamaindex)
+[![NPM Downloads](https://img.shields.io/npm/dm/llamaindex)](https://www.npmjs.com/package/llamaindex)
+[![Discord](https://img.shields.io/discord/1059199217496772688)](https://discord.com/invite/eN6D2HQ4aX)
+
 LlamaIndex is a data framework for your LLM application.
 
 Use your own data with large language models (LLMs, OpenAI ChatGPT and others) in Typescript and Javascript.

apps/demo-edge-runtime/.env.example

@@ -0,0 +1,3 @@
+# Rename this file to `.env.local` to use environment variables locally with `next dev`
+# https://nextjs.org/docs/pages/building-your-application/configuring/environment-variables
+MY_HOST="example.com"

apps/demo-edge-runtime/README.md

@@ -0,0 +1,30 @@
+This is a [LlamaIndex](https://www.llamaindex.ai/) project using [Next.js](https://nextjs.org/) bootstrapped with [`create-llama`](https://github.com/run-llama/LlamaIndexTS/tree/main/packages/create-llama).
+
+## Getting Started
+
+First, install the dependencies:
+
+```
+npm install
+```
+
+Second, run the development server:
+
+```
+npm run dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file.
+
+This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
+
+## Learn More
+
+To learn more about LlamaIndex, take a look at the following resources:
+
+- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex (Python features).
+- [LlamaIndexTS Documentation](https://ts.llamaindex.ai) - learn about LlamaIndex (Typescript features).
+
+You can check out [the LlamaIndexTS GitHub repository](https://github.com/run-llama/LlamaIndexTS) - your feedback and contributions are welcome!

apps/demo-edge-runtime/app/api/chat/engine/constants.mjs

@@ -0,0 +1,4 @@
+export const STORAGE_DIR = "./data";
+export const STORAGE_CACHE_DIR = "./cache";
+export const CHUNK_SIZE = 512;
+export const CHUNK_OVERLAP = 20;

apps/demo-edge-runtime/app/api/chat/engine/generate.mjs

@@ -0,0 +1,53 @@
+import {
+  serviceContextFromDefaults,
+  SimpleDirectoryReader,
+  storageContextFromDefaults,
+  VectorStoreIndex,
+} from "llamaindex";
+
+import * as dotenv from "dotenv";
+
+import {
+  CHUNK_OVERLAP,
+  CHUNK_SIZE,
+  STORAGE_CACHE_DIR,
+  STORAGE_DIR,
+} from "./constants.mjs";
+
+// Load environment variables from local .env file
+dotenv.config();
+
+async function getRuntime(func) {
+  const start = Date.now();
+  await func();
+  const end = Date.now();
+  return end - start;
+}
+
+async function generateDatasource(serviceContext) {
+  console.log(`Generating storage context...`);
+  // Split documents, create embeddings and store them in the storage context
+  const ms = await getRuntime(async () => {
+    const storageContext = await storageContextFromDefaults({
+      persistDir: STORAGE_CACHE_DIR,
+    });
+    const documents = await new SimpleDirectoryReader().loadData({
+      directoryPath: STORAGE_DIR,
+    });
+    await VectorStoreIndex.fromDocuments(documents, {
+      storageContext,
+      serviceContext,
+    });
+  });
+  console.log(`Storage context successfully generated in ${ms / 1000}s.`);
+}
+
+(async () => {
+  const serviceContext = serviceContextFromDefaults({
+    chunkSize: CHUNK_SIZE,
+    chunkOverlap: CHUNK_OVERLAP,
+  });
+
+  await generateDatasource(serviceContext);
+  console.log("Finished generating storage.");
+})();

apps/demo-edge-runtime/app/api/chat/engine/index.ts

@@ -0,0 +1,59 @@
+import {
+  ContextChatEngine,
+  LLM,
+  SimpleDocumentStore,
+  VectorStoreIndex,
+  genericFileSystem,
+  serviceContextFromDefaults,
+  storageContextFromDefaults,
+} from "llamaindex";
+import { CHUNK_OVERLAP, CHUNK_SIZE, STORAGE_CACHE_DIR } from "./constants.mjs";
+
+async function getDataSource(llm: LLM) {
+  const fs = genericFileSystem;
+  await fs.writeFile(
+    `${STORAGE_CACHE_DIR}/doc_store.json`,
+    JSON.stringify(await import("../../../../cache/doc_store.json")),
+  );
+  await fs.writeFile(
+    `${STORAGE_CACHE_DIR}/index_store.json`,
+    JSON.stringify(await import("../../../../cache/index_store.json")),
+  );
+  await fs.writeFile(
+    `${STORAGE_CACHE_DIR}/vector_store.json`,
+    JSON.stringify(await import("../../../../cache/vector_store.json")),
+  );
+
+  const serviceContext = serviceContextFromDefaults({
+    llm,
+    chunkSize: CHUNK_SIZE,
+    chunkOverlap: CHUNK_OVERLAP,
+  });
+  let storageContext = await storageContextFromDefaults({
+    persistDir: `${STORAGE_CACHE_DIR}`,
+  });
+
+  const numberOfDocs = Object.keys(
+    (storageContext.docStore as SimpleDocumentStore).toDict(),
+  ).length;
+  if (numberOfDocs === 0) {
+    throw new Error(
+      `StorageContext is empty - call 'npm run generate' to generate the storage first`,
+    );
+  }
+  return await VectorStoreIndex.init({
+    storageContext,
+    serviceContext,
+  });
+}
+
+export async function createChatEngine(llm: LLM) {
+  const index = await getDataSource(llm);
+  const retriever = index.asRetriever();
+  retriever.similarityTopK = 5;
+
+  return new ContextChatEngine({
+    chatModel: llm,
+    retriever,
+  });
+}

apps/demo-edge-runtime/app/api/chat/llamaindex-stream.ts

@@ -0,0 +1,66 @@
+import {
+  JSONValue,
+  createCallbacksTransformer,
+  createStreamDataTransformer,
+  experimental_StreamData,
+  trimStartOfStreamHelper,
+  type AIStreamCallbacksAndOptions,
+} from "ai";
+
+type ParserOptions = {
+  image_url?: string;
+};
+
+function createParser(
+  res: AsyncGenerator<any>,
+  data: experimental_StreamData,
+  opts?: ParserOptions,
+) {
+  const trimStartOfStream = trimStartOfStreamHelper();
+  return new ReadableStream<string>({
+    start() {
+      // if image_url is provided, send it via the data stream
+      if (opts?.image_url) {
+        const message: JSONValue = {
+          type: "image_url",
+          image_url: {
+            url: opts.image_url,
+          },
+        };
+        data.append(message);
+      } else {
+        data.append({}); // send an empty image response for the user's message
+      }
+    },
+    async pull(controller): Promise<void> {
+      const { value, done } = await res.next();
+      if (done) {
+        controller.close();
+        data.append({}); // send an empty image response for the assistant's message
+        data.close();
+        return;
+      }
+
+      const text = trimStartOfStream(value ?? "");
+      if (text) {
+        controller.enqueue(text);
+      }
+    },
+  });
+}
+
+export function LlamaIndexStream(
+  res: AsyncGenerator<any>,
+  opts?: {
+    callbacks?: AIStreamCallbacksAndOptions;
+    parserOptions?: ParserOptions;
+  },
+): { stream: ReadableStream; data: experimental_StreamData } {
+  const data = new experimental_StreamData();
+  return {
+    stream: createParser(res, data, opts?.parserOptions)
+      .pipeThrough(createCallbacksTransformer(opts?.callbacks))
+      .pipeThrough(createStreamDataTransformer(true)),
+    data,
+  };
+}

apps/demo-edge-runtime/app/api/chat/route.ts

@@ -0,0 +1,82 @@
+import { Message, StreamingTextResponse } from "ai";
+import { ChatMessage, MessageContent, OpenAI } from "llamaindex";
+import { NextRequest, NextResponse } from "next/server";
+import { createChatEngine } from "./engine";
+import { LlamaIndexStream } from "./llamaindex-stream";
+
+export const runtime = "edge";
+export const dynamic = "force-dynamic";
+
+const getLastMessageContent = (
+  textMessage: string,
+  imageUrl: string | undefined,
+): MessageContent => {
+  if (!imageUrl) return textMessage;
+  return [
+    {
+      type: "text",
+      text: textMessage,
+    },
+    {
+      type: "image_url",
+      image_url: {
+        url: imageUrl,
+      },
+    },
+  ];
+};
+
+export async function POST(request: NextRequest) {
+  try {
+    const body = await request.json();
+    const { messages, data }: { messages: Message[]; data: any } = body;
+    const lastMessage = messages.pop();
+    if (!messages || !lastMessage || lastMessage.role !== "user") {
+      return NextResponse.json(
+        {
+          error:
+            "messages are required in the request body and the last message must be from the user",
+        },
+        { status: 400 },
+      );
+    }
+
+    const llm = new OpenAI({
+      model: (process.env.MODEL as any) ?? "gpt-3.5-turbo",
+      maxTokens: 512,
+    });
+
+    const chatEngine = await createChatEngine(llm);
+
+    const lastMessageContent = getLastMessageContent(
+      lastMessage.content,
+      data?.imageUrl,
+    );
+
+    const response = await chatEngine.chat(
+      lastMessageContent as MessageContent,
+      messages as ChatMessage[],
+      true,
+    );
+
+    // Transform the response into a readable stream
+    const { stream, data: streamData } = LlamaIndexStream(response, {
+      parserOptions: {
+        image_url: data?.imageUrl,
+      },
+    });
+
+    // Return a StreamingTextResponse, which can be consumed by the client
+    return new StreamingTextResponse(stream, {}, streamData);
+  } catch (error) {
+    console.error("[LlamaIndex]", error);
+    return NextResponse.json(
+      {
+        error: (error as Error).message,
+      },
+      {
+        status: 500,
+      },
+    );
+  }
+}

apps/demo-edge-runtime/app/components/chat-section.tsx

@@ -0,0 +1,46 @@
+"use client";
+
+import { useChat } from "ai/react";
+import { useMemo } from "react";
+import { insertDataIntoMessages } from "./transform";
+import { ChatInput, ChatMessages } from "./ui/chat";
+
+export default function ChatSection() {
+  const {
+    messages,
+    input,
+    isLoading,
+    handleSubmit,
+    handleInputChange,
+    reload,
+    stop,
+    data,
+  } = useChat({
+    api: process.env.NEXT_PUBLIC_CHAT_API,
+    headers: {
+      "Content-Type": "application/json", // using JSON because of vercel/ai 2.2.26
+    },
+  });
+
+  const transformedMessages = useMemo(() => {
+    return insertDataIntoMessages(messages, data);
+  }, [messages, data]);
+
+  return (
+    <div className="space-y-4 max-w-5xl w-full">
+      <ChatMessages
+        messages={transformedMessages}
+        isLoading={isLoading}
+        reload={reload}
+        stop={stop}
+      />
+      <ChatInput
+        input={input}
+        handleSubmit={handleSubmit}
+        handleInputChange={handleInputChange}
+        isLoading={isLoading}
+        multiModal={process.env.NEXT_PUBLIC_MODEL === "gpt-4-vision-preview"}
+      />
+    </div>
+  );
+}

apps/demo-edge-runtime/app/components/header.tsx

@@ -0,0 +1,28 @@
+import Image from "next/image";
+
+export default function Header() {
+  return (
+    <div className="z-10 max-w-5xl w-full items-center justify-between font-mono text-sm lg:flex">
+      <p className="fixed left-0 top-0 flex w-full justify-center border-b border-gray-300 bg-gradient-to-b from-zinc-200 pb-6 pt-8 backdrop-blur-2xl dark:border-neutral-800 dark:bg-zinc-800/30 dark:from-inherit lg:static lg:w-auto  lg:rounded-xl lg:border lg:bg-gray-200 lg:p-4 lg:dark:bg-zinc-800/30">
+        Get started by editing&nbsp;
+        <code className="font-mono font-bold">app/page.tsx</code>
+      </p>
+      <div className="fixed bottom-0 left-0 flex h-48 w-full items-end justify-center bg-gradient-to-t from-white via-white dark:from-black dark:via-black lg:static lg:h-auto lg:w-auto lg:bg-none">
+        <a
+          href="https://www.llamaindex.ai/"
+          className="flex items-center justify-center font-nunito text-lg font-bold gap-2"
+        >
+          <span>Built by LlamaIndex</span>
+          <Image
+            className="rounded-xl"
+            src="/llama.png"
+            alt="Llama Logo"
+            width={40}
+            height={40}
+            priority
+          />
+        </a>
+      </div>
+    </div>
+  );
+}

apps/demo-edge-runtime/app/components/transform.ts

@@ -0,0 +1,19 @@
+import { JSONValue, Message } from "ai";
+
+export const isValidMessageData = (rawData: JSONValue | undefined) => {
+  if (!rawData || typeof rawData !== "object") return false;
+  if (Object.keys(rawData).length === 0) return false;
+  return true;
+};
+
+export const insertDataIntoMessages = (
+  messages: Message[],
+  data: JSONValue[] | undefined,
+) => {
+  if (!data) return messages;
+  messages.forEach((message, i) => {
+    const rawData = data[i];
+    if (isValidMessageData(rawData)) message.data = rawData;
+  });
+  return messages;
+};

apps/demo-edge-runtime/app/components/ui/chat/chat-avatar.tsx

@@ -0,0 +1,34 @@
+"use client";
+
+import Image from "next/image";
+import { Message } from "./chat-messages";
+
+export default function ChatAvatar(message: Message) {
+  if (message.role === "user") {
+    return (
+      <div className="flex h-8 w-8 shrink-0 select-none items-center justify-center rounded-md border shadow bg-background">
+        <svg
+          xmlns="http://www.w3.org/2000/svg"
+          viewBox="0 0 256 256"
+          fill="currentColor"
+          className="h-4 w-4"
+        >
+          <path d="M230.92 212c-15.23-26.33-38.7-45.21-66.09-54.16a72 72 0 1 0-73.66 0c-27.39 8.94-50.86 27.82-66.09 54.16a8 8 0 1 0 13.85 8c18.84-32.56 52.14-52 89.07-52s70.23 19.44 89.07 52a8 8 0 1 0 13.85-8ZM72 96a56 56 0 1 1 56 56 56.06 56.06 0 0 1-56-56Z"></path>
+        </svg>
+      </div>
+    );
+  }
+
+  return (
+    <div className="flex h-8 w-8 shrink-0 select-none items-center justify-center rounded-md border  bg-black text-white">
+      <Image
+        className="rounded-md"
+        src="/llama.png"
+        alt="Llama Logo"
+        width={24}
+        height={24}
+        priority
+      />
+    </div>
+  );
+}

apps/demo-edge-runtime/app/components/ui/chat/chat-input.tsx

@@ -0,0 +1,43 @@
+"use client";
+
+export interface ChatInputProps {
+  /** The current value of the input */
+  input?: string;
+  /** An input/textarea-ready onChange handler to control the value of the input */
+  handleInputChange?: (
+    e:
+      | React.ChangeEvent<HTMLInputElement>
+      | React.ChangeEvent<HTMLTextAreaElement>,
+  ) => void;
+  /** Form submission handler to automatically reset input and append a user message  */
+  handleSubmit: (e: React.FormEvent<HTMLFormElement>) => void;
+  isLoading: boolean;
+  multiModal?: boolean;
+}
+
+export default function ChatInput(props: ChatInputProps) {
+  return (
+    <>
+      <form
+        onSubmit={props.handleSubmit}
+        className="flex items-start justify-between w-full max-w-5xl p-4 bg-white rounded-xl shadow-xl gap-4"
+      >
+        <input
+          autoFocus
+          name="message"
+          placeholder="Type a message"
+          className="w-full p-4 rounded-xl shadow-inner flex-1"
+          value={props.input}
+          onChange={props.handleInputChange}
+        />
+        <button
+          disabled={props.isLoading}
+          type="submit"
+          className="p-4 text-white rounded-xl shadow-xl bg-gradient-to-r from-cyan-500 to-sky-500 disabled:opacity-50 disabled:cursor-not-allowed"
+        >
+          Send message
+        </button>
+      </form>
+    </>
+  );
+}

apps/demo-edge-runtime/app/components/ui/chat/chat-item.tsx

@@ -0,0 +1,13 @@
+"use client";
+
+import ChatAvatar from "./chat-avatar";
+import { Message } from "./chat-messages";
+
+export default function ChatItem(message: Message) {
+  return (
+    <div className="flex items-start gap-4 pt-5">
+      <ChatAvatar {...message} />
+      <p className="break-words">{message.content}</p>
+    </div>
+  );
+}

apps/demo-edge-runtime/app/components/ui/chat/chat-messages.tsx

@@ -0,0 +1,48 @@
+"use client";
+
+import { useEffect, useRef } from "react";
+import ChatItem from "./chat-item";
+
+export interface Message {
+  id: string;
+  content: string;
+  role: string;
+}
+
+export default function ChatMessages({
+  messages,
+  isLoading,
+  reload,
+  stop,
+}: {
+  messages: Message[];
+  isLoading?: boolean;
+  stop?: () => void;
+  reload?: () => void;
+}) {
+  const scrollableChatContainerRef = useRef<HTMLDivElement>(null);
+
+  const scrollToBottom = () => {
+    if (scrollableChatContainerRef.current) {
+      scrollableChatContainerRef.current.scrollTop =
+        scrollableChatContainerRef.current.scrollHeight;
+    }
+  };
+
+  useEffect(() => {
+    scrollToBottom();
+  }, [messages.length]);
+
+  return (
+    <div className="w-full max-w-5xl p-4 bg-white rounded-xl shadow-xl">
+      <div
+        className="flex flex-col gap-5 divide-y h-[50vh] overflow-auto"
+        ref={scrollableChatContainerRef}
+      >
+        {messages.map((m: Message) => (
+          <ChatItem key={m.id} {...m} />
+        ))}
+      </div>
+    </div>
+  );
+}

apps/demo-edge-runtime/app/components/ui/chat/index.ts

@@ -0,0 +1,6 @@
+import ChatInput from "./chat-input";
+import ChatMessages from "./chat-messages";
+
+export type { ChatInputProps } from "./chat-input";
+export type { Message } from "./chat-messages";
+export { ChatInput, ChatMessages };

apps/demo-edge-runtime/app/globals.css

@@ -0,0 +1,94 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+@layer base {
+  :root {
+    --background: 0 0% 100%;
+    --foreground: 222.2 47.4% 11.2%;
+
+    --muted: 210 40% 96.1%;
+    --muted-foreground: 215.4 16.3% 46.9%;
+
+    --popover: 0 0% 100%;
+    --popover-foreground: 222.2 47.4% 11.2%;
+
+    --border: 214.3 31.8% 91.4%;
+    --input: 214.3 31.8% 91.4%;
+
+    --card: 0 0% 100%;
+    --card-foreground: 222.2 47.4% 11.2%;
+
+    --primary: 222.2 47.4% 11.2%;
+    --primary-foreground: 210 40% 98%;
+
+    --secondary: 210 40% 96.1%;
+    --secondary-foreground: 222.2 47.4% 11.2%;
+
+    --accent: 210 40% 96.1%;
+    --accent-foreground: 222.2 47.4% 11.2%;
+
+    --destructive: 0 100% 50%;
+    --destructive-foreground: 210 40% 98%;
+
+    --ring: 215 20.2% 65.1%;
+
+    --radius: 0.5rem;
+  }
+
+  .dark {
+    --background: 224 71% 4%;
+    --foreground: 213 31% 91%;
+
+    --muted: 223 47% 11%;
+    --muted-foreground: 215.4 16.3% 56.9%;
+
+    --accent: 216 34% 17%;
+    --accent-foreground: 210 40% 98%;
+
+    --popover: 224 71% 4%;
+    --popover-foreground: 215 20.2% 65.1%;
+
+    --border: 216 34% 17%;
+    --input: 216 34% 17%;
+
+    --card: 224 71% 4%;
+    --card-foreground: 213 31% 91%;
+
+    --primary: 210 40% 98%;
+    --primary-foreground: 222.2 47.4% 1.2%;
+
+    --secondary: 222.2 47.4% 11.2%;
+    --secondary-foreground: 210 40% 98%;
+
+    --destructive: 0 63% 31%;
+    --destructive-foreground: 210 40% 98%;
+
+    --ring: 216 34% 17%;
+
+    --radius: 0.5rem;
+  }
+}
+
+@layer base {
+  * {
+    @apply border-border;
+  }
+  body {
+    @apply bg-background text-foreground;
+    font-feature-settings:
+      "rlig" 1,
+      "calt" 1;
+  }
+  .background-gradient {
+    background-color: #fff;
+    background-image: radial-gradient(
+        at 21% 11%,
+        rgba(186, 186, 233, 0.53) 0,
+        transparent 50%
+      ),
+      radial-gradient(at 85% 0, hsla(46, 57%, 78%, 0.52) 0, transparent 50%),
+      radial-gradient(at 91% 36%, rgba(194, 213, 255, 0.68) 0, transparent 50%),
+      radial-gradient(at 8% 40%, rgba(251, 218, 239, 0.46) 0, transparent 50%);
+  }
+}

apps/demo-edge-runtime/app/layout.tsx

@@ -0,0 +1,22 @@
+import type { Metadata } from "next";
+import { Inter } from "next/font/google";
+import "./globals.css";
+
+const inter = Inter({ subsets: ["latin"] });
+
+export const metadata: Metadata = {
+  title: "Create Llama App",
+  description: "Generated by create-llama",
+};
+
+export default function RootLayout({
+  children,
+}: {
+  children: React.ReactNode;
+}) {
+  return (
+    <html lang="en">
+      <body className={inter.className}>{children}</body>
+    </html>
+  );
+}

apps/demo-edge-runtime/app/page.tsx

@@ -0,0 +1,11 @@
+import Header from "@/app/components/header";
+import ChatSection from "./components/chat-section";
+
+export default function Home() {
+  return (
+    <main className="flex min-h-screen flex-col items-center gap-10 p-24 background-gradient">
+      <Header />
+      <ChatSection />
+    </main>
+  );
+}

apps/demo-edge-runtime/next-env.d.ts

@@ -0,0 +1,5 @@
+/// <reference types="next" />
+/// <reference types="next/image-types/global" />
+
+// NOTE: This file should not be edited
+// see https://nextjs.org/docs/basic-features/typescript for more information.

apps/demo-edge-runtime/next.config.js

@@ -0,0 +1,4 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {};
+
+module.exports = nextConfig;

apps/demo-edge-runtime/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "demo-edge-runtime",
+  "version": "0.1.0",
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint",
+    "generate": "node app/api/chat/engine/generate.mjs"
+  },
+  "dependencies": {
+    "ai": "^2.2.27",
+    "dotenv": "^16.3.1",
+    "llamaindex": "0.0.46",
+    "next": "^14.0.4",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "supports-color": "^9.4.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.10.3",
+    "@types/react": "^18.2.42",
+    "@types/react-dom": "^18.2.17",
+    "autoprefixer": "^10.4.16",
+    "eslint": "^8.55.0",
+    "eslint-config-next": "^14.0.3",
+    "postcss": "^8.4.32",
+    "tailwindcss": "^3.3.6",
+    "typescript": "^5.3.2",
+    "cross-env": "^7.0.3"
+  }
+}

apps/demo-edge-runtime/postcss.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+};

apps/demo-edge-runtime/tailwind.config.ts

@@ -0,0 +1,78 @@
+import type { Config } from "tailwindcss";
+import { fontFamily } from "tailwindcss/defaultTheme";
+
+const config: Config = {
+  darkMode: ["class"],
+  content: ["app/**/*.{ts,tsx}", "components/**/*.{ts,tsx}"],
+  theme: {
+    container: {
+      center: true,
+      padding: "2rem",
+      screens: {
+        "2xl": "1400px",
+      },
+    },
+    extend: {
+      colors: {
+        border: "hsl(var(--border))",
+        input: "hsl(var(--input))",
+        ring: "hsl(var(--ring))",
+        background: "hsl(var(--background))",
+        foreground: "hsl(var(--foreground))",
+        primary: {
+          DEFAULT: "hsl(var(--primary))",
+          foreground: "hsl(var(--primary-foreground))",
+        },
+        secondary: {
+          DEFAULT: "hsl(var(--secondary))",
+          foreground: "hsl(var(--secondary-foreground))",
+        },
+        destructive: {
+          DEFAULT: "hsl(var(--destructive) / <alpha-value>)",
+          foreground: "hsl(var(--destructive-foreground) / <alpha-value>)",
+        },
+        muted: {
+          DEFAULT: "hsl(var(--muted))",
+          foreground: "hsl(var(--muted-foreground))",
+        },
+        accent: {
+          DEFAULT: "hsl(var(--accent))",
+          foreground: "hsl(var(--accent-foreground))",
+        },
+        popover: {
+          DEFAULT: "hsl(var(--popover))",
+          foreground: "hsl(var(--popover-foreground))",
+        },
+        card: {
+          DEFAULT: "hsl(var(--card))",
+          foreground: "hsl(var(--card-foreground))",
+        },
+      },
+      borderRadius: {
+        xl: `calc(var(--radius) + 4px)`,
+        lg: `var(--radius)`,
+        md: `calc(var(--radius) - 2px)`,
+        sm: "calc(var(--radius) - 4px)",
+      },
+      fontFamily: {
+        sans: ["var(--font-sans)", ...fontFamily.sans],
+      },
+      keyframes: {
+        "accordion-down": {
+          from: { height: "0" },
+          to: { height: "var(--radix-accordion-content-height)" },
+        },
+        "accordion-up": {
+          from: { height: "var(--radix-accordion-content-height)" },
+          to: { height: "0" },
+        },
+      },
+      animation: {
+        "accordion-down": "accordion-down 0.2s ease-out",
+        "accordion-up": "accordion-up 0.2s ease-out",
+      },
+    },
+  },
+  plugins: [],
+};
+export default config;

apps/demo-edge-runtime/tsconfig.json

@@ -0,0 +1,28 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ],
+    "paths": {
+      "@/*": ["./*"]
+    },
+    "forceConsistentCasingInFileNames": true
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "exclude": ["node_modules"]
+}

apps/docs/.gitignore

@@ -7,6 +7,7 @@
 # Generated files
 .docusaurus
 .cache-loader
+lib
 
 # Misc
 .DS_Store

apps/docs/docs/modules/low_level/response_synthesizer.md

@@ -35,13 +35,26 @@ const nodesWithScore: NodeWithScore[] = [
   },
 ];
 
-const response = await responseSynthesizer.synthesize(
-  "What age am I?",
+const response = await responseSynthesizer.synthesize({
+  query: "What age am I?",
   nodesWithScore,
-);
+});
 console.log(response.response);
 ```
 
+The `synthesize` function also supports streaming, just add `stream: true` as an option:
+
+```typescript
+const stream = await responseSynthesizer.synthesize({
+  query: "What age am I?",
+  nodesWithScore,
+  stream: true,
+});
+for await (const chunk of stream) {
+  process.stdout.write(chunk.response);
+}
+```
+
 ## API Reference
 
 - [ResponseSynthesizer](../../api/classes/ResponseSynthesizer.md)

apps/docs/docs/modules/observability/_category_.yml

@@ -0,0 +1 @@
+label: Observability

apps/docs/docs/modules/observability/index.md

@@ -0,0 +1,35 @@
+# Observability
+
+LlamaIndex provides **one-click observability** 🔭 to allow you to build principled LLM applications in a production setting.
+
+A key requirement for principled development of LLM applications over your data (RAG systems, agents) is being able to observe, debug, and evaluate
+your system - both as a whole and for each component.
+
+This feature allows you to seamlessly integrate the LlamaIndex library with powerful observability/evaluation tools offered by our partners.
+Configure a variable once, and you'll be able to do things like the following:
+
+- View LLM/prompt inputs/outputs
+- Ensure that the outputs of any component (LLMs, embeddings) are performing as expected
+- View call traces for both indexing and querying
+
+Each provider has similarities and differences. Take a look below for the full set of guides for each one!
+
+## OpenLLMetry
+
+[OpenLLMetry](https://github.com/traceloop/openllmetry-js) is an open-source project based on OpenTelemetry for tracing and monitoring
+LLM applications. It connects to [all major observability platforms](https://www.traceloop.com/docs/openllmetry/integrations/introduction) and installs in minutes.
+
+### Usage Pattern
+
+```bash
+npm install @traceloop/node-server-sdk
+```
+
+```js
+import * as traceloop from "@traceloop/node-server-sdk";
+
+traceloop.initialize({
+  apiKey: process.env.TRACELOOP_API_KEY,
+  disableBatch: true,
+});
+```

apps/docs/package.json

@@ -28,8 +28,10 @@
   },
   "devDependencies": {
     "@docusaurus/module-type-aliases": "2.4.3",
+    "@docusaurus/theme-classic": "^2.4.3",
     "@docusaurus/types": "^2.4.3",
     "@tsconfig/docusaurus": "^2.0.1",
+    "@types/node": "^18.19.6",
     "docusaurus-plugin-typedoc": "^0.19.2",
     "typedoc": "^0.24.8",
     "typedoc-plugin-markdown": "^3.16.0",

apps/docs/tsconfig.json

@@ -1,7 +1,11 @@
 {
   // This file is not used in compilation. It is here just for a nice editor experience.
-  "extends": "@tsconfig/docusaurus/tsconfig.json",
+  "extends": "./node_modules/@tsconfig/docusaurus/tsconfig.json",
   "compilerOptions": {
-    "baseUrl": "."
+    "baseUrl": ".",
+    "composite": true,
+    "incremental": true,
+    "outDir": "./lib",
+    "tsBuildInfoFile": "./lib/.tsbuildinfo"
   }
 }

examples/anthropic.ts

@@ -2,13 +2,15 @@ import { Anthropic } from "llamaindex";
 
 (async () => {
   const anthropic = new Anthropic();
-  const result = await anthropic.chat([
-    { content: "You want to talk in rhymes.", role: "system" },
-    {
-      content:
-        "How much wood would a woodchuck chuck if a woodchuck could chuck wood?",
-      role: "user",
-    },
-  ]);
+  const result = await anthropic.chat({
+    messages: [
+      { content: "You want to talk in rhymes.", role: "system" },
+      {
+        content:
+          "How much wood would a woodchuck chuck if a woodchuck could chuck wood?",
+        role: "user",
+      },
+    ],
+  });
   console.log(result);
 })();

examples/gptllama.ts

@@ -25,12 +25,12 @@ import { ChatMessage, LlamaDeuce, OpenAI } from "llamaindex";
 
   while (true) {
     const next = history.length % 2 === 1 ? gpt4 : l2;
-    const r = await next.chat(
-      history.map(({ content, role }) => ({
+    const r = await next.chat({
+      messages: history.map(({ content, role }) => ({
         content,
         role: next === l2 ? role : role === "user" ? "assistant" : "user",
       })),
-    );
+    });
     history.push({
       content: r.message.content,
       role: next === l2 ? "assistant" : "user",

examples/jsonExtract.ts

@@ -21,18 +21,20 @@ async function main() {
     action_items: ["action item 1", "action item 2"],
   };
 
-  const response = await llm.chat([
-    {
-      role: "system",
-      content: `You are an expert assistant for summarizing and extracting insights from sales call transcripts.\n\nGenerate a valid JSON in the following format:\n\n${JSON.stringify(
-        example,
-      )}`,
-    },
-    {
-      role: "user",
-      content: `Here is the transcript: \n------\n${transcript}\n------`,
-    },
-  ]);
+  const response = await llm.chat({
+    messages: [
+      {
+        role: "system",
+        content: `You are an expert assistant for summarizing and extracting insights from sales call transcripts.\n\nGenerate a valid JSON in the following format:\n\n${JSON.stringify(
+          example,
+        )}`,
+      },
+      {
+        role: "user",
+        content: `Here is the transcript: \n------\n${transcript}\n------`,
+      },
+    ],
+  });
 
   const json = JSON.parse(response.message.content);
 

examples/llamadeuce.ts

@@ -2,6 +2,8 @@ import { DeuceChatStrategy, LlamaDeuce } from "llamaindex";
 
 (async () => {
   const deuce = new LlamaDeuce({ chatStrategy: DeuceChatStrategy.META });
-  const result = await deuce.chat([{ content: "Hello, world!", role: "user" }]);
+  const result = await deuce.chat({
+    messages: [{ content: "Hello, world!", role: "user" }],
+  });
   console.log(result);
 })();

examples/lowlevel.ts

@@ -27,9 +27,12 @@ import {
     },
   ];
 
-  const response = await responseSynthesizer.synthesize(
-    "What age am I?",
+  const stream = await responseSynthesizer.synthesize({
+    query: "What age am I?",
     nodesWithScore,
-  );
-  console.log(response.response);
+    stream: true,
+  });
+  for await (const chunk of stream) {
+    process.stdout.write(chunk.response);
+  }
 })();

examples/mistral.ts

@@ -43,19 +43,20 @@ async function rag(llm: LLM, embedModel: BaseEmbedding, query: string) {
 
   // chat api (non-streaming)
   const llm = new MistralAI({ model: "mistral-tiny" });
-  const response = await llm.chat([
-    { content: "What is the best French cheese?", role: "user" },
-  ]);
+  const response = await llm.chat({
+    messages: [{ content: "What is the best French cheese?", role: "user" }],
+  });
   console.log(response.message.content);
 
   // chat api (streaming)
-  const stream = await llm.chat(
-    [{ content: "Who is the most renowned French painter?", role: "user" }],
-    undefined,
-    true,
-  );
+  const stream = await llm.chat({
+    messages: [
+      { content: "Who is the most renowned French painter?", role: "user" },
+    ],
+    stream: true,
+  });
   for await (const chunk of stream) {
-    process.stdout.write(chunk);
+    process.stdout.write(chunk.delta);
   }
 
   // rag

examples/ollama.ts

@@ -3,32 +3,34 @@ import { Ollama } from "llamaindex";
 (async () => {
   const llm = new Ollama({ model: "llama2", temperature: 0.75 });
   {
-    const response = await llm.chat([
-      { content: "Tell me a joke.", role: "user" },
-    ]);
+    const response = await llm.chat({
+      messages: [{ content: "Tell me a joke.", role: "user" }],
+    });
     console.log("Response 1:", response.message.content);
   }
   {
-    const response = await llm.complete("How are you?");
-    console.log("Response 2:", response.message.content);
+    const response = await llm.complete({ prompt: "How are you?" });
+    console.log("Response 2:", response.text);
   }
   {
-    const response = await llm.chat(
-      [{ content: "Tell me a joke.", role: "user" }],
-      undefined,
-      true,
-    );
+    const response = await llm.chat({
+      messages: [{ content: "Tell me a joke.", role: "user" }],
+      stream: true,
+    });
     console.log("Response 3:");
     for await (const message of response) {
-      process.stdout.write(message); // no newline
+      process.stdout.write(message.delta); // no newline
     }
     console.log(); // newline
   }
   {
-    const response = await llm.complete("How are you?", undefined, true);
+    const response = await llm.complete({
+      prompt: "How are you?",
+      stream: true,
+    });
     console.log("Response 4:");
     for await (const message of response) {
-      process.stdout.write(message); // no newline
+      process.stdout.write(message.text); // no newline
     }
     console.log(); // newline
   }

examples/openai.ts

@@ -4,12 +4,12 @@ import { OpenAI } from "llamaindex";
   const llm = new OpenAI({ model: "gpt-4-1106-preview", temperature: 0.1 });
 
   // complete api
-  const response1 = await llm.complete("How are you?");
-  console.log(response1.message.content);
+  const response1 = await llm.complete({ prompt: "How are you?" });
+  console.log(response1.text);
 
   // chat api
-  const response2 = await llm.chat([
-    { content: "Tell me a joke.", role: "user" },
-  ]);
+  const response2 = await llm.chat({
+    messages: [{ content: "Tell me a joke.", role: "user" }],
+  });
   console.log(response2.message.content);
 })();

examples/package.json

@@ -4,7 +4,7 @@
   "name": "examples",
   "dependencies": {
     "@datastax/astra-db-ts": "^0.1.2",
-    "@notionhq/client": "^2.2.13",
+    "@notionhq/client": "^2.2.14",
     "@pinecone-database/pinecone": "^1.1.2",
     "chromadb": "^1.7.3",
     "commander": "^11.1.0",

examples/portkey.ts

@@ -1,7 +1,6 @@
 import { Portkey } from "llamaindex";
 
 (async () => {
-  const llms = [{}];
   const portkey = new Portkey({
     mode: "single",
     llms: [
@@ -13,11 +12,14 @@ import { Portkey } from "llamaindex";
       },
     ],
   });
-  const result = portkey.stream_chat([
-    { role: "system", content: "You are a helpful assistant." },
-    { role: "user", content: "Tell me a joke." },
-  ]);
+  const result = await portkey.chat({
+    messages: [
+      { role: "system", content: "You are a helpful assistant." },
+      { role: "user", content: "Tell me a joke." },
+    ],
+    stream: true,
+  });
   for await (const res of result) {
-    process.stdout.write(res);
+    process.stdout.write(res.delta);
   }
 })();

examples/together-ai.ts

@@ -0,0 +1,29 @@
+import { TogetherEmbedding, TogetherLLM } from "llamaindex";
+
+// process.env.TOGETHER_API_KEY is required
+const together = new TogetherLLM({
+  model: "mistralai/Mixtral-8x7B-Instruct-v0.1",
+});
+
+(async () => {
+  const generator = await together.chat({
+    messages: [
+      {
+        role: "system",
+        content: "You are an AI assistant",
+      },
+      {
+        role: "user",
+        content: "Tell me about San Francisco",
+      },
+    ],
+    stream: true,
+  });
+  console.log("Chatting with Together AI...");
+  for await (const message of generator) {
+    process.stdout.write(message.delta);
+  }
+  const embedding = new TogetherEmbedding();
+  const vector = await embedding.getTextEmbedding("Hello world!");
+  console.log("vector:", vector);
+})();

examples/together-ai/vector-index.ts

@@ -0,0 +1,39 @@
+import fs from "node:fs/promises";
+
+import {
+  Document,
+  TogetherEmbedding,
+  TogetherLLM,
+  VectorStoreIndex,
+  serviceContextFromDefaults,
+} from "llamaindex";
+
+async function main() {
+  const apiKey = process.env.TOGETHER_API_KEY;
+  if (!apiKey) {
+    throw new Error("Missing TOGETHER_API_KEY");
+  }
+  const path = require.resolve("llamaindex/examples/abramov.txt");
+  const essay = await fs.readFile(path, "utf-8");
+
+  const document = new Document({ text: essay, id_: path });
+
+  const serviceContext = serviceContextFromDefaults({
+    llm: new TogetherLLM({ model: "mistralai/Mixtral-8x7B-Instruct-v0.1" }),
+    embedModel: new TogetherEmbedding(),
+  });
+
+  const index = await VectorStoreIndex.fromDocuments([document], {
+    serviceContext,
+  });
+
+  const queryEngine = index.asQueryEngine();
+
+  const response = await queryEngine.query(
+    "What did the author do in college?",
+  );
+
+  console.log(response.toString());
+}
+
+main().catch(console.error);

examples/tsconfig.json

@@ -1,4 +1,5 @@
 {
+  "extends": "../tsconfig.json",
   "compilerOptions": {
     "target": "es2016",
     "module": "commonjs",

examples/vision.ts

@@ -4,12 +4,12 @@ import { OpenAI } from "llamaindex";
   const llm = new OpenAI({ model: "gpt-4-vision-preview", temperature: 0.1 });
 
   // complete api
-  const response1 = await llm.complete("How are you?");
-  console.log(response1.message.content);
+  const response1 = await llm.complete({ prompt: "How are you?" });
+  console.log(response1.text);
 
   // chat api
-  const response2 = await llm.chat([
-    { content: "Tell me a joke!", role: "user" },
-  ]);
+  const response2 = await llm.chat({
+    messages: [{ content: "Tell me a joke!", role: "user" }],
+  });
   console.log(response2.message.content);
 })();

package.json

@@ -8,6 +8,7 @@
     "lint": "turbo run lint",
     "prepare": "husky install",
     "test": "turbo run test",
+    "type-check": "tsc -b --diagnostics",
     "new-version": "turbo run build lint test --filter=\"!docs\" && changeset version",
     "new-snapshot": "turbo run build lint test --filter=\"!docs\" && changeset version --snapshot"
   },
@@ -23,7 +24,8 @@
     "prettier": "^3.1.1",
     "prettier-plugin-organize-imports": "^3.2.4",
     "ts-jest": "^29.1.1",
-    "turbo": "^1.11.2"
+    "turbo": "^1.11.2",
+    "typescript": "^5.3.3"
   },
   "packageManager": "pnpm@8.10.5+sha256.a4bd9bb7b48214bbfcd95f264bd75bb70d100e5d4b58808f5cd6ab40c6ac21c5",
   "pnpm": {

packages/core/CHANGELOG.md

@@ -1,5 +1,19 @@
 # llamaindex
 
+## 0.0.46
+
+### Patch Changes
+
+- 977f284: fixing import statement
+- 5d3bb66: fix: class SimpleKVStore might throw error in ES module
+- f18c9f6: refactor: Updated low-level streaming interface
+
+## 0.0.45
+
+### Patch Changes
+
+- 2e6b36e: feat: support together AI
+
 ## 0.0.44
 
 ### Patch Changes

packages/core/README.md

@@ -1,5 +1,10 @@
 # LlamaIndex.TS
 
+[![NPM Version](https://img.shields.io/npm/v/llamaindex)](https://www.npmjs.com/package/llamaindex)
+[![NPM License](https://img.shields.io/npm/l/llamaindex)](https://www.npmjs.com/package/llamaindex)
+[![NPM Downloads](https://img.shields.io/npm/dm/llamaindex)](https://www.npmjs.com/package/llamaindex)
+[![Discord](https://img.shields.io/discord/1059199217496772688)](https://discord.com/invite/eN6D2HQ4aX)
+
 LlamaIndex is a data framework for your LLM application.
 
 Use your own data with large language models (LLMs, OpenAI ChatGPT and others) in Typescript and Javascript.
@@ -12,7 +17,7 @@ LlamaIndex.TS aims to be a lightweight, easy to use set of libraries to help you
 
 ## Getting started with an example:
 
-LlamaIndex.TS requries Node v18 or higher. You can download it from https://nodejs.org or use https://nvm.sh (our preferred option).
+LlamaIndex.TS requires Node v18 or higher. You can download it from https://nodejs.org or use https://nvm.sh (our preferred option).
 
 In a new folder:
 
@@ -84,11 +89,38 @@ Check out our NextJS playground at https://llama-playground.vercel.app/. The sou
 
 - [SimplePrompt](/packages/core/src/Prompt.ts): A simple standardized function call definition that takes in inputs and formats them in a template literal. SimplePrompts can be specialized using currying and combined using other SimplePrompt functions.
 
+## Note: NextJS:
+
+If you're using NextJS App Router, you'll need to use the NodeJS runtime (default) and add the following config to your next.config.js to have it use imports/exports in the same way Node does.
+
+```js
+export const runtime = "nodejs"; // default
+```
+
+```js
+// next.config.js
+/** @type {import('next').NextConfig} */
+const nextConfig = {
+  webpack: (config) => {
+    config.resolve.alias = {
+      ...config.resolve.alias,
+      sharp$: false,
+      "onnxruntime-node$": false,
+      mongodb$: false,
+    };
+    return config;
+  },
+};
+
+module.exports = nextConfig;
+```
+
 ## Supported LLMs:
 
 - OpenAI GPT-3.5-turbo and GPT-4
 - Anthropic Claude Instant and Claude 2
 - Llama2 Chat LLMs (70B, 13B, and 7B parameters)
+- MistralAI Chat LLMs
 
 ## Contributing:
 

packages/core/package.json

@@ -1,9 +1,10 @@
 {
   "name": "llamaindex",
-  "version": "0.0.44",
+  "version": "0.0.46",
   "license": "MIT",
   "dependencies": {
     "@anthropic-ai/sdk": "^0.9.1",
+    "@aws-crypto/sha256-browser": "^5.2.0",
     "@datastax/astra-db-ts": "^0.1.2",
     "@mistralai/mistralai": "^0.0.7",
     "@notionhq/client": "^2.2.14",
@@ -27,31 +28,192 @@
     "rake-modified": "^1.0.8",
     "replicate": "^0.21.1",
     "string-strip-html": "^13.4.3",
-    "uuid": "^9.0.1",
     "wink-nlp": "^1.14.3"
   },
   "devDependencies": {
     "@types/jest": "^29.5.11",
     "@types/lodash": "^4.14.202",
-    "@types/node": "^18.19.2",
+    "@types/node": "^18.19.6",
     "@types/papaparse": "^5.3.14",
     "@types/pg": "^8.10.9",
-    "@types/uuid": "^9.0.7",
+    "bunchee": "^4.4.0",
+    "madge": "^6.1.0",
     "node-stdlib-browser": "^1.2.0",
-    "tsup": "^7.2.0",
-    "typescript": "^5.3.2"
+    "typescript": "^5.3.3"
   },
   "engines": {
     "node": ">=18.0.0"
   },
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",
-  "module": "./dist/index.mjs",
-  "repository": "run-llama/LlamaIndexTS",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "edge-light": "./dist/index.edge-light.mjs",
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.js"
+    },
+    "./storage/FileSystem": {
+      "types": "./dist/storage/FileSystem.d.mts",
+      "edge-light": "./dist/storage/FileSystem.edge-light.mjs",
+      "import": "./dist/storage/FileSystem.mjs",
+      "require": "./dist/storage/FileSystem.js"
+    },
+    "./ChatEngine": {
+      "types": "./dist/ChatEngine.d.mts",
+      "import": "./dist/ChatEngine.mjs",
+      "require": "./dist/ChatEngine.js"
+    },
+    "./ChatHistory": {
+      "types": "./dist/ChatHistory.d.mts",
+      "import": "./dist/ChatHistory.mjs",
+      "require": "./dist/ChatHistory.js"
+    },
+    "./constants": {
+      "types": "./dist/constants.d.mts",
+      "import": "./dist/constants.mjs",
+      "require": "./dist/constants.js"
+    },
+    "./GlobalsHelper": {
+      "types": "./dist/GlobalsHelper.d.mts",
+      "import": "./dist/GlobalsHelper.mjs",
+      "require": "./dist/GlobalsHelper.js"
+    },
+    "./Node": {
+      "types": "./dist/Node.d.mts",
+      "import": "./dist/Node.mjs",
+      "require": "./dist/Node.js"
+    },
+    "./OutputParser": {
+      "types": "./dist/OutputParser.d.mts",
+      "import": "./dist/OutputParser.mjs",
+      "require": "./dist/OutputParser.js"
+    },
+    "./Prompt": {
+      "types": "./dist/Prompt.d.mts",
+      "import": "./dist/Prompt.mjs",
+      "require": "./dist/Prompt.js"
+    },
+    "./PromptHelper": {
+      "types": "./dist/PromptHelper.d.mts",
+      "import": "./dist/PromptHelper.mjs",
+      "require": "./dist/PromptHelper.js"
+    },
+    "./QueryEngine": {
+      "types": "./dist/QueryEngine.d.mts",
+      "import": "./dist/QueryEngine.mjs",
+      "require": "./dist/QueryEngine.js"
+    },
+    "./QuestionGenerator": {
+      "types": "./dist/QuestionGenerator.d.mts",
+      "import": "./dist/QuestionGenerator.mjs",
+      "require": "./dist/QuestionGenerator.js"
+    },
+    "./Response": {
+      "types": "./dist/Response.d.mts",
+      "import": "./dist/Response.mjs",
+      "require": "./dist/Response.js"
+    },
+    "./Retriever": {
+      "types": "./dist/Retriever.d.mts",
+      "import": "./dist/Retriever.mjs",
+      "require": "./dist/Retriever.js"
+    },
+    "./ServiceContext": {
+      "types": "./dist/ServiceContext.d.mts",
+      "import": "./dist/ServiceContext.mjs",
+      "require": "./dist/ServiceContext.js"
+    },
+    "./TextSplitter": {
+      "types": "./dist/TextSplitter.d.mts",
+      "import": "./dist/TextSplitter.mjs",
+      "require": "./dist/TextSplitter.js"
+    },
+    "./Tool": {
+      "types": "./dist/Tool.d.mts",
+      "import": "./dist/Tool.mjs",
+      "require": "./dist/Tool.js"
+    },
+    "./readers/AssemblyAI": {
+      "types": "./dist/readers/AssemblyAI.d.mts",
+      "import": "./dist/readers/AssemblyAI.mjs",
+      "require": "./dist/readers/AssemblyAI.js"
+    },
+    "./readers/base": {
+      "types": "./dist/readers/base.d.mts",
+      "import": "./dist/readers/base.mjs",
+      "require": "./dist/readers/base.js"
+    },
+    "./readers/CSVReader": {
+      "types": "./dist/readers/CSVReader.d.mts",
+      "import": "./dist/readers/CSVReader.mjs",
+      "require": "./dist/readers/CSVReader.js"
+    },
+    "./readers/DocxReader": {
+      "types": "./dist/readers/DocxReader.d.mts",
+      "import": "./dist/readers/DocxReader.mjs",
+      "require": "./dist/readers/DocxReader.js"
+    },
+    "./readers/HTMLReader": {
+      "types": "./dist/readers/HTMLReader.d.mts",
+      "import": "./dist/readers/HTMLReader.mjs",
+      "require": "./dist/readers/HTMLReader.js"
+    },
+    "./readers/ImageReader": {
+      "types": "./dist/readers/ImageReader.d.mts",
+      "import": "./dist/readers/ImageReader.mjs",
+      "require": "./dist/readers/ImageReader.js"
+    },
+    "./readers/MarkdownReader": {
+      "types": "./dist/readers/MarkdownReader.d.mts",
+      "import": "./dist/readers/MarkdownReader.mjs",
+      "require": "./dist/readers/MarkdownReader.js"
+    },
+    "./readers/NotionReader": {
+      "types": "./dist/readers/NotionReader.d.mts",
+      "import": "./dist/readers/NotionReader.mjs",
+      "require": "./dist/readers/NotionReader.js"
+    },
+    "./readers/PDFReader": {
+      "types": "./dist/readers/PDFReader.d.mts",
+      "import": "./dist/readers/PDFReader.mjs",
+      "require": "./dist/readers/PDFReader.js"
+    },
+    "./readers/SimpleDirectoryReader": {
+      "types": "./dist/readers/SimpleDirectoryReader.d.mts",
+      "import": "./dist/readers/SimpleDirectoryReader.mjs",
+      "require": "./dist/readers/SimpleDirectoryReader.js"
+    },
+    "./readers/SimpleMongoReader": {
+      "types": "./dist/readers/SimpleMongoReader.d.mts",
+      "import": "./dist/readers/SimpleMongoReader.mjs",
+      "require": "./dist/readers/SimpleMongoReader.js"
+    },
+    "./environments": {
+      "types": "./dist/environments.d.mts",
+      "edge-light": "./dist/environments.edge-light.mjs",
+      "import": "./dist/environments.mjs",
+      "require": "./dist/environments.js"
+    },
+    "./examples/*": "./examples/*"
+  },
+  "files": [
+    "dist",
+    "examples",
+    "src",
+    "types",
+    "CHANGELOG.md"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/run-llama/LlamaIndexTS.git",
+    "directory": "packages/core"
+  },
   "scripts": {
     "lint": "eslint .",
     "test": "jest",
-    "build": "tsup src/index.ts --format esm,cjs --dts",
-    "dev": "tsup src/index.ts --format esm,cjs --dts --watch"
+    "build": "bunchee",
+    "dev": "bunchee -w",
+    "check:circular": "madge --circular ./src/index.ts"
   }
 }

packages/core/src/ChatEngine.ts

@@ -1,4 +1,3 @@
-import { v4 as uuidv4 } from "uuid";
 import { ChatHistory } from "./ChatHistory";
 import { NodeWithScore, TextNode } from "./Node";
 import {
@@ -13,7 +12,9 @@ import { Response } from "./Response";
 import { BaseRetriever } from "./Retriever";
 import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
 import { Event } from "./callbacks/CallbackManager";
-import { ChatMessage, LLM, OpenAI } from "./llm";
+import { randomUUID } from "./environments";
+import { ChatMessage, LLM } from "./llm";
+import { OpenAI } from "./llm/openai";
 import { BaseNodePostprocessor } from "./postprocessors";
 
 /**
@@ -69,7 +70,7 @@ export class SimpleChatEngine implements ChatEngine {
     //Non-streaming option
     chatHistory = chatHistory ?? this.chatHistory;
     chatHistory.push({ content: message, role: "user" });
-    const response = await this.llm.chat(chatHistory, undefined);
+    const response = await this.llm.chat({ messages: chatHistory });
     chatHistory.push(response.message);
     this.chatHistory = chatHistory;
     return new Response(response.message.content) as R;
@@ -81,16 +82,15 @@ export class SimpleChatEngine implements ChatEngine {
   ): AsyncGenerator<string, void, unknown> {
     chatHistory = chatHistory ?? this.chatHistory;
     chatHistory.push({ content: message, role: "user" });
-    const response_generator = await this.llm.chat(
-      chatHistory,
-      undefined,
-      true,
-    );
+    const response_generator = await this.llm.chat({
+      messages: chatHistory,
+      stream: true,
+    });
 
     var accumulator: string = "";
     for await (const part of response_generator) {
-      accumulator += part;
-      yield part;
+      accumulator += part.delta;
+      yield part.delta;
     }
 
     chatHistory.push({ content: accumulator, role: "assistant" });
@@ -136,12 +136,12 @@ export class CondenseQuestionChatEngine implements ChatEngine {
   private async condenseQuestion(chatHistory: ChatMessage[], question: string) {
     const chatHistoryStr = messagesToHistoryStr(chatHistory);
 
-    return this.serviceContext.llm.complete(
-      defaultCondenseQuestionPrompt({
+    return this.serviceContext.llm.complete({
+      prompt: defaultCondenseQuestionPrompt({
         question: question,
         chatHistory: chatHistoryStr,
       }),
-    );
+    });
   }
 
   async chat<
@@ -156,7 +156,7 @@ export class CondenseQuestionChatEngine implements ChatEngine {
 
     const condensedQuestion = (
       await this.condenseQuestion(chatHistory, extractText(message))
-    ).message.content;
+    ).text;
 
     const response = await this.queryEngine.query(condensedQuestion);
 
@@ -206,7 +206,7 @@ export class DefaultContextGenerator implements ContextGenerator {
   async generate(message: string, parentEvent?: Event): Promise<Context> {
     if (!parentEvent) {
       parentEvent = {
-        id: uuidv4(),
+        id: randomUUID(),
         type: "wrapper",
         tags: ["final"],
       };
@@ -272,7 +272,7 @@ export class ContextChatEngine implements ChatEngine {
     }
 
     const parentEvent: Event = {
-      id: uuidv4(),
+      id: randomUUID(),
       type: "wrapper",
       tags: ["final"],
     };
@@ -283,10 +283,10 @@ export class ContextChatEngine implements ChatEngine {
 
     chatHistory.push({ content: message, role: "user" });
 
-    const response = await this.chatModel.chat(
-      [context.message, ...chatHistory],
+    const response = await this.chatModel.chat({
+      messages: [context.message, ...chatHistory],
       parentEvent,
-    );
+    });
     chatHistory.push(response.message);
 
     this.chatHistory = chatHistory;
@@ -304,7 +304,7 @@ export class ContextChatEngine implements ChatEngine {
     chatHistory = chatHistory ?? this.chatHistory;
 
     const parentEvent: Event = {
-      id: uuidv4(),
+      id: randomUUID(),
       type: "wrapper",
       tags: ["final"],
     };
@@ -315,15 +315,15 @@ export class ContextChatEngine implements ChatEngine {
 
     chatHistory.push({ content: message, role: "user" });
 
-    const response_stream = await this.chatModel.chat(
-      [context.message, ...chatHistory],
+    const response_stream = await this.chatModel.chat({
+      messages: [context.message, ...chatHistory],
       parentEvent,
-      true,
-    );
+      stream: true,
+    });
     var accumulator: string = "";
     for await (const part of response_stream) {
-      accumulator += part;
-      yield part;
+      accumulator += part.delta;
+      yield part.delta;
     }
 
     chatHistory.push({ content: accumulator, role: "assistant" });
@@ -399,7 +399,7 @@ export class HistoryChatEngine {
       message,
       chatHistory,
     );
-    const response = await this.llm.chat(requestMessages);
+    const response = await this.llm.chat({ messages: requestMessages });
     chatHistory.addMessage(response.message);
     return new Response(response.message.content) as R;
   }
@@ -412,16 +412,15 @@ export class HistoryChatEngine {
       message,
       chatHistory,
     );
-    const response_stream = await this.llm.chat(
-      requestMessages,
-      undefined,
-      true,
-    );
+    const response_stream = await this.llm.chat({
+      messages: requestMessages,
+      stream: true,
+    });
 
     var accumulator = "";
     for await (const part of response_stream) {
-      accumulator += part;
-      yield part;
+      accumulator += part.delta;
+      yield part.delta;
     }
     chatHistory.addMessage({
       content: accumulator,

packages/core/src/ChatHistory.ts

@@ -1,9 +1,10 @@
-import { ChatMessage, LLM, MessageType, OpenAI } from "./llm/LLM";
 import {
+  SummaryPrompt,
   defaultSummaryPrompt,
   messagesToHistoryStr,
-  SummaryPrompt,
 } from "./Prompt";
+import { ChatMessage, LLM, MessageType } from "./llm/LLM";
+import { OpenAI } from "./llm/openai";
 
 /**
  * A ChatHistory is used to keep the state of back and forth chat messages
@@ -99,7 +100,7 @@ export class SummaryChatHistory implements ChatHistory {
       messagesToSummarize.shift();
     } while (this.llm.tokens(promptMessages) > this.tokensToSummarize);
 
-    const response = await this.llm.chat(promptMessages);
+    const response = await this.llm.chat({ messages: promptMessages });
     return { content: response.message.content, role: "memory" };
   }
 

packages/core/src/GlobalsHelper.ts

@@ -1,7 +1,7 @@
 import { encodingForModel } from "js-tiktoken";
 
-import { v4 as uuidv4 } from "uuid";
 import { Event, EventTag, EventType } from "./callbacks/CallbackManager";
+import { randomUUID } from "./environments";
 
 export enum Tokenizers {
   CL100K_BASE = "cl100k_base",
@@ -64,7 +64,7 @@ class GlobalsHelper {
     tags?: EventTag[];
   }): Event {
     return {
-      id: uuidv4(),
+      id: randomUUID(),
       type,
       // inherit parent tags if tags not set
       tags: tags || parentEvent?.tags,

packages/core/src/Node.ts

@@ -1,7 +1,5 @@
 import _ from "lodash";
-import { createHash } from "node:crypto";
-import path from "path";
-import { v4 as uuidv4 } from "uuid";
+import { createSHA256, randomUUID } from "./environments";
 
 export enum NodeRelationship {
   SOURCE = "SOURCE",
@@ -49,7 +47,7 @@ export abstract class BaseNode<T extends Metadata = Metadata> {
    *
    * Set to a UUID by default.
    */
-  id_: string = uuidv4();
+  id_: string = randomUUID();
   embedding?: number[];
 
   // Metadata fields
@@ -192,7 +190,7 @@ export class TextNode<T extends Metadata = Metadata> extends BaseNode<T> {
    * @returns
    */
   generateHash() {
-    const hashFunction = createHash("sha256");
+    const hashFunction = createSHA256();
     hashFunction.update(`type=${this.getType()}`);
     hashFunction.update(
       `startCharIdx=${this.startCharIdx} endCharIdx=${this.endCharIdx}`,
@@ -322,9 +320,12 @@ export class ImageNode<T extends Metadata = Metadata> extends TextNode<T> {
   }
 
   getUrl(): URL {
-    // id_ stores the relative path, convert it to the URL of the file
-    const absPath = path.resolve(this.id_);
-    return new URL(`file://${absPath}`);
+    if (typeof this.image === "string") {
+      return new URL(this.image);
+    } else if (this.image instanceof Blob) {
+      return new URL(URL.createObjectURL(this.image));
+    }
+    throw new Error("Invalid image type");
   }
 }
 

packages/core/src/OutputParser.ts

@@ -1,4 +1,7 @@
-import { SubQuestion } from "./QuestionGenerator";
+export interface SubQuestion {
+  subQuestion: string;
+  toolName: string;
+}
 
 /**
  * An OutputParser is used to extract structured data from the raw output of the LLM.

packages/core/src/Prompt.ts

@@ -1,6 +1,6 @@
 import { ChatMessage } from "./llm/LLM";
-import { SubQuestion } from "./QuestionGenerator";
 import { ToolMetadata } from "./Tool";
+import { SubQuestion } from './OutputParser'
 
 /**
  * A SimplePrompt is a function that takes a dictionary of inputs and returns a string.

packages/core/src/QueryEngine.ts

@@ -1,21 +1,21 @@
-import { v4 as uuidv4 } from "uuid";
 import { NodeWithScore, TextNode } from "./Node";
 import {
   BaseQuestionGenerator,
   LLMQuestionGenerator,
-  SubQuestion,
 } from "./QuestionGenerator";
 import { Response } from "./Response";
 import { BaseRetriever } from "./Retriever";
 import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
 import { QueryEngineTool, ToolMetadata } from "./Tool";
 import { Event } from "./callbacks/CallbackManager";
+import { randomUUID } from "./environments";
 import { BaseNodePostprocessor } from "./postprocessors";
 import {
   BaseSynthesizer,
   CompactAndRefine,
   ResponseSynthesizer,
 } from "./synthesizers";
+import { SubQuestion } from './OutputParser'
 
 /**
  * A query engine is a question answerer that can use one or more steps.
@@ -72,12 +72,16 @@ export class RetrieverQueryEngine implements BaseQueryEngine {
 
   async query(query: string, parentEvent?: Event) {
     const _parentEvent: Event = parentEvent || {
-      id: uuidv4(),
+      id: randomUUID(),
       type: "wrapper",
       tags: ["final"],
     };
-    const nodes = await this.retrieve(query, _parentEvent);
-    return this.responseSynthesizer.synthesize(query, nodes, _parentEvent);
+    const nodesWithScore = await this.retrieve(query, _parentEvent);
+    return this.responseSynthesizer.synthesize({
+      query,
+      nodesWithScore,
+      parentEvent: _parentEvent,
+    });
   }
 }
 
@@ -136,14 +140,14 @@ export class SubQuestionQueryEngine implements BaseQueryEngine {
 
     // groups final retrieval+synthesis operation
     const parentEvent: Event = {
-      id: uuidv4(),
+      id: randomUUID(),
       type: "wrapper",
       tags: ["final"],
     };
 
     // groups all sub-queries
     const subQueryParentEvent: Event = {
-      id: uuidv4(),
+      id: randomUUID(),
       parentId: parentEvent.id,
       type: "wrapper",
       tags: ["intermediate"],
@@ -153,10 +157,14 @@ export class SubQuestionQueryEngine implements BaseQueryEngine {
       subQuestions.map((subQ) => this.querySubQ(subQ, subQueryParentEvent)),
     );
 
-    const nodes = subQNodes
+    const nodesWithScore = subQNodes
       .filter((node) => node !== null)
       .map((node) => node as NodeWithScore);
-    return this.responseSynthesizer.synthesize(query, nodes, parentEvent);
+    return this.responseSynthesizer.synthesize({
+      query,
+      nodesWithScore,
+      parentEvent,
+    });
   }
 
   private async querySubQ(

packages/core/src/QuestionGenerator.ts

@@ -1,20 +1,17 @@
 import {
   BaseOutputParser,
   StructuredOutput,
-  SubQuestionOutputParser,
-} from "./OutputParser";
+  SubQuestion,
+  SubQuestionOutputParser
+} from './OutputParser'
 import {
   SubQuestionPrompt,
   buildToolsText,
   defaultSubQuestionPrompt,
 } from "./Prompt";
 import { ToolMetadata } from "./Tool";
-import { LLM, OpenAI } from "./llm/LLM";
-
-export interface SubQuestion {
-  subQuestion: string;
-  toolName: string;
-}
+import { LLM } from "./llm/LLM";
+import { OpenAI } from "./llm/openai";
 
 /**
  * QuestionGenerators generate new questions for the LLM using tools and a user query.
@@ -41,13 +38,13 @@ export class LLMQuestionGenerator implements BaseQuestionGenerator {
     const toolsStr = buildToolsText(tools);
     const queryStr = query;
     const prediction = (
-      await this.llm.complete(
-        this.prompt({
+      await this.llm.complete({
+        prompt: this.prompt({
           toolsStr,
           queryStr,
         }),
-      )
-    ).message.content;
+      })
+    ).text;
 
     const structuredOutput = this.outputParser.parse(prediction);
 

packages/core/src/Response.ts

@@ -1,7 +1,7 @@
 import { BaseNode } from "./Node";
 
 /**
- * Respone is the output of a LLM
+ * Response is the output of a LLM
  */
 export class Response {
   response: string;

packages/core/src/ServiceContext.ts

@@ -1,8 +1,9 @@
+import { PromptHelper } from "./PromptHelper";
 import { CallbackManager } from "./callbacks/CallbackManager";
 import { BaseEmbedding, OpenAIEmbedding } from "./embeddings";
-import { LLM, OpenAI } from "./llm";
+import { LLM } from "./llm";
+import { OpenAI } from "./llm/openai";
 import { NodeParser, SimpleNodeParser } from "./nodeParsers";
-import { PromptHelper } from "./PromptHelper";
 
 /**
  * The ServiceContext is a collection of components that are used in different parts of the application.

packages/core/src/TextSplitter.ts

@@ -1,7 +1,7 @@
-import { EOL } from "node:os";
 // GitHub translated
 import { globalsHelper } from "./GlobalsHelper";
 import { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE } from "./constants";
+import { EOL } from "./environments";
 
 class TextSplit {
   textChunk: string;

packages/core/src/embeddings/OpenAIEmbedding.ts

@@ -14,7 +14,7 @@ export enum OpenAIEmbeddingModelType {
 }
 
 export class OpenAIEmbedding extends BaseEmbedding {
-  model: OpenAIEmbeddingModelType;
+  model: OpenAIEmbeddingModelType | string;
 
   // OpenAI session params
   apiKey?: string = undefined;

packages/core/src/embeddings/index.ts

@@ -3,5 +3,6 @@ export * from "./HuggingFaceEmbedding";
 export * from "./MistralAIEmbedding";
 export * from "./MultiModalEmbedding";
 export * from "./OpenAIEmbedding";
+export { TogetherEmbedding } from "./together";
 export * from "./types";
 export * from "./utils";

packages/core/src/embeddings/together.ts

@@ -0,0 +1,16 @@
+import { OpenAIEmbedding } from "./OpenAIEmbedding";
+
+export class TogetherEmbedding extends OpenAIEmbedding {
+  override model: string;
+  constructor(init?: Partial<OpenAIEmbedding>) {
+    super({
+      apiKey: process.env.TOGETHER_API_KEY,
+      ...init,
+      additionalSessionOptions: {
+        ...init?.additionalSessionOptions,
+        baseURL: "https://api.together.xyz/v1",
+      },
+    });
+    this.model = init?.model ?? "togethercomputer/m2-bert-80M-32k-retrieval";
+  }
+}

packages/core/src/embeddings/types.ts

@@ -1,14 +1,4 @@
-import { similarity } from "./utils";
-
-/**
- * Similarity type
- * Default is cosine similarity. Dot product and negative Euclidean distance are also supported.
- */
-export enum SimilarityType {
-  DEFAULT = "cosine",
-  DOT_PRODUCT = "dot_product",
-  EUCLIDEAN = "euclidean",
-}
+import { similarity, SimilarityType } from "./utils";
 
 export abstract class BaseEmbedding {
   similarity(

packages/core/src/embeddings/utils.ts

@@ -1,8 +1,18 @@
 import _ from "lodash";
 import { ImageType } from "../Node";
 import { DEFAULT_SIMILARITY_TOP_K } from "../constants";
-import { DEFAULT_FS, VectorStoreQueryMode } from "../storage";
-import { SimilarityType } from "./types";
+import { genericFileSystem } from "../storage/FileSystem";
+import { VectorStoreQueryMode } from "../storage/vectorStore/types";
+
+/**
+ * Similarity type
+ * Default is cosine similarity. Dot product and negative Euclidean distance are also supported.
+ */
+export enum SimilarityType {
+  DEFAULT = "cosine",
+  DOT_PRODUCT = "dot_product",
+  EUCLIDEAN = "euclidean",
+}
 
 /**
  * The similarity between two embeddings.
@@ -186,7 +196,7 @@ export function getTopKMMREmbeddings(
 }
 
 async function blobToDataUrl(input: Blob) {
-  const { fileTypeFromBuffer } = await import("file-type");
+  const { fileTypeFromBuffer } = await import("../environments");
   const buffer = Buffer.from(await input.arrayBuffer());
   const type = await fileTypeFromBuffer(buffer);
   if (!type) {
@@ -241,7 +251,7 @@ export async function imageToDataUrl(input: ImageType): Promise<string> {
     _.isString(input)
   ) {
     // string or file URL
-    const fs = DEFAULT_FS;
+    const fs = genericFileSystem;
     const dataBuffer = await fs.readFile(
       input instanceof URL ? input.pathname : input,
     );

packages/core/src/environments.edge-light.ts

@@ -0,0 +1,48 @@
+import { Sha256 } from "@aws-crypto/sha256-browser";
+
+export const randomUUID = () => {
+  return globalThis.crypto.randomUUID();
+};
+
+export function dirname(path: string) {
+  return path.split("/").at(-2)!;
+}
+
+export function join(...paths: string[]) {
+  return paths.join("/");
+}
+
+export function fileTypeFromBuffer(buffer: Buffer) {
+  if (buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) {
+    return { ext: "jpg", mime: "image/jpeg" };
+  } else if (
+    buffer[0] === 0x89 &&
+    buffer[1] === 0x50 &&
+    buffer[2] === 0x4e &&
+    buffer[3] === 0x47
+  ) {
+    return { ext: "png", mime: "image/png" };
+  } else if (
+    buffer[0] === 0x47 &&
+    buffer[1] === 0x49 &&
+    buffer[2] === 0x46 &&
+    buffer[3] === 0x38
+  ) {
+    return { ext: "gif", mime: "image/gif" };
+  } else if (
+    buffer[0] === 0x42 &&
+    buffer[1] === 0x4d &&
+    buffer[2] === 0x46 &&
+    buffer[3] === 0x38
+  ) {
+    return { ext: "bmp", mime: "image/bmp" };
+  }
+  return null;
+}
+
+export const createSHA256 = () => {
+  return new Sha256();
+};
+
+// on edge runtime, there is always pretend to be a linux system
+export const EOL = "\n";

packages/core/src/environments.ts

@@ -0,0 +1,17 @@
+import { createHash, randomUUID } from "node:crypto";
+import { EOL } from "node:os";
+
+export { randomUUID };
+
+export { dirname, join } from "node:path";
+
+export const createSHA256 = () => {
+  return createHash("sha256");
+};
+
+export async function fileTypeFromBuffer(buffer: Buffer) {
+  const { fileTypeFromBuffer } = await import("file-type")
+  return fileTypeFromBuffer(buffer);
+}
+
+export { EOL };

packages/core/src/index.edge-light.ts

@@ -0,0 +1,35 @@
+export * from "./ChatEngine";
+export * from "./ChatHistory";
+export * from "./GlobalsHelper";
+export * from "./Node";
+export * from "./OutputParser";
+export * from "./Prompt";
+export * from "./PromptHelper";
+export * from "./QueryEngine";
+export * from "./QuestionGenerator";
+export * from "./Response";
+export * from "./Retriever";
+export * from "./ServiceContext";
+export * from "./TextSplitter";
+export * from "./Tool";
+export * from "./callbacks/CallbackManager";
+export * from "./constants";
+export * from "./embeddings";
+export * from "./indices";
+// export * from "./llm";
+export * from "./llm/openai";
+export * from "./nodeParsers";
+export * from "./postprocessors";
+// export * from "./readers/AssemblyAI";
+// export * from "./readers/CSVReader";
+// export * from "./readers/DocxReader";
+// export * from "./readers/HTMLReader";
+// export * from "./readers/MarkdownReader";
+// export * from "./readers/NotionReader";
+// export * from "./readers/PDFReader";
+// export * from "./readers/SimpleDirectoryReader";
+// export * from "./readers/SimpleMongoReader";
+// export * from "./readers/base";
+export * from "./storage/FileSystem";
+export * from "./storage/StorageContext";
+export * from "./synthesizers";

packages/core/src/index.ts

@@ -29,5 +29,22 @@ export * from "./readers/PDFReader";
 export * from "./readers/SimpleDirectoryReader";
 export * from "./readers/SimpleMongoReader";
 export * from "./readers/base";
-export * from "./storage";
+// #region storage
+export * from "./storage/FileSystem";
+export * from "./storage/StorageContext";
+export * from "./storage/constants";
+export { SimpleDocumentStore } from "./storage/docStore/SimpleDocumentStore";
+export * from "./storage/docStore/types";
+export { SimpleIndexStore } from "./storage/indexStore/SimpleIndexStore";
+export * from "./storage/indexStore/types";
+export { SimpleKVStore } from "./storage/kvStore/SimpleKVStore";
+export * from "./storage/kvStore/types";
+export { AstraDBVectorStore } from "./storage/vectorStore/AstraDBVectorStore";
+export { ChromaVectorStore } from "./storage/vectorStore/ChromaVectorStore";
+export { MongoDBAtlasVectorSearch } from "./storage/vectorStore/MongoDBAtlasVectorStore";
+export { PGVectorStore } from "./storage/vectorStore/PGVectorStore";
+export { PineconeVectorStore } from "./storage/vectorStore/PineconeVectorStore";
+export { SimpleVectorStore } from "./storage/vectorStore/SimpleVectorStore";
+export * from "./storage/vectorStore/types";
+// #endregion
 export * from "./synthesizers";

packages/core/src/indices/BaseIndex.ts

@@ -1,8 +1,8 @@
-import { v4 as uuidv4 } from "uuid";
 import { BaseNode, Document, jsonToNode } from "../Node";
 import { BaseQueryEngine } from "../QueryEngine";
 import { BaseRetriever } from "../Retriever";
 import { ServiceContext } from "../ServiceContext";
+import { randomUUID } from "../environments";
 import { StorageContext } from "../storage/StorageContext";
 import { BaseDocumentStore } from "../storage/docStore/types";
 import { BaseIndexStore } from "../storage/indexStore/types";
@@ -16,7 +16,7 @@ export abstract class IndexStruct {
   indexId: string;
   summary?: string;
 
-  constructor(indexId = uuidv4(), summary = undefined) {
+  constructor(indexId = randomUUID(), summary = undefined) {
     this.indexId = indexId;
     this.summary = summary;
   }

packages/core/src/indices/keyword/KeywordTableIndex.ts

@@ -8,10 +8,10 @@ import {
 } from "../../ServiceContext";
 import { BaseNodePostprocessor } from "../../postprocessors";
 import {
-  BaseDocumentStore,
   StorageContext,
   storageContextFromDefaults,
-} from "../../storage";
+} from "../../storage/StorageContext";
+import { BaseDocumentStore } from "../../storage/docStore/types";
 import { BaseSynthesizer } from "../../synthesizers";
 import {
   BaseIndex,
@@ -149,12 +149,12 @@ export class KeywordTableIndex extends BaseIndex<KeywordTable> {
     text: string,
     serviceContext: ServiceContext,
   ): Promise<Set<string>> {
-    const response = await serviceContext.llm.complete(
-      defaultKeywordExtractPrompt({
+    const response = await serviceContext.llm.complete({
+      prompt: defaultKeywordExtractPrompt({
         context: text,
       }),
-    );
-    return extractKeywordsGivenResponse(response.message.content, "KEYWORDS:");
+    });
+    return extractKeywordsGivenResponse(response.text, "KEYWORDS:");
   }
 
   /**

packages/core/src/indices/keyword/KeywordTableIndexRetriever.ts

@@ -86,16 +86,13 @@ abstract class BaseKeywordTableRetriever implements BaseRetriever {
 // Extracts keywords using LLMs.
 export class KeywordTableLLMRetriever extends BaseKeywordTableRetriever {
   async getKeywords(query: string): Promise<string[]> {
-    const response = await this.serviceContext.llm.complete(
-      this.queryKeywordExtractTemplate({
+    const response = await this.serviceContext.llm.complete({
+      prompt: this.queryKeywordExtractTemplate({
         question: query,
         maxKeywords: this.maxKeywordsPerQuery,
       }),
-    );
-    const keywords = extractKeywordsGivenResponse(
-      response.message.content,
-      "KEYWORDS:",
-    );
+    });
+    const keywords = extractKeywordsGivenResponse(response.text, "KEYWORDS:");
     return [...keywords];
   }
 }

packages/core/src/indices/summary/SummaryIndex.ts

@@ -8,11 +8,10 @@ import {
 } from "../../ServiceContext";
 import { BaseNodePostprocessor } from "../../postprocessors";
 import {
-  BaseDocumentStore,
-  RefDocInfo,
   StorageContext,
   storageContextFromDefaults,
-} from "../../storage";
+} from "../../storage/StorageContext";
+import { BaseDocumentStore, RefDocInfo } from "../../storage/docStore/types";
 import {
   BaseSynthesizer,
   CompactAndRefine,

packages/core/src/indices/summary/SummaryIndexRetriever.ts

@@ -89,8 +89,10 @@ export class SummaryIndexLLMRetriever implements BaseRetriever {
       const fmtBatchStr = this.formatNodeBatchFn(nodesBatch);
       const input = { context: fmtBatchStr, query: query };
       const rawResponse = (
-        await this.serviceContext.llm.complete(this.choiceSelectPrompt(input))
-      ).message.content;
+        await this.serviceContext.llm.complete({
+          prompt: this.choiceSelectPrompt(input),
+        })
+      ).text;
 
       // parseResult is a map from doc number to relevance score
       const parseResult = this.parseChoiceSelectAnswerFn(

packages/core/src/indices/vectorStore/VectorStoreIndex.ts

@@ -19,11 +19,11 @@ import {
 } from "../../embeddings";
 import { BaseNodePostprocessor } from "../../postprocessors";
 import {
-  BaseIndexStore,
   StorageContext,
-  VectorStore,
   storageContextFromDefaults,
-} from "../../storage";
+} from "../../storage/StorageContext";
+import { BaseIndexStore } from "../../storage/indexStore/types";
+import { VectorStore } from "../../storage/vectorStore/types";
 import { BaseSynthesizer } from "../../synthesizers";
 import {
   BaseIndex,

packages/core/src/llm/LLM.ts

@@ -1,32 +1,13 @@
-import OpenAILLM, { ClientOptions as OpenAIClientOptions } from "openai";
 import {
-  AnthropicStreamToken,
   CallbackManager,
   Event,
   EventType,
-  OpenAIStreamToken,
   StreamCallbackResponse,
 } from "../callbacks/CallbackManager";
 
-import { ChatCompletionMessageParam } from "openai/resources";
 import { LLMOptions } from "portkey-ai";
-import { MessageContent } from "../ChatEngine";
-import { globalsHelper, Tokenizers } from "../GlobalsHelper";
-import {
-  ANTHROPIC_AI_PROMPT,
-  ANTHROPIC_HUMAN_PROMPT,
-  AnthropicSession,
-  getAnthropicSession,
-} from "./anthropic";
-import {
-  AzureOpenAIConfig,
-  getAzureBaseUrl,
-  getAzureConfigFromEnv,
-  getAzureModel,
-  shouldUseAzure,
-} from "./azure";
-import { getOpenAISession, OpenAISession } from "./openai";
-import { getPortkeySession, PortkeySession } from "./portkey";
+import { Tokenizers } from "../GlobalsHelper";
+import { PortkeySession, getPortkeySession } from "./portkey";
 import { ReplicateSession } from "./replicate";
 
 export type MessageType =
@@ -45,11 +26,16 @@ export interface ChatMessage {
 export interface ChatResponse {
   message: ChatMessage;
   raw?: Record<string, any>;
-  delta?: string;
 }
 
-// NOTE in case we need CompletionResponse to diverge from ChatResponse in the future
-export type CompletionResponse = ChatResponse;
+export interface ChatResponseChunk {
+  delta: string;
+}
+
+export interface CompletionResponse {
+  text: string;
+  raw?: Record<string, any>;
+}
 
 export interface LLMMetadata {
   model: string;
@@ -60,40 +46,59 @@ export interface LLMMetadata {
   tokenizer: Tokenizers | undefined;
 }
 
+export interface LLMChatParamsBase {
+  messages: ChatMessage[];
+  parentEvent?: Event;
+  extraParams?: Record<string, any>;
+}
+
+export interface LLMChatParamsStreaming extends LLMChatParamsBase {
+  stream: true;
+}
+
+export interface LLMChatParamsNonStreaming extends LLMChatParamsBase {
+  stream?: false | null;
+}
+
+export interface LLMCompletionParamsBase {
+  prompt: any;
+  parentEvent?: Event;
+}
+
+export interface LLMCompletionParamsStreaming extends LLMCompletionParamsBase {
+  stream: true;
+}
+
+export interface LLMCompletionParamsNonStreaming
+  extends LLMCompletionParamsBase {
+  stream?: false | null;
+}
+
 /**
  * Unified language model interface
  */
 export interface LLM {
   metadata: LLMMetadata;
-  // Whether a LLM has streaming support
-  hasStreaming: boolean;
   /**
    * Get a chat response from the LLM
-   * @param messages
    *
-   * The return type of chat() and complete() are set by the "streaming" parameter being set to True.
+   * @param params
    */
-  chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    messages: ChatMessage[],
-    parentEvent?: Event,
-    streaming?: T,
-  ): Promise<R>;
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
 
   /**
    * Get a prompt completion from the LLM
-   * @param prompt the prompt to complete
+   * @param params
    */
-  complete<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    prompt: MessageContent,
-    parentEvent?: Event,
-    streaming?: T,
-  ): Promise<R>;
+  complete(
+    params: LLMCompletionParamsStreaming,
+  ): Promise<AsyncIterable<CompletionResponse>>;
+  complete(
+    params: LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse>;
 
   /**
    * Calculates the number of tokens needed for the given chat messages
@@ -101,6 +106,50 @@ export interface LLM {
   tokens(messages: ChatMessage[]): number;
 }
 
+export abstract class BaseLLM implements LLM {
+  abstract metadata: LLMMetadata;
+
+  private async *chatToComplete(
+    stream: AsyncIterable<ChatResponseChunk>,
+  ): AsyncIterable<CompletionResponse> {
+    for await (const chunk of stream) {
+      yield { text: chunk.delta };
+    }
+  }
+
+  complete(
+    params: LLMCompletionParamsStreaming,
+  ): Promise<AsyncIterable<CompletionResponse>>;
+  complete(
+    params: LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse>;
+  async complete(
+    params: LLMCompletionParamsStreaming | LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse | AsyncIterable<CompletionResponse>> {
+    const { prompt, parentEvent, stream } = params;
+    if (stream) {
+      const stream = await this.chat({
+        messages: [{ content: prompt, role: "user" }],
+        parentEvent,
+        stream: true,
+      });
+      return this.chatToComplete(stream);
+    }
+    const chatResponse = await this.chat({
+      messages: [{ content: prompt, role: "user" }],
+      parentEvent,
+    });
+    return { text: chatResponse.message.content as string };
+  }
+
+  abstract chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  abstract chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+
+  abstract tokens(messages: ChatMessage[]): number;
+}
+
 export const GPT4_MODELS = {
   "gpt-4": { contextWindow: 8192 },
   "gpt-4-32k": { contextWindow: 32768 },
@@ -122,254 +171,6 @@ export const ALL_AVAILABLE_OPENAI_MODELS = {
   ...GPT35_MODELS,
 };
 
-/**
- * OpenAI LLM implementation
- */
-export class OpenAI implements LLM {
-  hasStreaming: boolean = true;
-
-  // Per completion OpenAI params
-  model: keyof typeof ALL_AVAILABLE_OPENAI_MODELS;
-  temperature: number;
-  topP: number;
-  maxTokens?: number;
-  additionalChatOptions?: Omit<
-    Partial<OpenAILLM.Chat.ChatCompletionCreateParams>,
-    "max_tokens" | "messages" | "model" | "temperature" | "top_p" | "streaming"
-  >;
-
-  // OpenAI session params
-  apiKey?: string = undefined;
-  maxRetries: number;
-  timeout?: number;
-  session: OpenAISession;
-  additionalSessionOptions?: Omit<
-    Partial<OpenAIClientOptions>,
-    "apiKey" | "maxRetries" | "timeout"
-  >;
-
-  callbackManager?: CallbackManager;
-
-  constructor(
-    init?: Partial<OpenAI> & {
-      azure?: AzureOpenAIConfig;
-    },
-  ) {
-    this.model = init?.model ?? "gpt-3.5-turbo";
-    this.temperature = init?.temperature ?? 0.1;
-    this.topP = init?.topP ?? 1;
-    this.maxTokens = init?.maxTokens ?? undefined;
-
-    this.maxRetries = init?.maxRetries ?? 10;
-    this.timeout = init?.timeout ?? 60 * 1000; // Default is 60 seconds
-    this.additionalChatOptions = init?.additionalChatOptions;
-    this.additionalSessionOptions = init?.additionalSessionOptions;
-
-    if (init?.azure || shouldUseAzure()) {
-      const azureConfig = getAzureConfigFromEnv({
-        ...init?.azure,
-        model: getAzureModel(this.model),
-      });
-
-      if (!azureConfig.apiKey) {
-        throw new Error(
-          "Azure API key is required for OpenAI Azure models. Please set the AZURE_OPENAI_KEY environment variable.",
-        );
-      }
-
-      this.apiKey = azureConfig.apiKey;
-      this.session =
-        init?.session ??
-        getOpenAISession({
-          azure: true,
-          apiKey: this.apiKey,
-          baseURL: getAzureBaseUrl(azureConfig),
-          maxRetries: this.maxRetries,
-          timeout: this.timeout,
-          defaultQuery: { "api-version": azureConfig.apiVersion },
-          ...this.additionalSessionOptions,
-        });
-    } else {
-      this.apiKey = init?.apiKey ?? undefined;
-      this.session =
-        init?.session ??
-        getOpenAISession({
-          apiKey: this.apiKey,
-          maxRetries: this.maxRetries,
-          timeout: this.timeout,
-          ...this.additionalSessionOptions,
-        });
-    }
-
-    this.callbackManager = init?.callbackManager;
-  }
-
-  get metadata() {
-    return {
-      model: this.model,
-      temperature: this.temperature,
-      topP: this.topP,
-      maxTokens: this.maxTokens,
-      contextWindow: ALL_AVAILABLE_OPENAI_MODELS[this.model].contextWindow,
-      tokenizer: Tokenizers.CL100K_BASE,
-    };
-  }
-
-  tokens(messages: ChatMessage[]): number {
-    // for latest OpenAI models, see https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
-    const tokenizer = globalsHelper.tokenizer(this.metadata.tokenizer);
-    const tokensPerMessage = 3;
-    let numTokens = 0;
-    for (const message of messages) {
-      numTokens += tokensPerMessage;
-      for (const value of Object.values(message)) {
-        numTokens += tokenizer(value).length;
-      }
-    }
-    numTokens += 3; // every reply is primed with <|im_start|>assistant<|im_sep|>
-    return numTokens;
-  }
-
-  mapMessageType(
-    messageType: MessageType,
-  ): "user" | "assistant" | "system" | "function" {
-    switch (messageType) {
-      case "user":
-        return "user";
-      case "assistant":
-        return "assistant";
-      case "system":
-        return "system";
-      case "function":
-        return "function";
-      default:
-        return "user";
-    }
-  }
-
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(messages: ChatMessage[], parentEvent?: Event, streaming?: T): Promise<R> {
-    const baseRequestParams: OpenAILLM.Chat.ChatCompletionCreateParams = {
-      model: this.model,
-      temperature: this.temperature,
-      max_tokens: this.maxTokens,
-      messages: messages.map(
-        (message) =>
-          ({
-            role: this.mapMessageType(message.role),
-            content: message.content,
-          }) as ChatCompletionMessageParam,
-      ),
-      top_p: this.topP,
-      ...this.additionalChatOptions,
-    };
-    // Streaming
-    if (streaming) {
-      if (!this.hasStreaming) {
-        throw Error("No streaming support for this LLM.");
-      }
-      return this.streamChat(messages, parentEvent) as R;
-    }
-    // Non-streaming
-    const response = await this.session.openai.chat.completions.create({
-      ...baseRequestParams,
-      stream: false,
-    });
-
-    const content = response.choices[0].message?.content ?? "";
-    return {
-      message: { content, role: response.choices[0].message.role },
-    } as R;
-  }
-
-  async complete<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(prompt: string, parentEvent?: Event, streaming?: T): Promise<R> {
-    return this.chat(
-      [{ content: prompt, role: "user" }],
-      parentEvent,
-      streaming,
-    );
-  }
-
-  //We can wrap a stream in a generator to add some additional logging behavior
-  //For future edits: syntax for generator type is <typeof Yield, typeof Return, typeof Accept>
-  //"typeof Accept" refers to what types you'll accept when you manually call generator.next(<AcceptType>)
-  protected async *streamChat(
-    messages: ChatMessage[],
-    parentEvent?: Event,
-  ): AsyncGenerator<string, void, unknown> {
-    const baseRequestParams: OpenAILLM.Chat.ChatCompletionCreateParams = {
-      model: this.model,
-      temperature: this.temperature,
-      max_tokens: this.maxTokens,
-      messages: messages.map(
-        (message) =>
-          ({
-            role: this.mapMessageType(message.role),
-            content: message.content,
-          }) as ChatCompletionMessageParam,
-      ),
-      top_p: this.topP,
-      ...this.additionalChatOptions,
-    };
-
-    //Now let's wrap our stream in a callback
-    const onLLMStream = this.callbackManager?.onLLMStream
-      ? this.callbackManager.onLLMStream
-      : () => {};
-
-    const chunk_stream: AsyncIterable<OpenAIStreamToken> =
-      await this.session.openai.chat.completions.create({
-        ...baseRequestParams,
-        stream: true,
-      });
-
-    const event: Event = parentEvent
-      ? parentEvent
-      : {
-          id: "unspecified",
-          type: "llmPredict" as EventType,
-        };
-
-    //Indices
-    var idx_counter: number = 0;
-    for await (const part of chunk_stream) {
-      if (!part.choices.length) continue;
-
-      //Increment
-      part.choices[0].index = idx_counter;
-      const is_done: boolean =
-        part.choices[0].finish_reason === "stop" ? true : false;
-      //onLLMStream Callback
-
-      const stream_callback: StreamCallbackResponse = {
-        event: event,
-        index: idx_counter,
-        isDone: is_done,
-        token: part,
-      };
-      onLLMStream(stream_callback);
-
-      idx_counter++;
-
-      yield part.choices[0].delta.content ? part.choices[0].delta.content : "";
-    }
-    return;
-  }
-
-  //streamComplete doesn't need to be async because it's child function is already async
-  protected streamComplete(
-    query: string,
-    parentEvent?: Event,
-  ): AsyncGenerator<string, void, unknown> {
-    return this.streamChat([{ content: query, role: "user" }], parentEvent);
-  }
-}
-
 export const ALL_AVAILABLE_LLAMADEUCE_MODELS = {
   "Llama-2-70b-chat-old": {
     contextWindow: 4096,
@@ -425,16 +226,16 @@ export enum DeuceChatStrategy {
 /**
  * Llama2 LLM implementation
  */
-export class LlamaDeuce implements LLM {
+export class LlamaDeuce extends BaseLLM {
   model: keyof typeof ALL_AVAILABLE_LLAMADEUCE_MODELS;
   chatStrategy: DeuceChatStrategy;
   temperature: number;
   topP: number;
   maxTokens?: number;
   replicateSession: ReplicateSession;
-  hasStreaming: boolean;
 
   constructor(init?: Partial<LlamaDeuce>) {
+    super();
     this.model = init?.model ?? "Llama-2-70b-chat-4bit";
     this.chatStrategy =
       init?.chatStrategy ??
@@ -447,7 +248,6 @@ export class LlamaDeuce implements LLM {
       init?.maxTokens ??
       ALL_AVAILABLE_LLAMADEUCE_MODELS[this.model].contextWindow; // For Replicate, the default is 500 tokens which is too low.
     this.replicateSession = init?.replicateSession ?? new ReplicateSession();
-    this.hasStreaming = init?.hasStreaming ?? false;
   }
 
   tokens(messages: ChatMessage[]): number {
@@ -592,10 +392,14 @@ If a question does not make any sense, or is not factually coherent, explain why
     };
   }
 
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(messages: ChatMessage[], _parentEvent?: Event, streaming?: T): Promise<R> {
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+  async chat(
+    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
+  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+    const { messages, parentEvent, stream } = params;
     const api = ALL_AVAILABLE_LLAMADEUCE_MODELS[this.model]
       .replicateApi as `${string}/${string}:${string}`;
 
@@ -617,6 +421,9 @@ If a question does not make any sense, or is not factually coherent, explain why
     }
 
     //TODO: Add streaming for this
+    if (stream) {
+      throw new Error("Streaming not supported for LlamaDeuce");
+    }
 
     //Non-streaming
     const response = await this.replicateSession.replicate.run(
@@ -629,14 +436,7 @@ If a question does not make any sense, or is not factually coherent, explain why
         //^ We need to do this because Replicate returns a list of strings (for streaming functionality which is not exposed by the run function)
         role: "assistant",
       },
-    } as R;
-  }
-
-  async complete<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(prompt: string, parentEvent?: Event, streaming?: T): Promise<R> {
-    return this.chat([{ content: prompt, role: "user" }], parentEvent);
+    };
   }
 }
 
@@ -646,165 +446,7 @@ export const ALL_AVAILABLE_ANTHROPIC_MODELS = {
   "claude-instant-1": { contextWindow: 100000 },
 };
 
-/**
- * Anthropic LLM implementation
- */
-
-export class Anthropic implements LLM {
-  hasStreaming: boolean = true;
-
-  // Per completion Anthropic params
-  model: keyof typeof ALL_AVAILABLE_ANTHROPIC_MODELS;
-  temperature: number;
-  topP: number;
-  maxTokens?: number;
-
-  // Anthropic session params
-  apiKey?: string = undefined;
-  maxRetries: number;
-  timeout?: number;
-  session: AnthropicSession;
-
-  callbackManager?: CallbackManager;
-
-  constructor(init?: Partial<Anthropic>) {
-    this.model = init?.model ?? "claude-2";
-    this.temperature = init?.temperature ?? 0.1;
-    this.topP = init?.topP ?? 0.999; // Per Ben Mann
-    this.maxTokens = init?.maxTokens ?? undefined;
-
-    this.apiKey = init?.apiKey ?? undefined;
-    this.maxRetries = init?.maxRetries ?? 10;
-    this.timeout = init?.timeout ?? 60 * 1000; // Default is 60 seconds
-    this.session =
-      init?.session ??
-      getAnthropicSession({
-        apiKey: this.apiKey,
-        maxRetries: this.maxRetries,
-        timeout: this.timeout,
-      });
-
-    this.callbackManager = init?.callbackManager;
-  }
-
-  tokens(messages: ChatMessage[]): number {
-    throw new Error("Method not implemented.");
-  }
-
-  get metadata() {
-    return {
-      model: this.model,
-      temperature: this.temperature,
-      topP: this.topP,
-      maxTokens: this.maxTokens,
-      contextWindow: ALL_AVAILABLE_ANTHROPIC_MODELS[this.model].contextWindow,
-      tokenizer: undefined,
-    };
-  }
-
-  mapMessagesToPrompt(messages: ChatMessage[]) {
-    return (
-      messages.reduce((acc, message) => {
-        return (
-          acc +
-          `${
-            message.role === "system"
-              ? ""
-              : message.role === "assistant"
-                ? ANTHROPIC_AI_PROMPT + " "
-                : ANTHROPIC_HUMAN_PROMPT + " "
-          }${message.content.trim()}`
-        );
-      }, "") + ANTHROPIC_AI_PROMPT
-    );
-  }
-
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    messages: ChatMessage[],
-    parentEvent?: Event | undefined,
-    streaming?: T,
-  ): Promise<R> {
-    //Streaming
-    if (streaming) {
-      if (!this.hasStreaming) {
-        throw Error("No streaming support for this LLM.");
-      }
-      return this.streamChat(messages, parentEvent) as R;
-    }
-
-    //Non-streaming
-    const response = await this.session.anthropic.completions.create({
-      model: this.model,
-      prompt: this.mapMessagesToPrompt(messages),
-      max_tokens_to_sample: this.maxTokens ?? 100000,
-      temperature: this.temperature,
-      top_p: this.topP,
-    });
-
-    return {
-      message: { content: response.completion.trimStart(), role: "assistant" },
-      //^ We're trimming the start because Anthropic often starts with a space in the response
-      // That space will be re-added when we generate the next prompt.
-    } as R;
-  }
-
-  protected async *streamChat(
-    messages: ChatMessage[],
-    parentEvent?: Event | undefined,
-  ): AsyncGenerator<string, void, unknown> {
-    // AsyncIterable<AnthropicStreamToken>
-    const stream: AsyncIterable<AnthropicStreamToken> =
-      await this.session.anthropic.completions.create({
-        model: this.model,
-        prompt: this.mapMessagesToPrompt(messages),
-        max_tokens_to_sample: this.maxTokens ?? 100000,
-        temperature: this.temperature,
-        top_p: this.topP,
-        stream: true,
-      });
-
-    var idx_counter: number = 0;
-    for await (const part of stream) {
-      //TODO: LLM Stream Callback, pending re-work.
-
-      idx_counter++;
-      yield part.completion;
-    }
-    return;
-  }
-
-  async complete<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    prompt: string,
-    parentEvent?: Event | undefined,
-    streaming?: T,
-  ): Promise<R> {
-    if (streaming) {
-      return this.streamComplete(prompt, parentEvent) as R;
-    }
-    return this.chat(
-      [{ content: prompt, role: "user" }],
-      parentEvent,
-      streaming,
-    ) as R;
-  }
-
-  protected streamComplete(
-    prompt: string,
-    parentEvent?: Event | undefined,
-  ): AsyncGenerator<string, void, unknown> {
-    return this.streamChat([{ content: prompt, role: "user" }], parentEvent);
-  }
-}
-
-export class Portkey implements LLM {
-  hasStreaming: boolean = true;
-
+export class Portkey extends BaseLLM {
   apiKey?: string = undefined;
   baseURL?: string = undefined;
   mode?: string = undefined;
@@ -813,6 +455,7 @@ export class Portkey implements LLM {
   callbackManager?: CallbackManager;
 
   constructor(init?: Partial<Portkey>) {
+    super();
     this.apiKey = init?.apiKey;
     this.baseURL = init?.baseURL;
     this.mode = init?.mode;
@@ -834,50 +477,34 @@ export class Portkey implements LLM {
     throw new Error("metadata not implemented for Portkey");
   }
 
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    messages: ChatMessage[],
-    parentEvent?: Event | undefined,
-    streaming?: T,
-    params?: Record<string, any>,
-  ): Promise<R> {
-    if (streaming) {
-      return this.streamChat(messages, parentEvent, params) as R;
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+  async chat(
+    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
+  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+    const { messages, parentEvent, stream, extraParams } = params;
+    if (stream) {
+      return this.streamChat(messages, parentEvent, extraParams);
     } else {
-      const resolvedParams = params || {};
+      const bodyParams = extraParams || {};
       const response = await this.session.portkey.chatCompletions.create({
         messages,
-        ...resolvedParams,
+        ...bodyParams,
       });
 
       const content = response.choices[0].message?.content ?? "";
       const role = response.choices[0].message?.role || "assistant";
-      return { message: { content, role: role as MessageType } } as R;
+      return { message: { content, role: role as MessageType } };
     }
   }
 
-  async complete<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    prompt: string,
-    parentEvent?: Event | undefined,
-    streaming?: T,
-  ): Promise<R> {
-    return this.chat(
-      [{ content: prompt, role: "user" }],
-      parentEvent,
-      streaming,
-    );
-  }
-
   async *streamChat(
     messages: ChatMessage[],
     parentEvent?: Event,
     params?: Record<string, any>,
-  ): AsyncGenerator<string, void, unknown> {
+  ): AsyncIterable<ChatResponseChunk> {
     // Wrapping the stream in a callback.
     const onLLMStream = this.callbackManager?.onLLMStream
       ? this.callbackManager.onLLMStream
@@ -915,15 +542,8 @@ export class Portkey implements LLM {
 
       idx_counter++;
 
-      yield part.choices[0].delta?.content ?? "";
+      yield { delta: part.choices[0].delta?.content ?? "" };
     }
     return;
   }
-
-  streamComplete(
-    query: string,
-    parentEvent?: Event,
-  ): AsyncGenerator<string, void, unknown> {
-    return this.streamChat([{ content: query, role: "user" }], parentEvent);
-  }
 }

packages/core/src/llm/anthropic.ts

@@ -1,12 +1,26 @@
-import Anthropic, {
+import AnthropicBase, {
   AI_PROMPT,
   ClientOptions,
   HUMAN_PROMPT,
 } from "@anthropic-ai/sdk";
 import _ from "lodash";
+import {
+  AnthropicStreamToken,
+  CallbackManager,
+  Event,
+} from "../callbacks/CallbackManager";
+import {
+  ALL_AVAILABLE_ANTHROPIC_MODELS,
+  BaseLLM,
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+} from "./LLM";
 
 export class AnthropicSession {
-  anthropic: Anthropic;
+  anthropic: AnthropicBase;
 
   constructor(options: ClientOptions = {}) {
     if (!options.apiKey) {
@@ -19,7 +33,7 @@ export class AnthropicSession {
       throw new Error("Set Anthropic Key in ANTHROPIC_API_KEY env variable"); // Overriding Anthropic package's error message
     }
 
-    this.anthropic = new Anthropic(options);
+    this.anthropic = new AnthropicBase(options);
   }
 }
 
@@ -52,3 +66,130 @@ export function getAnthropicSession(options: ClientOptions = {}) {
 
 export const ANTHROPIC_HUMAN_PROMPT = HUMAN_PROMPT;
 export const ANTHROPIC_AI_PROMPT = AI_PROMPT;
+
+/**
+ * Anthropic LLM implementation
+ */
+
+export class Anthropic extends BaseLLM {
+  // Per completion Anthropic params
+  model: keyof typeof ALL_AVAILABLE_ANTHROPIC_MODELS;
+  temperature: number;
+  topP: number;
+  maxTokens?: number;
+
+  // Anthropic session params
+  apiKey?: string = undefined;
+  maxRetries: number;
+  timeout?: number;
+  session: AnthropicSession;
+
+  callbackManager?: CallbackManager;
+
+  constructor(init?: Partial<Anthropic>) {
+    super();
+    this.model = init?.model ?? "claude-2";
+    this.temperature = init?.temperature ?? 0.1;
+    this.topP = init?.topP ?? 0.999; // Per Ben Mann
+    this.maxTokens = init?.maxTokens ?? undefined;
+
+    this.apiKey = init?.apiKey ?? undefined;
+    this.maxRetries = init?.maxRetries ?? 10;
+    this.timeout = init?.timeout ?? 60 * 1000; // Default is 60 seconds
+    this.session =
+      init?.session ??
+      getAnthropicSession({
+        apiKey: this.apiKey,
+        maxRetries: this.maxRetries,
+        timeout: this.timeout,
+      });
+
+    this.callbackManager = init?.callbackManager;
+  }
+
+  tokens(messages: ChatMessage[]): number {
+    throw new Error("Method not implemented.");
+  }
+
+  get metadata() {
+    return {
+      model: this.model,
+      temperature: this.temperature,
+      topP: this.topP,
+      maxTokens: this.maxTokens,
+      contextWindow: ALL_AVAILABLE_ANTHROPIC_MODELS[this.model].contextWindow,
+      tokenizer: undefined,
+    };
+  }
+
+  mapMessagesToPrompt(messages: ChatMessage[]) {
+    return (
+      messages.reduce((acc, message) => {
+        return (
+          acc +
+          `${
+            message.role === "system"
+              ? ""
+              : message.role === "assistant"
+                ? ANTHROPIC_AI_PROMPT + " "
+                : ANTHROPIC_HUMAN_PROMPT + " "
+          }${message.content.trim()}`
+        );
+      }, "") + ANTHROPIC_AI_PROMPT
+    );
+  }
+
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+  async chat(
+    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
+  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+    const { messages, parentEvent, stream } = params;
+    //Streaming
+    if (stream) {
+      return this.streamChat(messages, parentEvent);
+    }
+
+    //Non-streaming
+    const response = await this.session.anthropic.completions.create({
+      model: this.model,
+      prompt: this.mapMessagesToPrompt(messages),
+      max_tokens_to_sample: this.maxTokens ?? 100000,
+      temperature: this.temperature,
+      top_p: this.topP,
+    });
+
+    return {
+      message: { content: response.completion.trimStart(), role: "assistant" },
+      //^ We're trimming the start because Anthropic often starts with a space in the response
+      // That space will be re-added when we generate the next prompt.
+    };
+  }
+
+  protected async *streamChat(
+    messages: ChatMessage[],
+    parentEvent?: Event | undefined,
+  ): AsyncIterable<ChatResponseChunk> {
+    // AsyncIterable<AnthropicStreamToken>
+    const stream: AsyncIterable<AnthropicStreamToken> =
+      await this.session.anthropic.completions.create({
+        model: this.model,
+        prompt: this.mapMessagesToPrompt(messages),
+        max_tokens_to_sample: this.maxTokens ?? 100000,
+        temperature: this.temperature,
+        top_p: this.topP,
+        stream: true,
+      });
+
+    var idx_counter: number = 0;
+    for await (const part of stream) {
+      //TODO: LLM Stream Callback, pending re-work.
+
+      idx_counter++;
+      yield { delta: part.completion };
+    }
+    return;
+  }
+}

packages/core/src/llm/index.ts

@@ -1,3 +1,4 @@
 export * from "./LLM";
 export * from "./mistral";
 export { Ollama } from "./ollama";
+export { TogetherLLM } from "./together";

packages/core/src/llm/mistral.ts

@@ -4,7 +4,14 @@ import {
   EventType,
   StreamCallbackResponse,
 } from "../callbacks/CallbackManager";
-import { ChatMessage, ChatResponse, LLM } from "./LLM";
+import {
+  BaseLLM,
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+} from "./LLM";
 
 export const ALL_AVAILABLE_MISTRAL_MODELS = {
   "mistral-tiny": { contextWindow: 32000 },
@@ -41,9 +48,7 @@ export class MistralAISession {
 /**
  * MistralAI LLM implementation
  */
-export class MistralAI implements LLM {
-  hasStreaming: boolean = true;
-
+export class MistralAI extends BaseLLM {
   // Per completion MistralAI params
   model: keyof typeof ALL_AVAILABLE_MISTRAL_MODELS;
   temperature: number;
@@ -57,6 +62,7 @@ export class MistralAI implements LLM {
   private session: MistralAISession;
 
   constructor(init?: Partial<MistralAI>) {
+    super();
     this.model = init?.model ?? "mistral-small";
     this.temperature = init?.temperature ?? 0.1;
     this.topP = init?.topP ?? 1;
@@ -94,16 +100,17 @@ export class MistralAI implements LLM {
     };
   }
 
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(messages: ChatMessage[], parentEvent?: Event, streaming?: T): Promise<R> {
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+  async chat(
+    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
+  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+    const { messages, stream } = params;
     // Streaming
-    if (streaming) {
-      if (!this.hasStreaming) {
-        throw Error("No streaming support for this LLM.");
-      }
-      return this.streamChat(messages, parentEvent) as R;
+    if (stream) {
+      return this.streamChat(params);
     }
     // Non-streaming
     const client = await this.session.getClient();
@@ -111,24 +118,13 @@ export class MistralAI implements LLM {
     const message = response.choices[0].message;
     return {
       message,
-    } as R;
+    };
   }
 
-  async complete<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(prompt: string, parentEvent?: Event, streaming?: T): Promise<R> {
-    return this.chat(
-      [{ content: prompt, role: "user" }],
-      parentEvent,
-      streaming,
-    );
-  }
-
-  protected async *streamChat(
-    messages: ChatMessage[],
-    parentEvent?: Event,
-  ): AsyncGenerator<string, void, unknown> {
+  protected async *streamChat({
+    messages,
+    parentEvent,
+  }: LLMChatParamsStreaming): AsyncIterable<ChatResponseChunk> {
     //Now let's wrap our stream in a callback
     const onLLMStream = this.callbackManager?.onLLMStream
       ? this.callbackManager.onLLMStream
@@ -163,16 +159,10 @@ export class MistralAI implements LLM {
 
       idx_counter++;
 
-      yield part.choices[0].delta.content ?? "";
+      yield {
+        delta: part.choices[0].delta.content ?? "",
+      };
     }
     return;
   }
-
-  //streamComplete doesn't need to be async because it's child function is already async
-  protected streamComplete(
-    query: string,
-    parentEvent?: Event,
-  ): AsyncGenerator<string, void, unknown> {
-    return this.streamChat([{ content: query, role: "user" }], parentEvent);
-  }
 }

packages/core/src/llm/ollama.ts

@@ -1,11 +1,27 @@
-import { ok } from "node:assert";
-import { MessageContent } from "../ChatEngine";
 import { CallbackManager, Event } from "../callbacks/CallbackManager";
 import { BaseEmbedding } from "../embeddings";
-import { ChatMessage, ChatResponse, LLM, LLMMetadata } from "./LLM";
+import { assertExists } from "../utils";
+import {
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  CompletionResponse,
+  LLM,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  LLMCompletionParamsNonStreaming,
+  LLMCompletionParamsStreaming,
+  LLMMetadata,
+} from "./LLM";
 
-const messageAccessor = (data: any) => data.message.content;
-const completionAccessor = (data: any) => data.response;
+const messageAccessor = (data: any): ChatResponseChunk => {
+  return {
+    delta: data.message.content,
+  };
+};
+const completionAccessor = (data: any): CompletionResponse => {
+  return { text: data.response };
+};
 
 // https://github.com/jmorganca/ollama
 export class Ollama extends BaseEmbedding implements LLM {
@@ -43,21 +59,21 @@ export class Ollama extends BaseEmbedding implements LLM {
     };
   }
 
-  async chat<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    messages: ChatMessage[],
-    parentEvent?: Event | undefined,
-    streaming?: T,
-  ): Promise<R> {
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+  async chat(
+    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
+  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+    const { messages, parentEvent, stream } = params;
     const payload = {
       model: this.model,
       messages: messages.map((message) => ({
         role: message.role,
         content: message.content,
       })),
-      stream: !!streaming,
+      stream: !!stream,
       options: {
         temperature: this.temperature,
         num_ctx: this.contextWindow,
@@ -73,7 +89,7 @@ export class Ollama extends BaseEmbedding implements LLM {
         "Content-Type": "application/json",
       },
     });
-    if (!streaming) {
+    if (!stream) {
       const raw = await response.json();
       const { message } = raw;
       return {
@@ -82,20 +98,20 @@ export class Ollama extends BaseEmbedding implements LLM {
           content: message.content,
         },
         raw,
-      } satisfies ChatResponse as R;
+      };
     } else {
       const stream = response.body;
-      ok(stream, "stream is null");
-      ok(stream instanceof ReadableStream, "stream is not readable");
-      return this.streamChat(stream, messageAccessor, parentEvent) as R;
+      assertExists(stream, "stream is null");
+      assertExists(stream instanceof ReadableStream, "stream is not readable");
+      return this.streamChat(stream, messageAccessor, parentEvent);
     }
   }
 
-  private async *streamChat(
+  private async *streamChat<T>(
     stream: ReadableStream<Uint8Array>,
-    accessor: (data: any) => string,
+    accessor: (data: any) => T,
     parentEvent?: Event,
-  ): AsyncGenerator<string, void, unknown> {
+  ): AsyncIterable<T> {
     const reader = stream.getReader();
     while (true) {
       const { done, value } = await reader.read();
@@ -119,18 +135,20 @@ export class Ollama extends BaseEmbedding implements LLM {
     }
   }
 
-  async complete<
-    T extends boolean | undefined = undefined,
-    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
-  >(
-    prompt: MessageContent,
-    parentEvent?: Event | undefined,
-    streaming?: T | undefined,
-  ): Promise<R> {
+  complete(
+    params: LLMCompletionParamsStreaming,
+  ): Promise<AsyncIterable<CompletionResponse>>;
+  complete(
+    params: LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse>;
+  async complete(
+    params: LLMCompletionParamsStreaming | LLMCompletionParamsNonStreaming,
+  ): Promise<CompletionResponse | AsyncIterable<CompletionResponse>> {
+    const { prompt, parentEvent, stream } = params;
     const payload = {
       model: this.model,
       prompt: prompt,
-      stream: !!streaming,
+      stream: !!stream,
       options: {
         temperature: this.temperature,
         num_ctx: this.contextWindow,
@@ -146,20 +164,17 @@ export class Ollama extends BaseEmbedding implements LLM {
         "Content-Type": "application/json",
       },
     });
-    if (!streaming) {
+    if (!stream) {
       const raw = await response.json();
       return {
-        message: {
-          role: "assistant",
-          content: raw.response,
-        },
+        text: raw.response,
         raw,
-      } satisfies ChatResponse as R;
+      };
     } else {
       const stream = response.body;
-      ok(stream, "stream is null");
-      ok(stream instanceof ReadableStream, "stream is not readable");
-      return this.streamChat(stream, completionAccessor, parentEvent) as R;
+      assertExists(stream, "stream is null");
+      assertExists(stream instanceof ReadableStream, "stream is not readable");
+      return this.streamChat(stream, completionAccessor, parentEvent);
     }
   }
 

packages/core/src/llm/openai.ts

@@ -1,16 +1,42 @@
 import _ from "lodash";
-import OpenAI, { ClientOptions } from "openai";
+import OpenAILLM, { ClientOptions as OpenAIClientOptions } from "openai";
+import { ChatCompletionMessageParam } from "openai/resources";
+import { Tokenizers, globalsHelper } from "../GlobalsHelper";
+import {
+  CallbackManager,
+  Event,
+  EventType,
+  OpenAIStreamToken,
+  StreamCallbackResponse,
+} from "../callbacks/CallbackManager";
+import {
+  ALL_AVAILABLE_OPENAI_MODELS,
+  BaseLLM,
+  ChatMessage,
+  ChatResponse,
+  ChatResponseChunk,
+  LLMChatParamsNonStreaming,
+  LLMChatParamsStreaming,
+  MessageType,
+} from "./LLM";
+import {
+  AzureOpenAIConfig,
+  getAzureBaseUrl,
+  getAzureConfigFromEnv,
+  getAzureModel,
+  shouldUseAzure,
+} from "./azure";
 
-export class AzureOpenAI extends OpenAI {
+export class AzureOpenAI extends OpenAILLM {
   protected override authHeaders() {
     return { "api-key": this.apiKey };
   }
 }
 
 export class OpenAISession {
-  openai: OpenAI;
+  openai: OpenAILLM;
 
-  constructor(options: ClientOptions & { azure?: boolean } = {}) {
+  constructor(options: OpenAIClientOptions & { azure?: boolean } = {}) {
     if (!options.apiKey) {
       if (typeof process !== undefined) {
         options.apiKey = process.env.OPENAI_API_KEY;
@@ -24,7 +50,7 @@ export class OpenAISession {
     if (options.azure) {
       this.openai = new AzureOpenAI(options);
     } else {
-      this.openai = new OpenAI({
+      this.openai = new OpenAILLM({
         ...options,
         // defaultHeaders: { "OpenAI-Beta": "assistants=v1" },
       });
@@ -35,8 +61,10 @@ export class OpenAISession {
 // I'm not 100% sure this is necessary vs. just starting a new session
 // every time we make a call. They say they try to reuse connections
 // so in theory this is more efficient, but we should test it in the future.
-let defaultOpenAISession: { session: OpenAISession; options: ClientOptions }[] =
-  [];
+let defaultOpenAISession: {
+  session: OpenAISession;
+  options: OpenAIClientOptions;
+}[] = [];
 
 /**
  * Get a session for the OpenAI API. If one already exists with the same options,
@@ -45,7 +73,7 @@ let defaultOpenAISession: { session: OpenAISession; options: ClientOptions }[] =
  * @returns
  */
 export function getOpenAISession(
-  options: ClientOptions & { azure?: boolean } = {},
+  options: OpenAIClientOptions & { azure?: boolean } = {},
 ) {
   let session = defaultOpenAISession.find((session) => {
     return _.isEqual(session.options, options);
@@ -58,3 +86,235 @@ export function getOpenAISession(
 
   return session;
 }
+
+/**
+ * OpenAI LLM implementation
+ */
+export class OpenAI extends BaseLLM {
+  // Per completion OpenAI params
+  model: keyof typeof ALL_AVAILABLE_OPENAI_MODELS | string;
+  temperature: number;
+  topP: number;
+  maxTokens?: number;
+  additionalChatOptions?: Omit<
+    Partial<OpenAILLM.Chat.ChatCompletionCreateParams>,
+    "max_tokens" | "messages" | "model" | "temperature" | "top_p" | "stream"
+  >;
+
+  // OpenAI session params
+  apiKey?: string = undefined;
+  maxRetries: number;
+  timeout?: number;
+  session: OpenAISession;
+  additionalSessionOptions?: Omit<
+    Partial<OpenAIClientOptions>,
+    "apiKey" | "maxRetries" | "timeout"
+  >;
+
+  callbackManager?: CallbackManager;
+
+  constructor(
+    init?: Partial<OpenAI> & {
+      azure?: AzureOpenAIConfig;
+    },
+  ) {
+    super();
+    this.model = init?.model ?? "gpt-3.5-turbo";
+    this.temperature = init?.temperature ?? 0.1;
+    this.topP = init?.topP ?? 1;
+    this.maxTokens = init?.maxTokens ?? undefined;
+
+    this.maxRetries = init?.maxRetries ?? 10;
+    this.timeout = init?.timeout ?? 60 * 1000; // Default is 60 seconds
+    this.additionalChatOptions = init?.additionalChatOptions;
+    this.additionalSessionOptions = init?.additionalSessionOptions;
+
+    if (init?.azure || shouldUseAzure()) {
+      const azureConfig = getAzureConfigFromEnv({
+        ...init?.azure,
+        model: getAzureModel(this.model),
+      });
+
+      if (!azureConfig.apiKey) {
+        throw new Error(
+          "Azure API key is required for OpenAI Azure models. Please set the AZURE_OPENAI_KEY environment variable.",
+        );
+      }
+
+      this.apiKey = azureConfig.apiKey;
+      this.session =
+        init?.session ??
+        getOpenAISession({
+          azure: true,
+          apiKey: this.apiKey,
+          baseURL: getAzureBaseUrl(azureConfig),
+          maxRetries: this.maxRetries,
+          timeout: this.timeout,
+          defaultQuery: { "api-version": azureConfig.apiVersion },
+          ...this.additionalSessionOptions,
+        });
+    } else {
+      this.apiKey = init?.apiKey ?? undefined;
+      this.session =
+        init?.session ??
+        getOpenAISession({
+          apiKey: this.apiKey,
+          maxRetries: this.maxRetries,
+          timeout: this.timeout,
+          ...this.additionalSessionOptions,
+        });
+    }
+
+    this.callbackManager = init?.callbackManager;
+  }
+
+  get metadata() {
+    const contextWindow =
+      ALL_AVAILABLE_OPENAI_MODELS[
+        this.model as keyof typeof ALL_AVAILABLE_OPENAI_MODELS
+      ]?.contextWindow ?? 1024;
+    return {
+      model: this.model,
+      temperature: this.temperature,
+      topP: this.topP,
+      maxTokens: this.maxTokens,
+      contextWindow,
+      tokenizer: Tokenizers.CL100K_BASE,
+    };
+  }
+
+  tokens(messages: ChatMessage[]): number {
+    // for latest OpenAI models, see https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
+    const tokenizer = globalsHelper.tokenizer(this.metadata.tokenizer);
+    const tokensPerMessage = 3;
+    let numTokens = 0;
+    for (const message of messages) {
+      numTokens += tokensPerMessage;
+      for (const value of Object.values(message)) {
+        numTokens += tokenizer(value).length;
+      }
+    }
+    numTokens += 3; // every reply is primed with <|im_start|>assistant<|im_sep|>
+    return numTokens;
+  }
+
+  mapMessageType(
+    messageType: MessageType,
+  ): "user" | "assistant" | "system" | "function" {
+    switch (messageType) {
+      case "user":
+        return "user";
+      case "assistant":
+        return "assistant";
+      case "system":
+        return "system";
+      case "function":
+        return "function";
+      default:
+        return "user";
+    }
+  }
+
+  chat(
+    params: LLMChatParamsStreaming,
+  ): Promise<AsyncIterable<ChatResponseChunk>>;
+  chat(params: LLMChatParamsNonStreaming): Promise<ChatResponse>;
+  async chat(
+    params: LLMChatParamsNonStreaming | LLMChatParamsStreaming,
+  ): Promise<ChatResponse | AsyncIterable<ChatResponseChunk>> {
+    const { messages, parentEvent, stream } = params;
+    const baseRequestParams: OpenAILLM.Chat.ChatCompletionCreateParams = {
+      model: this.model,
+      temperature: this.temperature,
+      max_tokens: this.maxTokens,
+      messages: messages.map(
+        (message) =>
+          ({
+            role: this.mapMessageType(message.role),
+            content: message.content,
+          }) as ChatCompletionMessageParam,
+      ),
+      top_p: this.topP,
+      ...this.additionalChatOptions,
+    };
+    // Streaming
+    if (stream) {
+      return this.streamChat(params);
+    }
+    // Non-streaming
+    const response = await this.session.openai.chat.completions.create({
+      ...baseRequestParams,
+      stream: false,
+    });
+
+    const content = response.choices[0].message?.content ?? "";
+    return {
+      message: { content, role: response.choices[0].message.role },
+    };
+  }
+
+  protected async *streamChat({
+    messages,
+    parentEvent,
+  }: LLMChatParamsStreaming): AsyncIterable<ChatResponseChunk> {
+    const baseRequestParams: OpenAILLM.Chat.ChatCompletionCreateParams = {
+      model: this.model,
+      temperature: this.temperature,
+      max_tokens: this.maxTokens,
+      messages: messages.map(
+        (message) =>
+          ({
+            role: this.mapMessageType(message.role),
+            content: message.content,
+          }) as ChatCompletionMessageParam,
+      ),
+      top_p: this.topP,
+      ...this.additionalChatOptions,
+    };
+
+    //Now let's wrap our stream in a callback
+    const onLLMStream = this.callbackManager?.onLLMStream
+      ? this.callbackManager.onLLMStream
+      : () => {};
+
+    const chunk_stream: AsyncIterable<OpenAIStreamToken> =
+      await this.session.openai.chat.completions.create({
+        ...baseRequestParams,
+        stream: true,
+      });
+
+    const event: Event = parentEvent
+      ? parentEvent
+      : {
+          id: "unspecified",
+          type: "llmPredict" as EventType,
+        };
+
+    //Indices
+    var idx_counter: number = 0;
+    for await (const part of chunk_stream) {
+      if (!part.choices.length) continue;
+
+      //Increment
+      part.choices[0].index = idx_counter;
+      const is_done: boolean =
+        part.choices[0].finish_reason === "stop" ? true : false;
+      //onLLMStream Callback
+
+      const stream_callback: StreamCallbackResponse = {
+        event: event,
+        index: idx_counter,
+        isDone: is_done,
+        token: part,
+      };
+      onLLMStream(stream_callback);
+
+      idx_counter++;
+
+      yield {
+        delta: part.choices[0].delta.content ?? "",
+      };
+    }
+    return;
+  }
+}

packages/core/src/llm/replicate.ts

@@ -28,5 +28,3 @@ export function getReplicateSession(replicateKey: string | null = null) {
 
   return defaultReplicateSession;
 }
-
-export * from "openai";

packages/core/src/llm/together.ts

@@ -0,0 +1,14 @@
+import { OpenAI } from "./openai";
+
+export class TogetherLLM extends OpenAI {
+  constructor(init?: Partial<OpenAI>) {
+    super({
+      ...init,
+      apiKey: process.env.TOGETHER_API_KEY,
+      additionalSessionOptions: {
+        ...init?.additionalSessionOptions,
+        baseURL: "https://api.together.xyz/v1",
+      },
+    });
+  }
+}

packages/core/src/llm/utils.ts

@@ -0,0 +1,10 @@
+// TODO: use for LLM.ts
+
+export async function* streamConverter<S, D>(
+  stream: AsyncIterable<S>,
+  converter: (s: S) => D,
+): AsyncIterable<D> {
+  for await (const data of stream) {
+    yield converter(data);
+  }
+}

packages/core/src/readers/CSVReader.ts

@@ -1,6 +1,6 @@
 import Papa, { ParseConfig } from "papaparse";
 import { Document } from "../Node";
-import { DEFAULT_FS, GenericFileSystem } from "../storage/FileSystem";
+import { genericFileSystem, GenericFileSystem } from "../storage/FileSystem";
 import { BaseReader } from "./base";
 
 /**
@@ -40,7 +40,7 @@ export class PapaCSVReader implements BaseReader {
    */
   async loadData(
     file: string,
-    fs: GenericFileSystem = DEFAULT_FS,
+    fs: GenericFileSystem = genericFileSystem,
   ): Promise<Document[]> {
     const fileContent: string = await fs.readFile(file, "utf-8");
     const result = Papa.parse(fileContent, this.papaConfig);

packages/core/src/readers/DocxReader.ts

@@ -1,14 +1,13 @@
 import mammoth from "mammoth";
 import { Document } from "../Node";
-import { GenericFileSystem } from "../storage/FileSystem";
-import { DEFAULT_FS } from "../storage/constants";
+import { genericFileSystem, GenericFileSystem } from "../storage/FileSystem";
 import { BaseReader } from "./base";
 
 export class DocxReader implements BaseReader {
   /** DocxParser */
   async loadData(
     file: string,
-    fs: GenericFileSystem = DEFAULT_FS,
+    fs: GenericFileSystem = genericFileSystem,
   ): Promise<Document[]> {
     const dataBuffer = (await fs.readFile(file)) as any;
     const { value } = await mammoth.extractRawText({ buffer: dataBuffer });

packages/core/src/readers/HTMLReader.ts

@@ -1,6 +1,5 @@
 import { Document } from "../Node";
-import { DEFAULT_FS } from "../storage/constants";
-import { GenericFileSystem } from "../storage/FileSystem";
+import { genericFileSystem, GenericFileSystem } from "../storage/FileSystem";
 import { BaseReader } from "./base";
 
 /**
@@ -20,7 +19,7 @@ export class HTMLReader implements BaseReader {
    */
   async loadData(
     file: string,
-    fs: GenericFileSystem = DEFAULT_FS,
+    fs: GenericFileSystem = genericFileSystem,
   ): Promise<Document[]> {
     const dataBuffer = await fs.readFile(file, "utf-8");
     const htmlOptions = this.getOptions();

packages/core/src/readers/ImageReader.ts

@@ -1,6 +1,5 @@
 import { Document, ImageDocument } from "../Node";
-import { DEFAULT_FS } from "../storage/constants";
-import { GenericFileSystem } from "../storage/FileSystem";
+import { GenericFileSystem, genericFileSystem } from "../storage/FileSystem";
 import { BaseReader } from "./base";
 
 /**
@@ -16,7 +15,7 @@ export class ImageReader implements BaseReader {
    */
   async loadData(
     file: string,
-    fs: GenericFileSystem = DEFAULT_FS,
+    fs: GenericFileSystem = genericFileSystem,
   ): Promise<Document[]> {
     const dataBuffer = await fs.readFile(file);
     const blob = new Blob([dataBuffer]);

packages/core/src/readers/MarkdownReader.ts

@@ -1,5 +1,5 @@
 import { Document } from "../Node";
-import { DEFAULT_FS, GenericFileSystem } from "../storage";
+import { GenericFileSystem, genericFileSystem } from "../storage/FileSystem";
 import { BaseReader } from "./base";
 
 type MarkdownTuple = [string | null, string];
@@ -89,7 +89,7 @@ export class MarkdownReader implements BaseReader {
 
   async loadData(
     file: string,
-    fs: GenericFileSystem = DEFAULT_FS,
+    fs: GenericFileSystem = genericFileSystem,
   ): Promise<Document[]> {
     const content = await fs.readFile(file, { encoding: "utf-8" });
     const tups = this.parseTups(content);

packages/core/src/readers/PDFReader.ts

@@ -1,6 +1,5 @@
 import { Document } from "../Node";
-import { GenericFileSystem } from "../storage/FileSystem";
-import { DEFAULT_FS } from "../storage/constants";
+import { genericFileSystem, GenericFileSystem } from "../storage/FileSystem";
 import { BaseReader } from "./base";
 
 /**
@@ -9,7 +8,7 @@ import { BaseReader } from "./base";
 export class PDFReader implements BaseReader {
   async loadData(
     file: string,
-    fs: GenericFileSystem = DEFAULT_FS,
+    fs: GenericFileSystem = genericFileSystem,
   ): Promise<Document[]> {
     const content = (await fs.readFile(file)) as any;
     if (!(content instanceof Buffer)) {

packages/core/src/readers/SimpleDirectoryReader.ts

@@ -1,7 +1,10 @@
 import _ from "lodash";
 import { Document } from "../Node";
-import { CompleteFileSystem, walk } from "../storage/FileSystem";
-import { DEFAULT_FS } from "../storage/constants";
+import {
+  CompleteFileSystem,
+  genericFileSystem,
+  walk,
+} from "../storage/FileSystem";
 import { PapaCSVReader } from "./CSVReader";
 import { DocxReader } from "./DocxReader";
 import { HTMLReader } from "./HTMLReader";
@@ -28,7 +31,7 @@ enum ReaderStatus {
 export class TextFileReader implements BaseReader {
   async loadData(
     file: string,
-    fs: CompleteFileSystem = DEFAULT_FS as CompleteFileSystem,
+    fs: CompleteFileSystem = genericFileSystem as CompleteFileSystem,
   ): Promise<Document[]> {
     const dataBuffer = await fs.readFile(file, "utf-8");
     return [new Document({ text: dataBuffer, id_: file })];
@@ -66,7 +69,7 @@ export class SimpleDirectoryReader implements BaseReader {
 
   async loadData({
     directoryPath,
-    fs = DEFAULT_FS as CompleteFileSystem,
+    fs = genericFileSystem,
     defaultReader = new TextFileReader(),
     fileExtToReader = FILE_EXT_TO_READER,
   }: SimpleDirectoryReaderLoadDataProps): Promise<Document[]> {

packages/core/src/storage/FileSystem.core.ts

@@ -0,0 +1,96 @@
+import _ from "lodash";
+
+/**
+ * A filesystem interface that is meant to be compatible with
+ * the 'fs' module from Node.js.
+ * Allows for the use of similar inteface implementation on browsers.
+ */
+export interface GenericFileSystem {
+  writeFile(path: string, content: string, options?: any): Promise<void>;
+  readFile(path: string, options?: any): Promise<string>;
+  access(path: string): Promise<void>;
+  mkdir(path: string, options?: any): Promise<void>;
+}
+
+export interface WalkableFileSystem {
+  readdir(path: string): Promise<string[]>;
+  stat(path: string): Promise<any>;
+}
+
+/**
+ * A filesystem implementation that stores files in memory.
+ */
+export class InMemoryFileSystem implements GenericFileSystem {
+  private files: Record<string, any> = {};
+
+  async writeFile(path: string, content: string, options?: any): Promise<void> {
+    this.files[path] = content;
+  }
+
+  async readFile(path: string, options?: any): Promise<string> {
+    if (!(path in this.files)) {
+      throw new Error(`File ${path} does not exist`);
+    }
+    return this.files[path];
+  }
+
+  async access(path: string): Promise<void> {
+    if (!(path in this.files)) {
+      throw new Error(`File ${path} does not exist`);
+    }
+  }
+
+  async mkdir(path: string, options?: any): Promise<void> {
+    this.files[path] = _.get(this.files, path, null);
+  }
+}
+
+export type CompleteFileSystem = GenericFileSystem & WalkableFileSystem;
+
+// FS utility functions
+
+/**
+ * Checks if a file exists.
+ * Analogous to the os.path.exists function from Python.
+ * @param fs The filesystem to use.
+ * @param path The path to the file to check.
+ * @returns A promise that resolves to true if the file exists, false otherwise.
+ */
+export async function exists(
+  fs: GenericFileSystem,
+  path: string,
+): Promise<boolean> {
+  try {
+    await fs.access(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Recursively traverses a directory and yields all the paths to the files in it.
+ * @param fs The filesystem to use.
+ * @param dirPath The path to the directory to traverse.
+ */
+export async function* walk(
+  fs: WalkableFileSystem,
+  dirPath: string,
+): AsyncIterable<string> {
+  if (fs instanceof InMemoryFileSystem) {
+    throw new Error(
+      "The InMemoryFileSystem does not support directory traversal.",
+    );
+  }
+
+  const entries = await fs.readdir(dirPath);
+  for (const entry of entries) {
+    const fullPath = `${dirPath}/${entry}`;
+    const stats = await fs.stat(fullPath);
+    if (stats.isDirectory()) {
+      yield* walk(fs, fullPath);
+    } else {
+      yield fullPath;
+    }
+  }
+}