Merge pull request #139 from run-llama/esm

Esm

.changeset/big-hairs-eat.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Streaming improvements including Anthropic (thanks @kkang2097)

.changeset/breezy-days-admire.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Portkey integration (Thank you @noble-varghese)

.changeset/honest-students-care.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Add export for PromptHelper (thanks @zigamall)

.changeset/six-hats-compare.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Publish ESM module again

.changeset/twelve-otters-wash.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Pinecone demo (thanks @Einsenhorn)

apps/docs/package.json

@@ -15,24 +15,24 @@
     "typecheck": "tsc"
   },
   "dependencies": {
-    "@docusaurus/core": "2.4.1",
-    "@docusaurus/preset-classic": "2.4.1",
-    "@docusaurus/remark-plugin-npm2yarn": "^2.4.1",
+    "@docusaurus/core": "2.4.3",
+    "@docusaurus/preset-classic": "2.4.3",
+    "@docusaurus/remark-plugin-npm2yarn": "^2.4.3",
     "@mdx-js/react": "^1.6.22",
     "clsx": "^1.2.1",
-    "postcss": "^8.4.28",
+    "postcss": "^8.4.31",
     "prism-react-renderer": "^1.3.5",
     "raw-loader": "^4.0.2",
     "react": "^17.0.2",
     "react-dom": "^17.0.2"
   },
   "devDependencies": {
-    "@docusaurus/module-type-aliases": "2.4.1",
-    "@docusaurus/types": "^2.4.1",
-    "@tsconfig/docusaurus": "^1.0.7",
+    "@docusaurus/module-type-aliases": "2.4.3",
+    "@docusaurus/types": "^2.4.3",
+    "@tsconfig/docusaurus": "^2.0.1",
     "docusaurus-plugin-typedoc": "^0.19.2",
     "typedoc": "^0.24.8",
-    "typedoc-plugin-markdown": "^3.15.4",
+    "typedoc-plugin-markdown": "^3.16.0",
     "typescript": "^4.9.5"
   },
   "browserslist": {

package.json

@@ -11,7 +11,7 @@
     "publish-snapshot": "turbo run build lint test && changeset version --snapshot && changeset publish"
   },
   "devDependencies": {
-    "@turbo/gen": "^1.10.14",
+    "@turbo/gen": "^1.10.15",
     "@types/jest": "^29.5.5",
     "eslint": "^7.32.0",
     "eslint-config-custom": "workspace:*",
@@ -20,7 +20,7 @@
     "prettier": "^3.0.3",
     "prettier-plugin-organize-imports": "^3.2.3",
     "ts-jest": "^29.1.1",
-    "turbo": "^1.10.14"
+    "turbo": "^1.10.15"
   },
   "packageManager": "pnpm@7.15.0",
   "dependencies": {

packages/core/package.json

@@ -9,30 +9,32 @@
     "md-utils-ts": "^2.0.0",
     "mongodb": "^6.1.0",
     "notion-md-crawler": "^0.0.2",
-    "openai": "^4.10.0",
+    "openai": "^4.11.1",
     "papaparse": "^5.4.1",
     "pdf-parse": "^1.1.1",
+    "portkey-ai": "^0.1.11",
     "rake-modified": "^1.0.8",
-    "replicate": "^0.18.1",
+    "replicate": "^0.20.0",
     "tiktoken": "^1.0.10",
     "uuid": "^9.0.1",
-    "wink-nlp": "^1.14.3",
-    "portkey-ai": "^0.1.11"
+    "wink-nlp": "^1.14.3"
   },
   "devDependencies": {
     "@types/lodash": "^4.14.199",
-    "@types/node": "^18.18.0",
+    "@types/node": "^18.18.4",
     "@types/papaparse": "^5.3.9",
     "@types/pdf-parse": "^1.1.2",
-    "@types/uuid": "^9.0.4",
+    "@types/uuid": "^9.0.5",
     "node-stdlib-browser": "^1.2.0",
-    "tsup": "^7.2.0"
+    "tsup": "^7.2.0",
+    "typescript": "^4.9.5"
   },
   "engines": {
     "node": ">=18.0.0"
   },
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",
+  "module": "./dist/index.mjs",
   "scripts": {
     "lint": "eslint .",
     "test": "jest",

packages/core/src/ChatEngine.ts

@@ -317,7 +317,7 @@ export class HistoryChatEngine implements ChatEngine {
       return this.streamChat(message, chatHistory) as R;
     }
     this.chatHistory.addMessage({ content: message, role: "user" });
-    const response = await this.llm.chat(this.chatHistory.messages);
+    const response = await this.llm.chat(this.chatHistory.requestMessages);
     this.chatHistory.addMessage(response.message);
     return new Response(response.message.content) as R;
   }
@@ -328,7 +328,7 @@ export class HistoryChatEngine implements ChatEngine {
   ): AsyncGenerator<string, void, unknown> {
     this.chatHistory.addMessage({ content: message, role: "user" });
     const response_stream = await this.llm.chat(
-      this.chatHistory.messages,
+      this.chatHistory.requestMessages,
       undefined,
       true,
     );

packages/core/src/ChatHistory.ts

@@ -16,6 +16,11 @@ export interface ChatHistory {
    */
   addMessage(message: ChatMessage): Promise<void>;
 
+  /**
+   * Returns the messages that should be used as input to the LLM.
+   */
+  requestMessages: ChatMessage[];
+
   /**
    * Resets the chat history so that it's empty.
    */
@@ -28,45 +33,86 @@ export class SimpleChatHistory implements ChatHistory {
   constructor(init?: Partial<SimpleChatHistory>) {
     this.messages = init?.messages ?? [];
   }
-
   async addMessage(message: ChatMessage) {
     this.messages.push(message);
   }
 
+  get requestMessages() {
+    return this.messages;
+  }
+
   reset() {
     this.messages = [];
   }
 }
 
 export class SummaryChatHistory implements ChatHistory {
+  messagesToSummarize: number;
   messages: ChatMessage[];
   summaryPrompt: SummaryPrompt;
   llm: LLM;
 
   constructor(init?: Partial<SummaryChatHistory>) {
+    this.messagesToSummarize = init?.messagesToSummarize ?? 5;
     this.messages = init?.messages ?? [];
     this.summaryPrompt = init?.summaryPrompt ?? defaultSummaryPrompt;
     this.llm = init?.llm ?? new OpenAI();
   }
 
   private async summarize() {
-    const chatHistoryStr = messagesToHistoryStr(this.messages);
+    // get all messages after the last summary message (including)
+    const chatHistoryStr = messagesToHistoryStr(
+      this.messages.slice(this.getLastSummaryIndex()),
+    );
 
     const response = await this.llm.complete(
       this.summaryPrompt({ context: chatHistoryStr }),
     );
 
-    this.messages = [{ content: response.message.content, role: "system" }];
+    this.messages.push({ content: response.message.content, role: "memory" });
   }
 
   async addMessage(message: ChatMessage) {
-    // TODO: check if summarization is necessary
-    // TBD what are good conditions, e.g. depending on the context length of the LLM?
-    // for now we just have a dummy implementation at always summarizes the messages
-    await this.summarize();
+    const lastSummaryIndex = this.getLastSummaryIndex();
+    // if there are more than or equal `messagesToSummarize` messages since the last summary, call summarize
+    if (
+      lastSummaryIndex !== -1 &&
+      this.messages.length - lastSummaryIndex - 1 >= this.messagesToSummarize
+    ) {
+      // TODO: define what are better conditions, e.g. depending on the context length of the LLM?
+      // for now we just summarize each `messagesToSummarize` messages
+      await this.summarize();
+    }
     this.messages.push(message);
   }
 
+  // Find last summary message
+  private getLastSummaryIndex() {
+    return this.messages
+      .slice()
+      .reverse()
+      .findIndex((message) => message.role === "memory");
+  }
+
+  get requestMessages() {
+    const lastSummaryIndex = this.getLastSummaryIndex();
+    // get array of all system messages
+    const systemMessages = this.messages.filter(
+      (message) => message.role === "system",
+    );
+    // convert summary message so it can be send to the LLM
+    const summaryMessage: ChatMessage = {
+      content: `This is a summary of conversation so far: ${this.messages[lastSummaryIndex].content}`,
+      role: "system",
+    };
+    // return system messages, last summary and all messages after the last summary message
+    return [
+      ...systemMessages,
+      summaryMessage,
+      ...this.messages.slice(lastSummaryIndex + 1),
+    ];
+  }
+
   reset() {
     this.messages = [];
   }

packages/core/src/index.ts

@@ -1,30 +1,27 @@
+export * from "./callbacks/CallbackManager";
 export * from "./ChatEngine";
+export * from "./constants";
 export * from "./Embedding";
 export * from "./GlobalsHelper";
+export * from "./indices";
+export * from "./llm/LLM";
 export * from "./Node";
 export * from "./NodeParser";
 export * from "./OutputParser";
 export * from "./Prompt";
+export * from "./PromptHelper";
 export * from "./QueryEngine";
 export * from "./QuestionGenerator";
-export * from "./Response";
-export * from "./ResponseSynthesizer";
-export * from "./Retriever";
-export * from "./ServiceContext";
-export * from "./TextSplitter";
-export * from "./Tool";
-export * from "./constants";
-export * from "./llm/LLM";
-
-export * from "./indices";
-
-export * from "./callbacks/CallbackManager";
-
+export * from "./readers/base";
 export * from "./readers/CSVReader";
 export * from "./readers/MarkdownReader";
 export * from "./readers/NotionReader";
 export * from "./readers/PDFReader";
 export * from "./readers/SimpleDirectoryReader";
-export * from "./readers/base";
-
+export * from "./Response";
+export * from "./ResponseSynthesizer";
+export * from "./Retriever";
+export * from "./ServiceContext";
 export * from "./storage";
+export * from "./TextSplitter";
+export * from "./Tool";

packages/core/src/llm/LLM.ts

@@ -23,7 +23,7 @@ import {
   shouldUseAzure,
 } from "./azure";
 import { getOpenAISession, OpenAISession } from "./openai";
-import { PortkeySession, getPortkeySession } from "./portkey";
+import { getPortkeySession, PortkeySession } from "./portkey";
 import { ReplicateSession } from "./replicate";
 
 export type MessageType =
@@ -31,7 +31,8 @@ export type MessageType =
   | "assistant"
   | "system"
   | "generic"
-  | "function";
+  | "function"
+  | "memory";
 
 export interface ChatMessage {
   content: string;
@@ -51,7 +52,7 @@ export type CompletionResponse = ChatResponse;
  * Unified language model interface
  */
 export interface LLM {
-  //Whether a LLM has streaming support
+  // Whether a LLM has streaming support
   hasStreaming: boolean;
   /**
    * Get a chat response from the LLM
@@ -104,16 +105,17 @@ export const ALL_AVAILABLE_OPENAI_MODELS = {
  * OpenAI LLM implementation
  */
 export class OpenAI implements LLM {
+  hasStreaming: boolean = true;
+
   // Per completion OpenAI params
   model: keyof typeof ALL_AVAILABLE_OPENAI_MODELS;
   temperature: number;
   topP: number;
   maxTokens?: number;
   additionalChatOptions?: Omit<
-    Partial<OpenAILLM.Chat.CompletionCreateParams>,
+    Partial<OpenAILLM.Chat.ChatCompletionCreateParams>,
     "max_tokens" | "messages" | "model" | "temperature" | "top_p" | "streaming"
   >;
-  hasStreaming: boolean;
 
   // OpenAI session params
   apiKey?: string = undefined;
@@ -141,7 +143,6 @@ export class OpenAI implements LLM {
     this.timeout = init?.timeout ?? 60 * 1000; // Default is 60 seconds
     this.additionalChatOptions = init?.additionalChatOptions;
     this.additionalSessionOptions = init?.additionalSessionOptions;
-    this.hasStreaming = init?.hasStreaming ?? true;
 
     if (init?.azure || shouldUseAzure()) {
       const azureConfig = getAzureConfigFromEnv({
@@ -203,7 +204,7 @@ export class OpenAI implements LLM {
     T extends boolean | undefined = undefined,
     R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
   >(messages: ChatMessage[], parentEvent?: Event, streaming?: T): Promise<R> {
-    const baseRequestParams: OpenAILLM.Chat.CompletionCreateParams = {
+    const baseRequestParams: OpenAILLM.Chat.ChatCompletionCreateParams = {
       model: this.model,
       temperature: this.temperature,
       max_tokens: this.maxTokens,
@@ -251,7 +252,7 @@ export class OpenAI implements LLM {
     messages: ChatMessage[],
     parentEvent?: Event,
   ): AsyncGenerator<string, void, unknown> {
-    const baseRequestParams: OpenAILLM.Chat.CompletionCreateParams = {
+    const baseRequestParams: OpenAILLM.Chat.ChatCompletionCreateParams = {
       model: this.model,
       temperature: this.temperature,
       max_tokens: this.maxTokens,
@@ -549,6 +550,8 @@ If a question does not make any sense, or is not factually coherent, explain why
  */
 
 export class Anthropic implements LLM {
+  hasStreaming: boolean = true;
+
   // Per completion Anthropic params
   model: string;
   temperature: number;
@@ -560,7 +563,6 @@ export class Anthropic implements LLM {
   maxRetries: number;
   timeout?: number;
   session: AnthropicSession;
-  hasStreaming: boolean;
 
   callbackManager?: CallbackManager;
 
@@ -580,7 +582,6 @@ export class Anthropic implements LLM {
         maxRetries: this.maxRetries,
         timeout: this.timeout,
       });
-    this.hasStreaming = init?.hasStreaming ?? true;
 
     this.callbackManager = init?.callbackManager;
   }
@@ -682,8 +683,9 @@ export class Anthropic implements LLM {
   }
 }
 
-
 export class Portkey implements LLM {
+  hasStreaming: boolean = true;
+
   apiKey?: string = undefined;
   baseURL?: string = undefined;
   mode?: string = undefined;
@@ -692,57 +694,73 @@ export class Portkey implements LLM {
   callbackManager?: CallbackManager;
 
   constructor(init?: Partial<Portkey>) {
-    this.apiKey = init?.apiKey
-    this.baseURL = init?.baseURL
-    this.mode = init?.mode
-    this.llms = init?.llms
+    this.apiKey = init?.apiKey;
+    this.baseURL = init?.baseURL;
+    this.mode = init?.mode;
+    this.llms = init?.llms;
     this.session = getPortkeySession({
       apiKey: this.apiKey,
       baseURL: this.baseURL,
       llms: this.llms,
-      mode: this.mode
-    })
+      mode: this.mode,
+    });
     this.callbackManager = init?.callbackManager;
   }
 
-  async chat(
+  async chat<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(
     messages: ChatMessage[],
     parentEvent?: Event | undefined,
-    params?: Record<string, any>
-  ): Promise<ChatResponse> {
-    const resolvedParams = params || {}
-    const response = await this.session.portkey.chatCompletions.create({
-      messages,
-      ...resolvedParams
-    });
+    streaming?: T,
+    params?: Record<string, any>,
+  ): Promise<R> {
+    if (streaming) {
+      return this.streamChat(messages, parentEvent, params) as R;
+    } else {
+      const resolvedParams = params || {};
+      const response = await this.session.portkey.chatCompletions.create({
+        messages,
+        ...resolvedParams,
+      });
 
-    const content = response.choices[0].message?.content ?? "";
-    const role = response.choices[0].message?.role || "assistant";
-    return { message: { content, role: role as MessageType } };
+      const content = response.choices[0].message?.content ?? "";
+      const role = response.choices[0].message?.role || "assistant";
+      return { message: { content, role: role as MessageType } } as R;
+    }
   }
-  async complete(
+
+  async complete<
+    T extends boolean | undefined = undefined,
+    R = T extends true ? AsyncGenerator<string, void, unknown> : ChatResponse,
+  >(
     prompt: string,
     parentEvent?: Event | undefined,
-  ): Promise<CompletionResponse> {
-    return this.chat([{ content: prompt, role: "user" }], parentEvent);
+    streaming?: T,
+  ): Promise<R> {
+    return this.chat(
+      [{ content: prompt, role: "user" }],
+      parentEvent,
+      streaming,
+    );
   }
 
-  async *stream_chat(
+  async *streamChat(
     messages: ChatMessage[],
     parentEvent?: Event,
-    params?: Record<string, any>
+    params?: Record<string, any>,
   ): AsyncGenerator<string, void, unknown> {
     // Wrapping the stream in a callback.
     const onLLMStream = this.callbackManager?.onLLMStream
       ? this.callbackManager.onLLMStream
       : () => {};
 
-    const chunk_stream =
-      await this.session.portkey.chatCompletions.create({
-        messages,
-        ...params,
-        stream: true,
-      });
+    const chunkStream = await this.session.portkey.chatCompletions.create({
+      messages,
+      ...params,
+      stream: true,
+    });
 
     const event: Event = parentEvent
       ? parentEvent
@@ -753,7 +771,7 @@ export class Portkey implements LLM {
 
     //Indices
     var idx_counter: number = 0;
-    for await (const part of chunk_stream) {
+    for await (const part of chunkStream) {
       //Increment
       part.choices[0].index = idx_counter;
       const is_done: boolean =
@@ -775,10 +793,10 @@ export class Portkey implements LLM {
     return;
   }
 
-  stream_complete(
+  streamComplete(
     query: string,
     parentEvent?: Event,
   ): AsyncGenerator<string, void, unknown> {
-    return this.stream_chat([{ content: query, role: "user" }], parentEvent);
+    return this.streamChat([{ content: query, role: "user" }], parentEvent);
   }
 }

packages/core/tsconfig.json

@@ -6,7 +6,6 @@
     "moduleResolution": "node",
     "preserveWatchOutput": true,
     "skipLibCheck": true,
-    "noEmit": true,
     "strict": true,
     "lib": ["es2015", "dom"],
     "target": "ES2015",