Updated packages/core/src/llm/LLM.ts

packages/core/src/ChatEngine.ts

@@ -17,6 +17,14 @@ import { ChatMessage, LLM, OpenAI } from "./llm/LLM";
 /**
  * A ChatEngine is used to handle back and forth chats between the application and the LLM.
  */
+export interface ChatHistorySummarizer {
+  /**
+   * Summarize the chat history.
+   * @param chatHistory
+   */
+  summarize(chatHistory: ChatMessage[]): ChatMessage[];
+}
+
 export interface ChatEngine {
   /**
    * Send message along with the class's current chat history to the LLM.
@@ -29,6 +37,11 @@ export interface ChatEngine {
    * Resets the chat history so that it's empty.
    */
   reset(): void;
+
+  /**
+   * Summarizer for the chat history.
+   */
+  chatHistorySummarizer: ChatHistorySummarizer;
 }
 
 /**
@@ -46,6 +59,9 @@ export class SimpleChatEngine implements ChatEngine {
   async chat(message: string, chatHistory?: ChatMessage[]): Promise<Response> {
     chatHistory = chatHistory ?? this.chatHistory;
     chatHistory.push({ content: message, role: "user" });
+    if (chatHistory.length > MAX_CHAT_HISTORY_SIZE) {
+      chatHistory = this.chatHistorySummarizer.summarize(chatHistory);
+    }
     const response = await this.llm.chat(chatHistory);
     chatHistory.push(response.message);
     this.chatHistory = chatHistory;

packages/core/src/llm/LLM.ts

@@ -174,6 +174,10 @@ export class OpenAI implements LLM {
     messages: ChatMessage[],
     parentEvent?: Event,
   ): Promise<ChatResponse> {
+    if (messages[0].role === "summary") {
+      // Handle summarized chat history
+    }
+  
     const baseRequestParams: OpenAILLM.Chat.CompletionCreateParams = {
       model: this.model,
       temperature: this.temperature,
@@ -403,11 +407,15 @@ If a question does not make any sense, or is not factually coherent, explain why
     messages: ChatMessage[],
     _parentEvent?: Event,
   ): Promise<ChatResponse> {
+    if (messages[0].role === "summary") {
+      // Handle summarized chat history
+    }
+  
     const api = ALL_AVAILABLE_LLAMADEUCE_MODELS[this.model]
       .replicateApi as `${string}/${string}:${string}`;
-
+  
     const { prompt, systemPrompt } = this.mapMessagesToPrompt(messages);
-
+  
     const replicateOptions: any = {
       input: {
         prompt,
@@ -502,6 +510,10 @@ export class Anthropic implements LLM {
     messages: ChatMessage[],
     parentEvent?: Event | undefined,
   ): Promise<ChatResponse> {
+    if (messages[0].role === "summary") {
+      // Handle summarized chat history
+    }
+  
     const response = await this.session.anthropic.completions.create({
       model: this.model,
       prompt: this.mapMessagesToPrompt(messages),
@@ -509,7 +521,7 @@ export class Anthropic implements LLM {
       temperature: this.temperature,
       top_p: this.topP,
     });
-
+  
     return {
       message: { content: response.completion.trimStart(), role: "assistant" },
       //^ We're trimming the start because Anthropic often starts with a space in the response