diff --git a/notebooks/lesson_2_baseline.ipynb b/notebooks/lesson_2_baseline.ipynb index 5cd0fc5..ba7d025 100644 --- a/notebooks/lesson_2_baseline.ipynb +++ b/notebooks/lesson_2_baseline.ipynb @@ -33,12 +33,20 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 1, "id": "bc3d445b", "metadata": { "lines_to_next_cell": 2 }, - "outputs": [], + "outputs": [ + { + "name": "stdin", + "output_type": "stream", + "text": [ + "ANTHROPIC_API_KEY: ········\n" + ] + } + ], "source": [ "import os\n", "from getpass import getpass\n", @@ -47,6 +55,56 @@ " os.environ[\"ANTHROPIC_API_KEY\"] = getpass(\"ANTHROPIC_API_KEY: \")" ] }, + { + "cell_type": "markdown", + "id": "507fee0d-e3e7-4a71-9a06-902b573ecdd0", + "metadata": {}, + "source": [ + "### Define profile information and example" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "913c8641-c277-442c-bd85-31f48383832e", + "metadata": {}, + "outputs": [], + "source": [ + "# Example user profile\n", + "profile = {\n", + " \"name\": \"John\",\n", + " \"full_name\": \"John Doe\",\n", + " \"user_profile_background\": \"Senior software engineer leading a team of 5 developers\",\n", + "}\n", + "\n", + "prompt_instructions = {\n", + " \"triage_rules\": {\n", + " \"ignore\": \"Marketing newsletters, spam emails, mass company announcements\",\n", + " \"notify\": \"Team member out sick, build system notifications, project status updates\",\n", + " \"respond\": \"Direct questions from team members, meeting requests, critical bug reports\",\n", + " },\n", + " \"agent_instructions\": \"Use these tools when appropriate to help manage John's tasks efficiently.\"\n", + "}\n", + "\n", + "# Example incoming email\n", + "email = {\n", + " \"from\": \"Alice Smith \",\n", + " \"to\": \"John Doe \",\n", + " \"subject\": \"Quick question about API documentation\",\n", + " \"body\": \"\"\"\n", + "Hi John,\n", + "\n", + "I was reviewing the API documentation for the new authentication service and noticed a few endpoints seem to be missing from the specs. Could you help clarify if this was intentional or if we should update the docs?\n", + "\n", + "Specifically, I'm looking at:\n", + "- /auth/refresh\n", + "- /auth/validate\n", + "\n", + "Thanks!\n", + "Alice\"\"\",\n", + "}" + ] + }, { "cell_type": "markdown", "id": "0dc3e26f", @@ -68,12 +126,12 @@ "output_type": "stream", "text": [ "Classification: respond\n", - "Reason: 1. This is a direct question from a team member (Alice) about technical documentation\n", - "2. The question is specific and requires John's expertise as the senior software engineer\n", - "3. The issue relates to API documentation for authentication service, which is important for the team's work\n", - "4. Missing API endpoints in documentation could lead to confusion or implementation issues\n", - "5. As team lead, John should provide clarity on whether these endpoints were intentionally omitted or need to be documented\n", - "6. A timely response would help maintain documentation accuracy and team productivity\n" + "Reason: 1. This is a direct question from a team member (Alice)\n", + "2. It's regarding important technical documentation (API endpoints)\n", + "3. The question requires specific information about whether certain API endpoints were intentionally omitted\n", + "4. As a team lead, John should address documentation concerns to ensure proper team coordination\n", + "5. This is related to authentication service which is typically a critical component\n", + "6. A lack of response could lead to confusion or implementation issues\n" ] } ], @@ -87,43 +145,14 @@ "# We'll use structured output to generate classification results\n", "llm_router = llm.with_structured_output(Router)\n", "\n", - "# Example user profile\n", - "profile = {\n", - " \"name\": \"John\",\n", - " \"full_name\": \"John Doe\",\n", - " \"user_profile_background\": \"Senior software engineer leading a team of 5 developers\",\n", - " \"triage_rules\": {\n", - " \"ignore\": \"Marketing newsletters, spam emails, mass company announcements\",\n", - " \"notify\": \"Team member out sick, build system notifications, project status updates\",\n", - " \"respond\": \"Direct questions from team members, meeting requests, critical bug reports\",\n", - " },\n", - "}\n", - "\n", - "# Example incoming email\n", - "email = {\n", - " \"from\": \"Alice Smith \",\n", - " \"to\": \"John Doe \",\n", - " \"subject\": \"Quick question about API documentation\",\n", - " \"body\": \"\"\"\n", - "Hi John,\n", - "\n", - "I was reviewing the API documentation for the new authentication service and noticed a few endpoints seem to be missing from the specs. Could you help clarify if this was intentional or if we should update the docs?\n", - "\n", - "Specifically, I'm looking at:\n", - "- /auth/refresh\n", - "- /auth/validate\n", - "\n", - "Thanks!\n", - "Alice\"\"\",\n", - "}\n", - "\n", "system_prompt = triage_system_prompt.format(\n", " full_name=profile[\"full_name\"],\n", " name=profile[\"name\"],\n", + " examples=None,\n", " user_profile_background=profile[\"user_profile_background\"],\n", - " triage_no=profile[\"triage_rules\"][\"ignore\"],\n", - " triage_notify=profile[\"triage_rules\"][\"notify\"],\n", - " triage_email=profile[\"triage_rules\"][\"respond\"],\n", + " triage_no=prompt_instructions[\"triage_rules\"][\"ignore\"],\n", + " triage_notify=prompt_instructions[\"triage_rules\"][\"notify\"],\n", + " triage_email=prompt_instructions[\"triage_rules\"][\"respond\"],\n", ")\n", "\n", "user_prompt = triage_user_prompt.format(\n", @@ -186,9 +215,30 @@ " return f\"Available times on {day}: 9:00 AM, 2:00 PM, 4:00 PM\"" ] }, + { + "cell_type": "markdown", + "id": "6d7b02e3-6105-46e0-93b7-3d507566ff35", + "metadata": {}, + "source": [ + "### Create agent" + ] + }, { "cell_type": "code", "execution_count": 6, + "id": "d9ddcdb8-0fd6-4320-b713-264df726f956", + "metadata": {}, + "outputs": [], + "source": [ + "def create_prompt(state):\n", + " return [\n", + " {\"role\": \"system\", \"content\": agent_system_prompt.format(instructions=prompt_instructions[\"agent_instructions\"], **profile)}\n", + " ] + state['messages']" + ] + }, + { + "cell_type": "code", + "execution_count": 7, "id": "167a1474", "metadata": {}, "outputs": [ @@ -198,24 +248,24 @@ "text": [ "==================================\u001b[1m Ai Message \u001b[0m==================================\n", "\n", - "John, you have three available time slots on Tuesday:\n", + "Based on the calendar check, you have three available time slots on Tuesday:\n", "- 9:00 AM\n", "- 2:00 PM\n", "- 4:00 PM\n", "\n", - "Would you like me to schedule any meetings during these times?\n" + "Would you like me to schedule any meetings during these available times?\n" ] } ], "source": [ - "# Create agent\n", + "\n", "from memory_course.prompts import agent_system_prompt\n", "from langgraph.prebuilt import create_react_agent\n", "\n", "agent = create_react_agent(\n", " \"anthropic:claude-3-5-sonnet-latest\",\n", " tools=[write_email, schedule_meeting, check_calendar_availability],\n", - " prompt=agent_system_prompt.format(**profile),\n", + " prompt=create_prompt,\n", ")\n", "\n", "# Test the agent\n", @@ -232,14 +282,12 @@ "source": [ "### Build agent\n", "\n", - "Combine triage with tool calling agent\n", - "\n", - "TODO: Explain `Command`\n" + "Combine triage with tool calling agent\n" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "id": "55cf741a", "metadata": { "lines_to_next_cell": 2 @@ -278,9 +326,9 @@ " full_name=profile[\"full_name\"],\n", " name=profile[\"name\"],\n", " user_profile_background=profile[\"user_profile_background\"],\n", - " triage_no=profile[\"triage_rules\"][\"ignore\"],\n", - " triage_notify=profile[\"triage_rules\"][\"notify\"],\n", - " triage_email=profile[\"triage_rules\"][\"respond\"],\n", + " triage_no=prompt_instructions[\"triage_rules\"][\"ignore\"],\n", + " triage_notify=prompt_instructions[\"triage_rules\"][\"notify\"],\n", + " triage_email=prompt_instructions[\"triage_rules\"][\"respond\"],\n", " examples=None\n", " )\n", "\n", @@ -294,8 +342,6 @@ " {\"role\": \"user\", \"content\": user_prompt},\n", " ]\n", " )\n", - " update = None\n", - " goto = END\n", " if result.classification == \"respond\":\n", " print(\"📧 Classification: RESPOND - This email requires a response\")\n", " goto = \"response_agent\"\n", @@ -309,8 +355,13 @@ " }\n", " elif result.classification == \"ignore\":\n", " print(\"🚫 Classification: IGNORE - This email can be safely ignored\")\n", + " update = None\n", + " goto = END\n", " elif result.classification == \"notify\":\n", + " # If real life, this would do something else\n", " print(\"🔔 Classification: NOTIFY - This email contains important information\")\n", + " update = None\n", + " goto = END\n", " else:\n", " raise ValueError(f\"Invalid classification: {result.classification}\")\n", " return Command(goto=goto, update=update)\n", @@ -331,7 +382,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "id": "12ae5d5c", "metadata": {}, "outputs": [ @@ -380,7 +431,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "id": "ebca8e98", "metadata": {}, "outputs": [ @@ -394,30 +445,36 @@ "Respond to the email {'author': 'Alice Smith ', 'to': 'John Doe ', 'subject': 'Quick question about API documentation', 'email_thread': \"Hi John,\\n\\nI was reviewing the API documentation for the new authentication service and noticed a few endpoints seem to be missing from the specs. Could you help clarify if this was intentional or if we should update the docs?\\n\\nSpecifically, I'm looking at:\\n- /auth/refresh\\n- /auth/validate\\n\\nThanks!\\nAlice\"}\n", "==================================\u001b[1m Ai Message \u001b[0m==================================\n", "\n", - "[{'citations': None, 'text': \"I'll help draft and send a response to Alice regarding her API documentation question. Since the email should be a continuation of the thread, I'll use the same subject.\", 'type': 'text'}, {'id': 'toolu_01VDxukMeEntJ19K6ZGJebk8', 'input': {'to': 'alice.smith@company.com', 'subject': 'Re: Quick question about API documentation', 'content': \"Hi Alice,\\n\\nThanks for bringing this to John's attention. I'm John's assistant, and I'll make sure he reviews this right away.\\n\\nI've added this to his priority list, and he will get back to you with clarification about the /auth/refresh and /auth/validate endpoints as soon as possible. Would you like me to schedule a quick meeting between you and John to discuss this in detail?\\n\\nBest regards,\\nExecutive Assistant to John Doe\"}, 'name': 'write_email', 'type': 'tool_use'}]\n", + "[{'citations': None, 'text': \"I'll help draft a polite and professional response to Alice regarding her API documentation query. I'll use the write_email function to send the response.\", 'type': 'text'}, {'id': 'toolu_015dZQfhhfrR3pUopagsZFhk', 'input': {'to': 'alice.smith@company.com', 'subject': 'Re: Quick question about API documentation', 'content': \"Hi Alice,\\n\\nThank you for bringing this to John's attention. I'm John's assistant, and I've received your inquiry about the API documentation for the authentication service endpoints.\\n\\nI'll make sure John reviews this specific concern about the missing endpoints (/auth/refresh and /auth/validate) in the documentation. He'll get back to you with clarification about whether these omissions were intentional or if the documentation needs to be updated.\\n\\nIn the meantime, would it be helpful to schedule a brief meeting with John to discuss this in detail?\\n\\nBest regards,\\nJohn's Assistant\"}, 'name': 'write_email', 'type': 'tool_use'}]\n", "Tool Calls:\n", - " write_email (toolu_01VDxukMeEntJ19K6ZGJebk8)\n", - " Call ID: toolu_01VDxukMeEntJ19K6ZGJebk8\n", + " write_email (toolu_015dZQfhhfrR3pUopagsZFhk)\n", + " Call ID: toolu_015dZQfhhfrR3pUopagsZFhk\n", " Args:\n", " to: alice.smith@company.com\n", " subject: Re: Quick question about API documentation\n", " content: Hi Alice,\n", "\n", - "Thanks for bringing this to John's attention. I'm John's assistant, and I'll make sure he reviews this right away.\n", + "Thank you for bringing this to John's attention. I'm John's assistant, and I've received your inquiry about the API documentation for the authentication service endpoints.\n", "\n", - "I've added this to his priority list, and he will get back to you with clarification about the /auth/refresh and /auth/validate endpoints as soon as possible. Would you like me to schedule a quick meeting between you and John to discuss this in detail?\n", + "I'll make sure John reviews this specific concern about the missing endpoints (/auth/refresh and /auth/validate) in the documentation. He'll get back to you with clarification about whether these omissions were intentional or if the documentation needs to be updated.\n", + "\n", + "In the meantime, would it be helpful to schedule a brief meeting with John to discuss this in detail?\n", "\n", "Best regards,\n", - "Executive Assistant to John Doe\n", + "John's Assistant\n", "=================================\u001b[1m Tool Message \u001b[0m=================================\n", "Name: write_email\n", "\n", "Email sent to alice.smith@company.com with subject 'Re: Quick question about API documentation'\n", "==================================\u001b[1m Ai Message \u001b[0m==================================\n", "\n", - "I've sent an initial response to acknowledge Alice's email and let her know that John will address her concerns about the API documentation. I've also offered to schedule a meeting if needed, which we can arrange once Alice responds with her preference. This ensures the matter is being handled while giving John time to review the specific API documentation concerns.\n", + "I've sent an initial response to Alice that:\n", + "1. Acknowledges receipt of her email\n", + "2. Confirms the specific endpoints she's asking about\n", + "3. Indicates that John will review the concern\n", + "4. Opens the door for a potential meeting if needed for detailed discussion\n", "\n", - "Would you like me to proactively schedule a meeting between John and Alice to discuss this matter in more detail?\n" + "Would you like me to also schedule a follow-up meeting between John and Alice to discuss this matter in more detail?\n" ] } ], @@ -443,22 +500,6 @@ " m.pretty_print()" ] }, - { - "cell_type": "markdown", - "id": "38e501e9", - "metadata": {}, - "source": [ - "### What We Learned\n", - "\n", - "Running these examples shows the code works, but it reveals some limitations: \n", - " \n", - "All the instructions and preferences are hard-coded in the prompts. \n", - "\n", - "We would like a more flexible approach.\n", - "\n", - "The next lesson explores what happens when we let the assistant learn from these interactions." - ] - }, { "cell_type": "code", "execution_count": null, @@ -484,7 +525,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.2" + "version": "3.11.7" } }, "nbformat": 4, diff --git a/notebooks/lesson_3_semantic.ipynb b/notebooks/lesson_3_semantic.ipynb index 17eda91..505ccc0 100644 --- a/notebooks/lesson_3_semantic.ipynb +++ b/notebooks/lesson_3_semantic.ipynb @@ -46,6 +46,54 @@ " os.environ[\"ANTHROPIC_API_KEY\"] = getpass(\"ANTHROPIC_API_KEY: \")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Define profile information and example" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# Example user profile\n", + "profile = {\n", + " \"name\": \"John\",\n", + " \"full_name\": \"John Doe\",\n", + " \"user_profile_background\": \"Senior software engineer leading a team of 5 developers\",\n", + "}\n", + "\n", + "prompt_instructions = {\n", + " \"triage_rules\": {\n", + " \"ignore\": \"Marketing newsletters, spam emails, mass company announcements\",\n", + " \"notify\": \"Team member out sick, build system notifications, project status updates\",\n", + " \"respond\": \"Direct questions from team members, meeting requests, critical bug reports\",\n", + " },\n", + " \"agent_instructions\": \"Use these tools when appropriate to help manage John's tasks efficiently.\"\n", + "}\n", + "\n", + "# Example incoming email\n", + "email = {\n", + " \"from\": \"Alice Smith \",\n", + " \"to\": \"John Doe \",\n", + " \"subject\": \"Quick question about API documentation\",\n", + " \"body\": \"\"\"\n", + "Hi John,\n", + "\n", + "I was reviewing the API documentation for the new authentication service and noticed a few endpoints seem to be missing from the specs. Could you help clarify if this was intentional or if we should update the docs?\n", + "\n", + "Specifically, I'm looking at:\n", + "- /auth/refresh\n", + "- /auth/validate\n", + "\n", + "Thanks!\n", + "Alice\"\"\",\n", + "}" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -57,7 +105,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -82,7 +130,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -115,100 +163,95 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Define memory store\n", + "### Define memory store and tools\n", "\n", - "TODO: \n", - "* Add details on the Store\n", - "* Add details on [LangMem](https://langchain-ai.github.io/langmem/)\n", - "* Explain tool creation and [signature](https://langchain-ai.github.io/langmem/reference/tools/#langmem.create_manage_memory_tool) used in langmem \n", - "* Explain whether LangMem support memory updating (profile) vs appending to collection?" + "We will now create two tools for managing memory. These are for creating and searching memory collections" ] }, { "cell_type": "code", - "execution_count": 75, - "metadata": {}, - "outputs": [], - "source": [ - "# Example user profile\n", - "example_user_profile = {\n", - " \"name\": \"John\",\n", - " \"full_name\": \"John Doe\",\n", - " \"user_profile_background\": \"Senior software engineer leading a team of 5 developers\",\n", - " \"triage_rules\": {\n", - " \"ignore\": \"Marketing newsletters, spam emails, mass company announcements\",\n", - " \"notify\": \"Team member out sick, build system notifications, project status updates\",\n", - " \"respond\": \"Direct questions from team members, meeting requests, critical bug reports\",\n", - " },\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": 55, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "from langgraph.store.memory import InMemoryStore\n", "\n", "# Memory store\n", - "store = InMemoryStore()\n", + "store = InMemoryStore()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "from langmem import create_manage_memory_tool, create_search_memory_tool\n", "\n", - "# Save user profile\n", - "user = \"lance\"\n", - "namespace = (\"email_assistant\", user, \"user_profile\")\n", - "key = \"user_profile\"\n", - "store.put(namespace, key, example_user_profile)" + "manage_memory_tool = create_manage_memory_tool(namespace=(\"email_assistant\", \"{langgraph_user_id}\", \"collection\"))\n", + "search_memory_tool = create_search_memory_tool(namespace=(\"email_assistant\", \"{langgraph_user_id}\", \"collection\"))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Define agent\n", - "\n", - "TODO: Update system prompt as shown here\n", - "\n", - "* https://langchain-ai.github.io/langmem/hot_path_quickstart/#agent" + "### Define agent\n" ] }, { "cell_type": "code", - "execution_count": 85, + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "agent_system_prompt_memory = \"\"\"\n", + "< Role >\n", + "You are {full_name}'s executive assistant. You are a top-notch executive assistant who cares about {name} performing as well as possible.\n", + "\n", + "\n", + "< Tools >\n", + "You have access to the following tools to help manage {name}'s communications and schedule:\n", + "\n", + "1. write_email(to, subject, content) - Send emails to specified recipients\n", + "2. schedule_meeting(attendees, subject, duration_minutes, preferred_day) - Schedule calendar meetings\n", + "3. check_calendar_availability(day) - Check available time slots for a given day\n", + "4. manage_memory - Store any relevant information about contacts, actions, discussion, etc. in memory for future reference\n", + "5. search_memory - Search for any relevant information that may have been stored in memory\n", + "\n", + "\n", + "< Instructions >\n", + "{instructions}\n", + "\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "# Create agent\n", - "from memory_course.prompts import agent_system_prompt_memory\n", - "\n", - "from langmem import create_manage_memory_tool, create_search_memory_tool\n", "\n", "from langgraph.prebuilt import create_react_agent\n", + "import json\n", "\n", - "# Get profile\n", - "profile = store.get(namespace, key)\n", + "def create_prompt(state):\n", + " return [\n", + " {\"role\": \"system\", \"content\": agent_system_prompt_memory.format(instructions=prompt_instructions[\"agent_instructions\"], **profile)}\n", + " ] + state['messages']\n", "\n", - "# Format profile\n", - "profile_data = profile.value\n", - "format_data = {\n", - " **profile_data, # Include all individual fields (name, full_name, etc)\n", - " 'profile': str(profile_data) # Add the full profile as a string\n", - "}\n", "\n", - "# Instructions\n", - "instructions = \"\"\"\n", - "If updating the user profile, do not omit past information. Simply it with new information from the email that that fits within the profile schema.\n", - "\"\"\"\n", "# Create agent\n", "response_agent = create_react_agent(\n", " \"anthropic:claude-3-5-sonnet-latest\",\n", " tools=[write_email, schedule_meeting, \n", " check_calendar_availability, \n", - " create_manage_memory_tool(namespace=(\"email_assistant\", user, \"collection\")),\n", - " create_manage_memory_tool(namespace=(\"email_assistant\", user, \"user_profile\")),\n", - " create_search_memory_tool(namespace=(\"email_assistant\", user, \"collection\"))\n", + " manage_memory_tool,\n", + " search_memory_tool\n", " ],\n", - " prompt=agent_system_prompt_memory.format(**format_data, instructions=instructions),\n", + " prompt=create_prompt,\n", " # Use this to ensure the store is passed to the agent \n", " store=store\n", ")" @@ -216,129 +259,110 @@ }, { "cell_type": "code", - "execution_count": 81, + "execution_count": 9, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "==================================\u001b[1m Ai Message \u001b[0m==================================\n", - "\n", - "Based on the calendar check, you have three available time slots on Tuesday:\n", - "- 9:00 AM\n", - "- 2:00 PM\n", - "- 4:00 PM\n", - "\n", - "Would you like me to schedule anything during these available times?\n" - ] - } - ], + "outputs": [], "source": [ "# Test the agent\n", + "config = {\"configurable\": {\"langgraph_user_id\": \"lance\"}}\n", "response = response_agent.invoke(\n", - " {\"messages\": [{\"role\": \"user\", \"content\": \"What's my availability on Tuesday?\"}]}\n", - ")\n", - "response[\"messages\"][-1].pretty_print()" + " {\"messages\": [{\"role\": \"user\", \"content\": \"Jim is my friend\"}]},\n", + " config=config\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see that it calls the memory tool to store information about Jim in the memory database." ] }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 10, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "==================================\u001b[1m Ai Message \u001b[0m==================================\n", - "\n", - "No further action is needed for this email as it falls under the \"ignore\" category in John's triage rules. This type of promotional email doesn't require any response or action from John.\n", - "\n", - "If you'd like me to handle this type of email differently in the future, please let me know and I can update the triage rules accordingly.\n" - ] + "data": { + "text/plain": [ + "[HumanMessage(content='Jim is my friend', additional_kwargs={}, response_metadata={}, id='10ba37bf-9734-4d55-8abb-eb1a2b9e1176'),\n", + " AIMessage(content=[{'citations': None, 'text': \"I'll help store this information about Jim in the memory system for future reference.\", 'type': 'text'}, {'id': 'toolu_01VRZj2xsThQuURjbdbNVvuU', 'input': {'content': \"Jim is John Doe's friend.\"}, 'name': 'manage_memory', 'type': 'tool_use'}], additional_kwargs={}, response_metadata={'id': 'msg_016CpRpEi23sKwyXzaj1qG14', 'model': 'claude-3-5-sonnet-20241022', 'stop_reason': 'tool_use', 'stop_sequence': None, 'usage': {'cache_creation_input_tokens': 0, 'cache_read_input_tokens': 0, 'input_tokens': 1121, 'output_tokens': 76}}, id='run-0b659560-f0f2-4134-a73b-7a6ea37a648d-0', tool_calls=[{'name': 'manage_memory', 'args': {'content': \"Jim is John Doe's friend.\"}, 'id': 'toolu_01VRZj2xsThQuURjbdbNVvuU', 'type': 'tool_call'}], usage_metadata={'input_tokens': 1121, 'output_tokens': 76, 'total_tokens': 1197, 'input_token_details': {'cache_read': 0, 'cache_creation': 0}}),\n", + " ToolMessage(content='created memory d890dac2-0b33-44ec-9ae0-1c5010a62793', name='manage_memory', id='27fe7ba6-5f1c-401e-a996-c9d133c2dd63', tool_call_id='toolu_01VRZj2xsThQuURjbdbNVvuU'),\n", + " AIMessage(content=\"I've made a note that Jim is your friend. This information will be helpful for future interactions or communications involving Jim. Is there anything specific about Jim that you'd like me to know or help you with, such as scheduling a meeting or sending an email?\", additional_kwargs={}, response_metadata={'id': 'msg_01YPgpoKSguUdUSx239jJk9T', 'model': 'claude-3-5-sonnet-20241022', 'stop_reason': 'end_turn', 'stop_sequence': None, 'usage': {'cache_creation_input_tokens': 0, 'cache_read_input_tokens': 0, 'input_tokens': 1234, 'output_tokens': 56}}, id='run-deedcba3-893a-4c68-9b15-48c499e9e404-0', usage_metadata={'input_tokens': 1234, 'output_tokens': 56, 'total_tokens': 1290, 'input_token_details': {'cache_read': 0, 'cache_creation': 0}})]" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "email_input = {\n", - " \"author\": \"Marketing Team \",\n", - " \"to\": \"John Doe \",\n", - " \"subject\": \"🔥 EXCLUSIVE OFFER: Limited Time Discount on Developer Tools! 🔥\",\n", - " \"email_thread\": \"\"\"Dear Valued Developer,\n", - "\n", - "Don't miss out on this INCREDIBLE opportunity! \n", - "\n", - "🚀 For a LIMITED TIME ONLY, get 80% OFF on our Premium Developer Suite! \n", - "\n", - "✨ FEATURES:\n", - "- Revolutionary AI-powered code completion\n", - "- Cloud-based development environment\n", - "- 24/7 customer support\n", - "- And much more!\n", - "\n", - "💰 Regular Price: $999/month\n", - "🎉 YOUR SPECIAL PRICE: Just $199/month!\n", - "\n", - "🕒 Hurry! This offer expires in:\n", - "24 HOURS ONLY!\n", - "\n", - "Click here to claim your discount: https://amazingdeals.com/special-offer\n", - "\n", - "Best regards,\n", - "Marketing Team\n", - "---\n", - "To unsubscribe, click here\n", - "\"\"\",\n", - "}\n", - "\n", - "# Test the agent\n", - "response = response_agent.invoke(\n", - " {\"messages\": [{\"role\": \"user\", \"content\": f\"Here's an incoming email: {email_input}\"}]}\n", - ")\n", - "response[\"messages\"][-1].pretty_print()" + "response[\"messages\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If we call it again, we can see that it remembers who Jim is" ] }, { "cell_type": "code", - "execution_count": 69, - "metadata": {}, + "execution_count": 11, + "metadata": { + "scrolled": true + }, "outputs": [ { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - "==================================\u001b[1m Ai Message \u001b[0m==================================\n", - "\n", - "I've taken the following actions:\n", - "1. Sent an acknowledgment email to Alice, letting her know that John will review and respond to her technical question about the API documentation.\n", - "2. Attempted to store the interaction in memory (though there seems to be an error with the memory function).\n", - "\n", - "The response aligns with John's triage rules as this is a direct question from a team member that requires his technical expertise. The immediate acknowledgment helps maintain good communication while ensuring John can provide the accurate technical information needed.\n" + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n" ] } ], "source": [ - "email_input = {\n", - " \"author\": \"Alice Smith \",\n", - " \"to\": \"John Doe \",\n", - " \"subject\": \"Quick question about API documentation\",\n", - " \"email_thread\": \"\"\"Hi John,\n", - "\n", - "I was reviewing the API documentation for the new authentication service and noticed a few endpoints seem to be missing from the specs. Could you help clarify if this was intentional or if we should update the docs?\n", - "\n", - "Specifically, I'm looking at:\n", - "- /auth/refresh\n", - "- /auth/validate\n", - "\n", - "Thanks!\n", - "Alice\"\"\",\n", - "}\n", - "\n", "# Test the agent\n", + "config = {\"configurable\": {\"langgraph_user_id\": \"lance\"}}\n", "response = response_agent.invoke(\n", - " {\"messages\": [{\"role\": \"user\", \"content\": f\"Here's an incoming email: {email_input}\"}]}\n", - ")\n", - "response[\"messages\"][-1].pretty_print()" + " {\"messages\": [{\"role\": \"user\", \"content\": \"who is jim?\"}]},\n", + " config=config\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[HumanMessage(content='who is jim?', additional_kwargs={}, response_metadata={}, id='535b96db-541e-48d6-ae85-45388a4687ab'),\n", + " AIMessage(content=[{'citations': None, 'text': 'Let me search through my memories to see if I have any information about Jim.', 'type': 'text'}, {'id': 'toolu_01QX1caNMSK6PVpnjuVhPQ9o', 'input': {'query': 'Jim'}, 'name': 'search_memory', 'type': 'tool_use'}], additional_kwargs={}, response_metadata={'id': 'msg_015kLvTsp25yfACx3TWnpWmo', 'model': 'claude-3-5-sonnet-20241022', 'stop_reason': 'tool_use', 'stop_sequence': None, 'usage': {'cache_creation_input_tokens': 0, 'cache_read_input_tokens': 0, 'input_tokens': 1121, 'output_tokens': 70}}, id='run-94d767d1-18b4-4567-8af6-11d709fee901-0', tool_calls=[{'name': 'search_memory', 'args': {'query': 'Jim'}, 'id': 'toolu_01QX1caNMSK6PVpnjuVhPQ9o', 'type': 'tool_call'}], usage_metadata={'input_tokens': 1121, 'output_tokens': 70, 'total_tokens': 1191, 'input_token_details': {'cache_read': 0, 'cache_creation': 0}}),\n", + " ToolMessage(content='[{\"namespace\": [\"email_assistant\", \"lance\", \"collection\"], \"key\": \"d890dac2-0b33-44ec-9ae0-1c5010a62793\", \"value\": {\"content\": \"Jim is John Doe\\'s friend.\"}, \"created_at\": \"2025-02-09T21:01:46.167740+00:00\", \"updated_at\": \"2025-02-09T21:01:46.167744+00:00\", \"score\": null}]', name='search_memory', id='994ab38c-bb11-4e16-9b9b-801b2d161ac3', tool_call_id='toolu_01QX1caNMSK6PVpnjuVhPQ9o', artifact=[Item(namespace=['email_assistant', 'lance', 'collection'], key='d890dac2-0b33-44ec-9ae0-1c5010a62793', value={'content': \"Jim is John Doe's friend.\"}, created_at='2025-02-09T21:01:46.167740+00:00', updated_at='2025-02-09T21:01:46.167744+00:00', score=None)]),\n", + " AIMessage(content=\"Based on the stored memory, Jim is John Doe's friend. However, I don't seem to have much detailed information about him. If you'd like to know more specific details about Jim or if you'd like me to remember additional information about him, please let me know and I can store that in my memory for future reference.\", additional_kwargs={}, response_metadata={'id': 'msg_017gcjTE1KDxvNjzjCEXquqo', 'model': 'claude-3-5-sonnet-20241022', 'stop_reason': 'end_turn', 'stop_sequence': None, 'usage': {'cache_creation_input_tokens': 0, 'cache_read_input_tokens': 0, 'input_tokens': 1319, 'output_tokens': 72}}, id='run-76c93bf2-c65a-4195-8aac-c140aee65004-0', usage_metadata={'input_tokens': 1319, 'output_tokens': 72, 'total_tokens': 1391, 'input_token_details': {'cache_read': 0, 'cache_creation': 0}})]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response[\"messages\"]" ] }, { @@ -353,7 +377,7 @@ }, { "cell_type": "code", - "execution_count": 83, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -373,11 +397,12 @@ "\n", "from langgraph.graph import StateGraph, START, END\n", "from langgraph.types import Command\n", + "from langgraph.store.base import BaseStore\n", "\n", "from memory_course.schemas import State\n", "from memory_course.utils import parse_email\n", "\n", - "def triage_router(state: State, store: BaseStore) -> Command[Literal[\"response_agent\", \"__end__\"]]:\n", + "def triage_router(state: State) -> Command[Literal[\"response_agent\", \"__end__\"]]:\n", " \"\"\"Analyze email content to decide if we should respond, notify, or ignore.\n", "\n", " The triage step prevents the assistant from wasting time on:\n", @@ -388,19 +413,13 @@ "\n", " # Parse email\n", " author, to, subject, email_thread = parse_email(state[\"email_input\"])\n", - " \n", - " # Get profile from store\n", - " profile = store.get(namespace, key)\n", - " profile_data = profile.value\n", - "\n", - " author, to, subject, email_thread = parse_email(state[\"email_input\"])\n", " system_prompt = triage_system_prompt.format(\n", - " full_name=profile_data[\"full_name\"],\n", - " name=profile_data[\"name\"],\n", - " user_profile_background=profile_data[\"user_profile_background\"],\n", - " triage_no=profile_data[\"triage_rules\"][\"ignore\"],\n", - " triage_notify=profile_data[\"triage_rules\"][\"notify\"],\n", - " triage_email=profile_data[\"triage_rules\"][\"respond\"],\n", + " full_name=profile[\"full_name\"],\n", + " name=profile[\"name\"],\n", + " user_profile_background=profile[\"user_profile_background\"],\n", + " triage_no=prompt_instructions[\"triage_rules\"][\"ignore\"],\n", + " triage_notify=prompt_instructions[\"triage_rules\"][\"notify\"],\n", + " triage_email=prompt_instructions[\"triage_rules\"][\"respond\"],\n", " examples=None\n", " )\n", "\n", @@ -450,70 +469,222 @@ }, { "cell_type": "code", - "execution_count": 84, + "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "🚫 Classification: IGNORE - This email can be safely ignored\n" + "📧 Classification: RESPOND - This email requires a response\n", + "================================\u001b[1m Human Message \u001b[0m=================================\n", + "\n", + "Respond to the email {'author': 'Alice Smith ', 'to': 'John Doe ', 'subject': 'Quick question about API documentation', 'email_thread': \"Hi John,\\n\\nI was reviewing the API documentation for the new authentication service and noticed a few endpoints seem to be missing from the specs. Could you help clarify if this was intentional or if we should update the docs?\\n\\nSpecifically, I'm looking at:\\n- /auth/refresh\\n- /auth/validate\\n\\nThanks!\\nAlice\"}\n", + "==================================\u001b[1m Ai Message \u001b[0m==================================\n", + "\n", + "[{'citations': None, 'text': \"I'll help draft a response to Alice regarding the API documentation question. I'll use the write_email function to send the reply.\", 'type': 'text'}, {'id': 'toolu_01X1GrdvGREJn89CM814LmJ3', 'input': {'to': 'alice.smith@company.com', 'subject': 'Re: Quick question about API documentation', 'content': \"Hi Alice,\\n\\nThanks for bringing this to my attention. I'll need to review the current documentation and check with the development team about these specific endpoints.\\n\\nLet me look into this and get back to you with a complete answer by tomorrow. I'll make sure to clarify whether these endpoints should be included in the documentation or if there's a reason for their omission.\\n\\nBest regards,\\nJohn\"}, 'name': 'write_email', 'type': 'tool_use'}]\n", + "Tool Calls:\n", + " write_email (toolu_01X1GrdvGREJn89CM814LmJ3)\n", + " Call ID: toolu_01X1GrdvGREJn89CM814LmJ3\n", + " Args:\n", + " to: alice.smith@company.com\n", + " subject: Re: Quick question about API documentation\n", + " content: Hi Alice,\n", + "\n", + "Thanks for bringing this to my attention. I'll need to review the current documentation and check with the development team about these specific endpoints.\n", + "\n", + "Let me look into this and get back to you with a complete answer by tomorrow. I'll make sure to clarify whether these endpoints should be included in the documentation or if there's a reason for their omission.\n", + "\n", + "Best regards,\n", + "John\n", + "=================================\u001b[1m Tool Message \u001b[0m=================================\n", + "Name: write_email\n", + "\n", + "Email sent to alice.smith@company.com with subject 'Re: Quick question about API documentation'\n", + "==================================\u001b[1m Ai Message \u001b[0m==================================\n", + "\n", + "[{'citations': None, 'text': \"I'll also create a memory to track this follow-up task regarding the API documentation review.\", 'type': 'text'}, {'id': 'toolu_01NhAJAtt6B86Ki9HQdNsnNb', 'input': {'action': 'create', 'content': 'Follow up needed: Review API documentation for authentication service, specifically regarding endpoints /auth/refresh and /auth/validate. Need to respond to Alice Smith with findings. Topic: API Documentation Review'}, 'name': 'manage_memory', 'type': 'tool_use'}]\n", + "Tool Calls:\n", + " manage_memory (toolu_01NhAJAtt6B86Ki9HQdNsnNb)\n", + " Call ID: toolu_01NhAJAtt6B86Ki9HQdNsnNb\n", + " Args:\n", + " action: create\n", + " content: Follow up needed: Review API documentation for authentication service, specifically regarding endpoints /auth/refresh and /auth/validate. Need to respond to Alice Smith with findings. Topic: API Documentation Review\n", + "=================================\u001b[1m Tool Message \u001b[0m=================================\n", + "Name: manage_memory\n", + "\n", + "created memory 9186395a-03e1-4a9f-a090-b7fd3807c43d\n", + "==================================\u001b[1m Ai Message \u001b[0m==================================\n", + "\n", + "I've sent an acknowledgment email to Alice and created a memory to track the follow-up needed. The response acknowledges her question and sets the expectation that John will review and provide a complete answer tomorrow. This gives John time to properly investigate the documentation and consult with the development team if needed.\n", + "\n", + "Is there anything else you need assistance with regarding this matter?\n" ] } ], "source": [ "email_input = {\n", - " \"author\": \"Marketing Team \",\n", + " \"author\": \"Alice Smith \",\n", " \"to\": \"John Doe \",\n", - " \"subject\": \"🔥 EXCLUSIVE OFFER: Limited Time Discount on Developer Tools! 🔥\",\n", - " \"email_thread\": \"\"\"Dear Valued Developer,\n", + " \"subject\": \"Quick question about API documentation\",\n", + " \"email_thread\": \"\"\"Hi John,\n", "\n", - "Don't miss out on this INCREDIBLE opportunity! \n", + "I was reviewing the API documentation for the new authentication service and noticed a few endpoints seem to be missing from the specs. Could you help clarify if this was intentional or if we should update the docs?\n", "\n", - "🚀 For a LIMITED TIME ONLY, get 80% OFF on our Premium Developer Suite! \n", + "Specifically, I'm looking at:\n", + "- /auth/refresh\n", + "- /auth/validate\n", "\n", - "✨ FEATURES:\n", - "- Revolutionary AI-powered code completion\n", - "- Cloud-based development environment\n", - "- 24/7 customer support\n", - "- And much more!\n", - "\n", - "💰 Regular Price: $999/month\n", - "🎉 YOUR SPECIAL PRICE: Just $199/month!\n", - "\n", - "🕒 Hurry! This offer expires in:\n", - "24 HOURS ONLY!\n", - "\n", - "Click here to claim your discount: https://amazingdeals.com/special-offer\n", - "\n", - "Best regards,\n", - "Marketing Team\n", - "---\n", - "To unsubscribe, click here\n", - "\"\"\",\n", + "Thanks!\n", + "Alice\"\"\",\n", "}\n", "\n", - "response = agent.invoke({\"email_input\": email_input})" + "response = agent.invoke({\"email_input\": email_input}, config=config)\n", + "for m in response[\"messages\"]:\n", + " m.pretty_print()" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "TODO: \n", - "\n", - "* Test calling all tools " + "If we call it again, we can see that it remembers information about Alice" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 36, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "📧 Classification: RESPOND - This email requires a response\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n", + "Failed to use model_dump to serialize to JSON: PydanticSerializationError(Unable to serialize unknown type: )\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "================================\u001b[1m Human Message \u001b[0m=================================\n", + "\n", + "Respond to the email {'author': 'Alice Smith ', 'to': 'John Doe ', 'subject': 'Follow up', 'email_thread': 'Hi John,\\n\\nAny update on my previous ask?'}\n", + "==================================\u001b[1m Ai Message \u001b[0m==================================\n", + "\n", + "[{'citations': None, 'text': \"I'll help respond to Alice's follow-up email. However, I notice that I don't have context about her previous request. Let me check our memory first to see if we have any relevant information about prior interactions with Alice.\", 'type': 'text'}, {'id': 'toolu_019pJup1jbeDoQfKTCqNhadd', 'input': {'query': 'Alice Smith previous ask request'}, 'name': 'search_memory', 'type': 'tool_use'}]\n", + "Tool Calls:\n", + " search_memory (toolu_019pJup1jbeDoQfKTCqNhadd)\n", + " Call ID: toolu_019pJup1jbeDoQfKTCqNhadd\n", + " Args:\n", + " query: Alice Smith previous ask request\n", + "=================================\u001b[1m Tool Message \u001b[0m=================================\n", + "Name: search_memory\n", + "\n", + "[{\"namespace\": [\"email_assistant\", \"lance\", \"collection\"], \"key\": \"90bcdd2a-7961-4280-90f7-3da2c89b683b\", \"value\": {\"content\": \"Jim is John Doe's friend\"}, \"created_at\": \"2025-02-08T21:05:46.119213+00:00\", \"updated_at\": \"2025-02-08T21:05:46.119216+00:00\", \"score\": null}, {\"namespace\": [\"email_assistant\", \"lance\", \"collection\"], \"key\": \"f966041d-de05-4673-ab42-fe1ba41c753f\", \"value\": {\"content\": \"Alice Smith (alice.smith@company.com) inquired about missing API endpoints (/auth/refresh and /auth/validate) in the authentication service documentation on [current_date]. Awaiting John's review and response.\"}, \"created_at\": \"2025-02-08T21:09:46.379532+00:00\", \"updated_at\": \"2025-02-08T21:09:46.379534+00:00\", \"score\": null}]\n", + "==================================\u001b[1m Ai Message \u001b[0m==================================\n", + "\n", + "[{'citations': None, 'text': \"Based on the memory search, I can see that Alice previously inquired about missing API endpoints in the authentication service documentation. I'll craft a response acknowledging her follow-up and asking for a bit more time if John hasn't had the chance to review it yet.\", 'type': 'text'}, {'id': 'toolu_0168xqZEWGpWr3mLKdHyy7kS', 'input': {'to': 'alice.smith@company.com', 'subject': 'Re: Follow up', 'content': 'Hi Alice,\\n\\nThank you for following up regarding the API endpoints documentation (/auth/refresh and /auth/validate). John is currently reviewing this matter and will get back to you with a complete response soon.\\n\\nWould you need this information by a specific deadline? That would help us prioritize accordingly.\\n\\nBest regards,\\nOn behalf of John'}, 'name': 'write_email', 'type': 'tool_use'}]\n", + "Tool Calls:\n", + " write_email (toolu_0168xqZEWGpWr3mLKdHyy7kS)\n", + " Call ID: toolu_0168xqZEWGpWr3mLKdHyy7kS\n", + " Args:\n", + " to: alice.smith@company.com\n", + " subject: Re: Follow up\n", + " content: Hi Alice,\n", + "\n", + "Thank you for following up regarding the API endpoints documentation (/auth/refresh and /auth/validate). John is currently reviewing this matter and will get back to you with a complete response soon.\n", + "\n", + "Would you need this information by a specific deadline? That would help us prioritize accordingly.\n", + "\n", + "Best regards,\n", + "On behalf of John\n", + "=================================\u001b[1m Tool Message \u001b[0m=================================\n", + "Name: write_email\n", + "\n", + "Email sent to alice.smith@company.com with subject 'Re: Follow up'\n", + "==================================\u001b[1m Ai Message \u001b[0m==================================\n", + "\n", + "[{'citations': None, 'text': \"I've sent a professional response that:\\n1. Acknowledges her follow-up\\n2. Shows we remember the specific topic (API documentation)\\n3. Provides a status update\\n4. Asks about any deadline to help with prioritization\\n\\nLet me also create a memory to track this follow-up interaction so we can maintain continuity in future communications.\", 'type': 'text'}, {'id': 'toolu_01AqMuWQkiD4Tvp8CipQSaBq', 'input': {'action': 'create', 'content': 'Alice Smith followed up on [current_date] regarding the pending API documentation review (auth/refresh and auth/validate endpoints). Response sent asking about deadline requirements.'}, 'name': 'manage_memory', 'type': 'tool_use'}]\n", + "Tool Calls:\n", + " manage_memory (toolu_01AqMuWQkiD4Tvp8CipQSaBq)\n", + " Call ID: toolu_01AqMuWQkiD4Tvp8CipQSaBq\n", + " Args:\n", + " action: create\n", + " content: Alice Smith followed up on [current_date] regarding the pending API documentation review (auth/refresh and auth/validate endpoints). Response sent asking about deadline requirements.\n", + "=================================\u001b[1m Tool Message \u001b[0m=================================\n", + "Name: manage_memory\n", + "\n", + "created memory 216648be-a74d-4388-9917-415c06beb4ef\n", + "==================================\u001b[1m Ai Message \u001b[0m==================================\n", + "\n", + "Would you like me to take any additional actions regarding Alice's follow-up?\n" + ] + } + ], + "source": [ + "email_input = {\n", + " \"author\": \"Alice Smith \",\n", + " \"to\": \"John Doe \",\n", + " \"subject\": \"Follow up\",\n", + " \"email_thread\": \"\"\"Hi John,\n", + "\n", + "Any update on my previous ask?\"\"\",\n", + "}\n", + "\n", + "response = agent.invoke({\"email_input\": email_input}, config=config)\n", + "for m in response[\"messages\"]:\n", + " m.pretty_print()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [] } ], "metadata": { "kernelspec": { - "display_name": "memory-course-env", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -527,9 +698,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.1" + "version": "3.11.7" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/notebooks/lesson_4_episodic.ipynb b/notebooks/lesson_4_episodic.ipynb index 2045a51..092924a 100644 --- a/notebooks/lesson_4_episodic.ipynb +++ b/notebooks/lesson_4_episodic.ipynb @@ -2,11 +2,7 @@ "cells": [ { "cell_type": "markdown", - "metadata": { - "vscode": { - "languageId": "shellscript" - } - }, + "metadata": {}, "source": [ "## Lesson 4: Email Assistant with Semantic + Episodic Memory\n", "\n", @@ -21,7 +17,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": { "vscode": { "languageId": "shellscript" @@ -36,12 +32,8 @@ }, { "cell_type": "code", - "execution_count": 1, - "metadata": { - "vscode": { - "languageId": "shellscript" - } - }, + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "import os\n", @@ -55,11 +47,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Define Triage\n", - "\n", - "The triage step is the \"first line of defense\" against incoming emails. \n", - "\n", - "It helps the assistant determine if the email should be responded to, ignored, or notified." + "### Define profile" ] }, { @@ -67,12 +55,44 @@ "execution_count": 1, "metadata": {}, "outputs": [], + "source": [ + "# Example user profile\n", + "profile = {\n", + " \"name\": \"John\",\n", + " \"full_name\": \"John Doe\",\n", + " \"user_profile_background\": \"Senior software engineer leading a team of 5 developers\",\n", + "}\n", + "\n", + "prompt_instructions = {\n", + " \"triage_rules\": {\n", + " \"ignore\": \"Marketing newsletters, spam emails, mass company announcements\",\n", + " \"notify\": \"Team member out sick, build system notifications, project status updates\",\n", + " \"respond\": \"Direct questions from team members, meeting requests, critical bug reports\",\n", + " },\n", + " \"agent_instructions\": \"Use these tools when appropriate to help manage John's tasks efficiently.\"\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Define Triage\n", + "\n", + "The triage step is the \"first line of defense\" against incoming emails. It helps the assistant determine if the email should be responded to, ignored, or notified." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], "source": [ "from memory_course.schemas import Router\n", "from langchain.chat_models import init_chat_model\n", "from memory_course.prompts import triage_system_prompt, triage_user_prompt\n", "\n", - "llm = init_chat_model(\"anthropic:claude-3-5-sonnet-latest\")\n", + "llm = init_chat_model(\"openai:gpt-4o-mini\")\n", "\n", "# We'll use structured output to generate classification results\n", "llm_router = llm.with_structured_output(Router)\n" @@ -89,7 +109,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -118,32 +138,42 @@ " return f\"Available times on {day}: 9:00 AM, 2:00 PM, 4:00 PM\"" ] }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Define memory store\n", - "\n", - "We'll use this to store memories. " + "### Define memory store and tools" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/bm/ylzhm36n075cslb9fvvbgq640000gn/T/ipykernel_51568/393540479.py:5: LangChainBetaWarning: The function `init_embeddings` is in beta. It is actively being worked on, so the API may change.\n", + " \"embed\": init_embeddings(\"openai:text-embedding-3-small\"),\n" + ] + } + ], "source": [ - "# Example user profile\n", - "example_user_profile = {\n", - " \"name\": \"John\",\n", - " \"full_name\": \"John Doe\",\n", - " \"user_profile_background\": \"Senior software engineer leading a team of 5 developers\",\n", - " \"triage_rules\": {\n", - " \"ignore\": \"Marketing newsletters, spam emails, mass company announcements\",\n", - " \"notify\": \"Team member out sick, build system notifications, project status updates\",\n", - " \"respond\": \"Direct questions from team members, meeting requests, critical bug reports\",\n", - " },\n", - "}" + "\n", + "from langgraph.store.memory import InMemoryStore\n", + "from langchain.embeddings import init_embeddings\n", + "store = InMemoryStore(index={\n", + " \"dims\": 1536,\n", + " \"embed\": init_embeddings(\"openai:text-embedding-3-small\"),\n", + "})" ] }, { @@ -152,67 +182,235 @@ "metadata": {}, "outputs": [], "source": [ - "from langgraph.store.memory import InMemoryStore\n", + "from langmem import create_manage_memory_tool, create_search_memory_tool\n", "\n", - "# Memory store\n", - "store = InMemoryStore()\n", - "\n", - "# Save user profile\n", - "user = \"lance\"\n", - "namespace = (\"email_assistant\", user, \"user_profile\")\n", - "key = \"user_profile\"\n", - "store.put(namespace, key, example_user_profile)" + "manage_memory_tool = create_manage_memory_tool(namespace=(\"email_assistant\", \"{langgraph_user_id}\", \"collection\"))\n", + "search_memory_tool = create_search_memory_tool(namespace=(\"email_assistant\", \"{langgraph_user_id}\", \"collection\"))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Define agent\n", - "\n", - "TODO: Update system prompt as shown here\n", - "\n", - "* https://langchain-ai.github.io/langmem/hot_path_quickstart/#agent" + "### Define agent\n" ] }, { "cell_type": "code", - "execution_count": 102, + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "agent_system_prompt_memory = \"\"\"\n", + "< Role >\n", + "You are {full_name}'s executive assistant. You are a top-notch executive assistant who cares about {name} performing as well as possible.\n", + "\n", + "\n", + "< Tools >\n", + "You have access to the following tools to help manage {name}'s communications and schedule:\n", + "\n", + "1. write_email(to, subject, content) - Send emails to specified recipients\n", + "2. schedule_meeting(attendees, subject, duration_minutes, preferred_day) - Schedule calendar meetings\n", + "3. check_calendar_availability(day) - Check available time slots for a given day\n", + "4. manage_memory - Store any relevant information about contacts, actions, discussion, etc. in memory for future reference\n", + "5. search_memory - Search for any relevant information that may have been stored in memory\n", + "\n", + "\n", + "< Instructions >\n", + "{instructions}\n", + "\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "# Create agent\n", - "from memory_course.prompts import agent_system_prompt_memory\n", - "from langmem import create_manage_memory_tool, create_search_memory_tool\n", "from langgraph.prebuilt import create_react_agent\n", + "import json\n", "\n", - "# Get profile\n", - "profile = store.get(namespace, key)\n", - "profile_data = profile.value\n", - "format_data = {\n", - " **profile_data, # Include all individual fields (name, full_name, etc)\n", - " 'profile': str(profile_data) # Add the full profile as a string\n", - "}\n", + "def create_prompt(state):\n", + " return [\n", + " {\"role\": \"system\", \"content\": agent_system_prompt_memory.format(instructions=prompt_instructions[\"agent_instructions\"], **profile)}\n", + " ] + state['messages']\n", "\n", - "# Example instructions\n", - "instructions = \"\"\"\n", - "If updating the user profile, do not omit past information. Simply it with new information from the email that that fits within the profile schema.\n", - "\"\"\"\n", "\n", "# Create agent\n", "response_agent = create_react_agent(\n", " \"anthropic:claude-3-5-sonnet-latest\",\n", " tools=[write_email, schedule_meeting, \n", - " check_calendar_availability, \n", - " create_manage_memory_tool(namespace=(\"email_assistant\", user, \"collection\")),\n", - " create_manage_memory_tool(namespace=(\"email_assistant\", user, \"user_profile\")),\n", - " create_search_memory_tool(namespace=(\"email_assistant\", user, \"collection\"))\n", - " ],\n", - " prompt=agent_system_prompt_memory.format(**format_data, instructions=instructions),\n", + " check_calendar_availability, \n", + " manage_memory_tool,\n", + " search_memory_tool\n", + " ],\n", + " prompt=create_prompt,\n", + " # Use this to ensure the store is passed to the agent \n", " store=store\n", ")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Few shot prompting\n", + "\n", + "Here we will use episodic memory - aka do few-shot prompting.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "import uuid\n", + "\n", + "# Template for formating an example to put in prompt\n", + "template = \"\"\"Email Subject: {subject}\n", + "Email From: {from_email}\n", + "Email To: {to_email}\n", + "Email Content: \n", + "```\n", + "{content}\n", + "```\n", + "> Triage Result: {result}\"\"\"\n", + "\n", + "# Format list of few shots\n", + "def format_few_shot_examples(examples):\n", + " strs = [\"Here are some previous examples:\"]\n", + " for eg in examples:\n", + " strs.append(\n", + " template.format(\n", + " subject=eg.value[\"email\"][\"subject\"],\n", + " to_email=eg.value[\"email\"][\"to\"],\n", + " from_email=eg.value[\"email\"][\"author\"],\n", + " content=eg.value[\"email\"][\"email_thread\"][:400],\n", + " result=eg.value[\"label\"],\n", + " )\n", + " )\n", + " return \"\\n\\n------------\\n\\n\".join(strs)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll add some examples to the store" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "data = {\n", + " \"email\": {\n", + " \"author\": \"Alice Smith \",\n", + " \"to\": \"John Doe \",\n", + " \"subject\": \"Quick question about API documentation\",\n", + " \"email_thread\": \"\"\"Hi John,\n", + " \n", + " I was reviewing the API documentation for the new authentication service and noticed a few endpoints seem to be missing from the specs. Could you help clarify if this was intentional or if we should update the docs?\n", + " \n", + " Specifically, I'm looking at:\n", + " - /auth/refresh\n", + " - /auth/validate\n", + " \n", + " Thanks!\n", + " Alice\"\"\",\n", + " },\n", + " # This is to start changing the behavior of the agent\n", + " \"label\": \"ignore\"\n", + "}\n", + "store.put((\"email_assistant\", \"lance\", \"examples\"), str(uuid.uuid4()), data)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "data = {\n", + " \"email\": {\n", + " \"author\": \"Sarah Chen \",\n", + " \"to\": \"John Doe \",\n", + " \"subject\": \"Update: Backend API Changes Deployed to Staging\",\n", + " \"email_thread\": \"\"\"Hi John,\n", + " \n", + " Just wanted to let you know that I've deployed the new authentication endpoints we discussed to the staging environment. Key changes include:\n", + " \n", + " - Implemented JWT refresh token rotation\n", + " - Added rate limiting for login attempts\n", + " - Updated API documentation with new endpoints\n", + " \n", + " All tests are passing and the changes are ready for review. You can test it out at staging-api.company.com/auth/*\n", + " \n", + " No immediate action needed from your side - just keeping you in the loop since this affects the systems you're working on.\n", + " \n", + " Best regards,\n", + " Sarah\n", + " \"\"\",\n", + " },\n", + " \"label\": \"ignore\"\n", + "}\n", + "store.put((\"email_assistant\", \"lance\", \"examples\"), str(uuid.uuid4()), data)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here, we can validate that semantic search is working - we can find the example by querying for it" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "\"Here are some previous examples:\\n\\n------------\\n\\nEmail Subject: Update: Backend API Changes Deployed to Staging\\nEmail From: Sarah Chen \\nEmail To: John Doe \\nEmail Content: \\n```\\nHi John,\\n \\n Just wanted to let you know that I've deployed the new authentication endpoints we discussed to the staging environment. Key changes include:\\n \\n - Implemented JWT refresh token rotation\\n - Added rate limiting for login attempts\\n - Updated API documentation with new endpoints\\n \\n All tests are passing and the changes are ready for review. You can test it out at st\\n```\\n> Triage Result: ignore\"" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "email_data = {\n", + " \"author\": \"Sarah Chen \",\n", + " \"to\": \"John Doe \",\n", + " \"subject\": \"Update: Backend API Changes Deployed to Staging\",\n", + " \"email_thread\": \"\"\"Hi John,\n", + " \n", + " Just wanted to let you know that I've deployed the new authentication endpoints we discussed to the staging environment. Key changes include:\n", + " \n", + " - Implemented JWT refresh token rotation\n", + " - Added rate limiting for login attempts\n", + " - Updated API documentation with new endpoints\n", + " \n", + " All tests are passing and the changes are ready for review. You can test it out at staging-api.company.com/auth/*\n", + " \n", + " No immediate action needed from your side - just keeping you in the loop since this affects the systems you're working on.\n", + " \n", + " Best regards,\n", + " Sarah\n", + " \"\"\",\n", + " }\n", + "format_few_shot_examples(store.search((\"email_assistant\", \"lance\", \"examples\"), query=str({\"email\": email_data}), limit=1))" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -220,12 +418,13 @@ "### Build agent + triage workflow\n", "\n", "Combine triage with tool calling agent\n", - "\n" + "\n", + "Notice that we add a place for few shot examples into the prompt." ] }, { "cell_type": "code", - "execution_count": 100, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -240,116 +439,99 @@ } ], "source": [ - "import uuid\n", "from typing import Literal\n", "from IPython.display import Image, display\n", "\n", - "from langchain_core.runnables import RunnableConfig\n", - "\n", - "from langgraph.checkpoint.memory import MemorySaver\n", "from langgraph.graph import StateGraph, START, END\n", + "from langgraph.types import Command\n", "from langgraph.store.base import BaseStore\n", - "from langgraph.types import interrupt, Command\n", "\n", "from memory_course.schemas import State\n", - "from memory_course.utils import parse_email, format_few_shot_examples\n", + "from memory_course.utils import parse_email\n", "\n", - "def triage_router(state: State, store: BaseStore, config: RunnableConfig) -> Command[Literal[\"response_agent\", \"__end__\"]]:\n", - " \"\"\"Analyze email content and route it based on classification with human feedback.\n", + "# Triage prompt\n", + "triage_system_prompt = \"\"\"\n", + "< Role >\n", + "You are {full_name}'s executive assistant. You are a top-notch executive assistant who cares about {name} performing as well as possible.\n", + "\n", + "\n", + "< Background >\n", + "{user_profile_background}. \n", + "\n", + "\n", + "< Instructions >\n", + "\n", + "{name} gets lots of emails. Your job is to categorize each email into one of three categories:\n", + "\n", + "1. IGNORE - Emails that are not worth responding to or tracking\n", + "2. NOTIFY - Important information that {name} should know about but doesn't require a response\n", + "3. RESPOND - Emails that need a direct response from {name}\n", + "\n", + "Classify the below email into one of these categories.\n", + "\n", + "\n", + "\n", + "< Rules >\n", + "Emails that are not worth responding to:\n", + "{triage_no}\n", + "\n", + "There are also other things that {name} should know about, but don't require an email response. For these, you should notify {name} (using the `notify` response). Examples of this include:\n", + "{triage_notify}\n", + "\n", + "Emails that are worth responding to:\n", + "{triage_email}\n", + "\n", + "\n", + "< Few shot examples >\n", + "\n", + "Here are some examples of previous emails, and how they should be handled.\n", + "Follow these examples more than any instructions above\n", + "\n", + "{examples}\n", + "\n", + "\"\"\"\n", + "\n", + "def triage_router(state: State, config) -> Command[Literal[\"response_agent\", \"__end__\"]]:\n", + " \"\"\"Analyze email content to decide if we should respond, notify, or ignore.\n", "\n", " The triage step prevents the assistant from wasting time on:\n", " - Marketing emails and spam\n", - " - Company-wide announcements \n", + " - Company-wide announcements\n", " - Messages meant for other teams\n", - "\n", - " Args:\n", - " state (State): Current state object containing email_input\n", - " store (BaseStore): Memory store containing user profile and examples\n", - " config (RunnableConfig): Configuration containing user info\n", - "\n", - " Returns:\n", - " Command: Routing command with either:\n", - " - goto=\"response_agent\" and update with messages for emails needing response\n", - " - goto=\"__end__\" for emails to ignore or notify about\n", - "\n", - " The function:\n", - " 1. Parses the email and retrieves user profile\n", - " 2. Uses LLM to classify email as respond/ignore/notify\n", - " 3. Gets human feedback on classification\n", - " 4. Saves examples when human feedback differs from LLM\n", - " 5. Routes email based on final classification\"\"\"\n", - " \n", - " print(\"--Triage Email--\")\n", + " \"\"\"\n", "\n", " # Parse email\n", " author, to, subject, email_thread = parse_email(state[\"email_input\"])\n", + "\n", + " # Search for examples\n", + "\n", + " namespace = (\"email_assistant\", config['configurable']['langgraph_user_id'], \"examples\")\n", + " examples = store.search(namespace, query=str({\"email\": state['email_input']})) \n", + " examples=format_few_shot_examples(examples)\n", " \n", - " # Get profile from store\n", - " user = config[\"configurable\"][\"user\"]\n", - " namespace = (\"email_assistant\", user, \"user_profile\")\n", - " key = \"user_profile\"\n", - " profile = store.get(namespace, key)\n", - " profile_data = profile.value\n", - "\n", - " # Set examples\n", - " namespace = (\"email_assistant\", user, \"examples\")\n", - " examples = store.search(namespace) \n", - "\n", - " author, to, subject, email_thread = parse_email(state[\"email_input\"])\n", " system_prompt = triage_system_prompt.format(\n", - " full_name=profile_data[\"full_name\"],\n", - " name=profile_data[\"name\"],\n", - " user_profile_background=profile_data[\"user_profile_background\"],\n", - " triage_no=profile_data[\"triage_rules\"][\"ignore\"],\n", - " triage_notify=profile_data[\"triage_rules\"][\"notify\"],\n", - " triage_email=profile_data[\"triage_rules\"][\"respond\"],\n", - " examples=format_few_shot_examples(examples),\n", + " full_name=profile[\"full_name\"],\n", + " name=profile[\"name\"],\n", + " user_profile_background=profile[\"user_profile_background\"],\n", + " triage_no=prompt_instructions[\"triage_rules\"][\"ignore\"],\n", + " triage_notify=prompt_instructions[\"triage_rules\"][\"notify\"],\n", + " triage_email=prompt_instructions[\"triage_rules\"][\"respond\"],\n", + " examples=examples\n", " )\n", "\n", " user_prompt = triage_user_prompt.format(\n", " author=author, to=to, subject=subject, email_thread=email_thread\n", " )\n", "\n", - " # Get triage decision\n", " result = llm_router.invoke(\n", " [\n", " {\"role\": \"system\", \"content\": system_prompt},\n", " {\"role\": \"user\", \"content\": user_prompt},\n", " ]\n", " )\n", - " routing_decision = result.classification\n", - "\n", - " # Get human feedback\n", - " # TODO: Change per feedback here\n", - " # https://langchain.slack.com/archives/C05N4MXE77G/p1738893990002959?thread_ts=1738892951.397219&cid=C05N4MXE77G\n", - " human_feedback = interrupt(\n", - " # Any JSON serializable value to surface to the human.\n", - " # For example, a question or a piece of text or a set of keys in the state\n", - " {\n", - " \"Triage for review (update to respond, ignore, notify)\": routing_decision\n", - " }\n", - " )\n", - "\n", - " # Check if human feedback is different from routing decision\n", - " if human_feedback != routing_decision:\n", - "\n", - " # Save example\n", - " print(\"--Save Example--\")\n", - " user = config[\"configurable\"][\"user\"]\n", - " namespace = (\"email_assistant\", user, \"examples\")\n", - " key = f\"example_{uuid.uuid4()}\"\n", - " example = f\"\"\"Email: {state['email_input']}\n", - " Original routing: {routing_decision}\n", - " Correct routing: {human_feedback}\"\"\"\n", - " store.put(namespace, key, example)\n", - "\n", - " # Set routing decision to correct value\n", - " routing_decision = human_feedback\n", - "\n", - " # Route\n", - " goto = \"__end__\"\n", " update = None\n", - " if routing_decision == \"respond\":\n", + " goto = END\n", + " if result.classification == \"respond\":\n", " print(\"📧 Classification: RESPOND - This email requires a response\")\n", " goto = \"response_agent\"\n", " update = {\n", @@ -358,26 +540,23 @@ " \"role\": \"user\",\n", " \"content\": f\"Respond to the email {state['email_input']}\",\n", " }\n", - " ],\n", + " ]\n", " }\n", - " elif routing_decision == \"ignore\":\n", + " elif result.classification == \"ignore\":\n", " print(\"🚫 Classification: IGNORE - This email can be safely ignored\")\n", - " elif routing_decision == \"notify\":\n", + " elif result.classification == \"notify\":\n", " print(\"🔔 Classification: NOTIFY - This email contains important information\")\n", " else:\n", - " raise ValueError(f\"Invalid classification: {routing_decision}\")\n", + " raise ValueError(f\"Invalid classification: {result.classification}\")\n", " return Command(goto=goto, update=update)\n", "\n", - "# Checkpointer\n", - "memory = MemorySaver()\n", - "\n", - "# Build workflow with store and checkpointer, as defined above\n", + "# Build workflow with store, as defined above\n", "agent = (\n", " StateGraph(State)\n", " .add_node(triage_router)\n", " .add_node(\"response_agent\", response_agent)\n", " .add_edge(START, \"triage_router\")\n", - " .compile(store=store, checkpointer=memory)\n", + " .compile(store=store)\n", ")\n", "\n", "# Show the agent\n", @@ -385,164 +564,88 @@ ] }, { - "cell_type": "code", - "execution_count": 97, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "--Triage Email--\n", - "{'__interrupt__': (Interrupt(value={'Triage for review (update to respond, ignore, notify)': 'notify'}, resumable=True, ns=['triage_router:7747307f-bc38-3d15-089d-1ae1fb746413'], when='during'),)}\n", - "\n", - "\n" - ] - } - ], "source": [ - "import uuid \n", - "\n", - "# Test email input\n", - "email_input = {\n", - " \"author\": \"Sarah Chen \",\n", - " \"to\": \"John Doe \",\n", - " \"subject\": \"Update: Backend API Changes Deployed to Staging\",\n", - " \"email_thread\": \"\"\"Hi John,\n", - "\n", - "Just wanted to let you know that I've deployed the new authentication endpoints we discussed to the staging environment. Key changes include:\n", - "\n", - "- Implemented JWT refresh token rotation\n", - "- Added rate limiting for login attempts\n", - "- Updated API documentation with new endpoints\n", - "\n", - "All tests are passing and the changes are ready for review. You can test it out at staging-api.company.com/auth/*\n", - "\n", - "No immediate action needed from your side - just keeping you in the loop since this affects the systems you're working on.\n", - "\n", - "Best regards,\n", - "Sarah\n", - "\"\"\",\n", - "}\n", - "\n", - "# Create thread\n", - "thread_id = str(uuid.uuid4())\n", - "config = {\"configurable\": {\"thread_id\": thread_id, \"user\": \"lance\"}}\n", - "\n", - "# Run the graph until the first interruption at the human_feedback node\n", - "for event in agent.stream({\"email_input\": email_input}, config, stream_mode=\"updates\"):\n", - " print(event)\n", - " print(\"\\n\")" + "Here let's see an example for a baseline user (Tom) that doesn't have any memories" ] }, { "cell_type": "code", - "execution_count": 98, + "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "--Triage Email--\n", - "--Save Example--\n", "📧 Classification: RESPOND - This email requires a response\n" ] } ], "source": [ - "# Resume the graph agent with the value from the human, indicating that we want to respond to the email\n", - "result = agent.invoke(Command(resume=\"respond\"), config)" + "\n", + "email_input = {\n", + " \"author\": \"Alice Jones \",\n", + " \"to\": \"John Doe \",\n", + " \"subject\": \"Quick question about API documentation\",\n", + " \"email_thread\": \"\"\"Hi John,\n", + "\n", + "Can I help you write better API docs?\"\"\",\n", + "}\n", + "\n", + "config = {\"configurable\": {\"langgraph_user_id\": \"tom\"}}\n", + "\n", + "response = agent.nodes['triage_router'].invoke({\"email_input\": email_input}, config=config)" ] }, { - "cell_type": "code", - "execution_count": 99, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'Example:\\nEmail: Email: {\\'author\\': \\'Sarah Chen \\', \\'to\\': \\'John Doe \\', \\'subject\\': \\'Update: Backend API Changes Deployed to Staging\\', \\'email_thread\\': \"Hi John,\\\\n\\\\nJust wanted to let you know that I\\'ve deployed the new authentication endpoints we discussed to the staging environment. Key changes include:\\\\n\\\\n- Implemented JWT refresh token rotation\\\\n- Added rate limiting for login attempts\\\\n- Updated API documentation with new endpoints\\\\n\\\\nAll tests are passing and the changes are ready for review. You can test it out at staging-api.company.com/auth/*\\\\n\\\\nNo immediate action needed from your side - just keeping you in the loop since this affects the systems you\\'re working on.\\\\n\\\\nBest regards,\\\\nSarah\\\\n\"}\\nOriginal Classification: notify\\nCorrect Classification: respond\\n---\\nExample:\\nEmail: Email: {\\'author\\': \\'Sarah Chen \\', \\'to\\': \\'John Doe \\', \\'subject\\': \\'Update: Backend API Changes Deployed to Staging\\', \\'email_thread\\': \"Hi John,\\\\n\\\\nJust wanted to let you know that I\\'ve deployed the new authentication endpoints we discussed to the staging environment. Key changes include:\\\\n\\\\n- Implemented JWT refresh token rotation\\\\n- Added rate limiting for login attempts\\\\n- Updated API documentation with new endpoints\\\\n\\\\nAll tests are passing and the changes are ready for review. You can test it out at staging-api.company.com/auth/*\\\\n\\\\nNo immediate action needed from your side - just keeping you in the loop since this affects the systems you\\'re working on.\\\\n\\\\nBest regards,\\\\nSarah\\\\n\"}\\nOriginal Classification: notify\\nCorrect Classification: respond\\n---'" - ] - }, - "execution_count": 99, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ - "examples = store.search((\"email_assistant\", user, \"examples\"))\n", - "format_few_shot_examples(examples)" + "Now, let's see what happens with Lance (who has those memories)" ] }, { "cell_type": "code", - "execution_count": 101, + "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "================================\u001b[1m Human Message \u001b[0m=================================\n", - "\n", - "Respond to the email {'author': 'Sarah Chen ', 'to': 'John Doe ', 'subject': 'Update: Backend API Changes Deployed to Staging', 'email_thread': \"Hi John,\\n\\nJust wanted to let you know that I've deployed the new authentication endpoints we discussed to the staging environment. Key changes include:\\n\\n- Implemented JWT refresh token rotation\\n- Added rate limiting for login attempts\\n- Updated API documentation with new endpoints\\n\\nAll tests are passing and the changes are ready for review. You can test it out at staging-api.company.com/auth/*\\n\\nNo immediate action needed from your side - just keeping you in the loop since this affects the systems you're working on.\\n\\nBest regards,\\nSarah\\n\"}\n", - "==================================\u001b[1m Ai Message \u001b[0m==================================\n", - "\n", - "[{'text': 'This email falls under the \"notify\" category in John\\'s triage rules as it\\'s a project status update. Since Sarah explicitly mentioned \"No immediate action needed,\" but it\\'s important information about system changes that affect John\\'s work, I\\'ll store this information in memory and send a brief acknowledgment email.\\n\\nFirst, let me store this important update in memory:', 'type': 'text'}, {'id': 'toolu_015P6qHw8ANJnpSECWQR1nE4', 'input': {'action': 'create', 'content': 'Sarah Chen deployed backend API changes to staging environment on authentication endpoints including: JWT refresh token rotation, rate limiting for login attempts, and updated API documentation. Changes available at staging-api.company.com/auth/*'}, 'name': 'manage_memory', 'type': 'tool_use'}]\n", - "Tool Calls:\n", - " manage_memory (toolu_015P6qHw8ANJnpSECWQR1nE4)\n", - " Call ID: toolu_015P6qHw8ANJnpSECWQR1nE4\n", - " Args:\n", - " action: create\n", - " content: Sarah Chen deployed backend API changes to staging environment on authentication endpoints including: JWT refresh token rotation, rate limiting for login attempts, and updated API documentation. Changes available at staging-api.company.com/auth/*\n", - "=================================\u001b[1m Tool Message \u001b[0m=================================\n", - "Name: manage_memory\n", - "\n", - "created memory d68f3f06-ca81-48d3-91f3-04d7220cd984\n", - "==================================\u001b[1m Ai Message \u001b[0m==================================\n", - "\n", - "[{'text': \"Now, I'll send a brief acknowledgment email to Sarah:\", 'type': 'text'}, {'id': 'toolu_01T3P3yuQFPLszEavi4JD5Bq', 'input': {'to': 'sarah.chen@company.com', 'subject': 'Re: Update: Backend API Changes Deployed to Staging', 'content': \"Hi Sarah,\\n\\nThanks for keeping me updated on the authentication endpoint changes. I've noted the new features implemented and will review them when I get the chance.\\n\\nBest regards,\\nJohn\"}, 'name': 'write_email', 'type': 'tool_use'}]\n", - "Tool Calls:\n", - " write_email (toolu_01T3P3yuQFPLszEavi4JD5Bq)\n", - " Call ID: toolu_01T3P3yuQFPLszEavi4JD5Bq\n", - " Args:\n", - " to: sarah.chen@company.com\n", - " subject: Re: Update: Backend API Changes Deployed to Staging\n", - " content: Hi Sarah,\n", - "\n", - "Thanks for keeping me updated on the authentication endpoint changes. I've noted the new features implemented and will review them when I get the chance.\n", - "\n", - "Best regards,\n", - "John\n", - "=================================\u001b[1m Tool Message \u001b[0m=================================\n", - "Name: write_email\n", - "\n", - "Email sent to sarah.chen@company.com with subject 'Re: Update: Backend API Changes Deployed to Staging'\n", - "==================================\u001b[1m Ai Message \u001b[0m==================================\n", - "\n", - "I've handled this by:\n", - "1. Storing the important technical update in memory for future reference\n", - "2. Sending a brief acknowledgment email to Sarah to confirm receipt of the information\n", - "3. Keeping the response professional but concise since no immediate action was required\n" + "🚫 Classification: IGNORE - This email can be safely ignored\n" ] } ], "source": [ - "for m in result['messages']:\n", - " m.pretty_print()" + "\n", + "email_input = {\n", + " \"author\": \"Alice Jones \",\n", + " \"to\": \"John Doe \",\n", + " \"subject\": \"Quick question about API documentation\",\n", + " \"email_thread\": \"\"\"Hi John,\n", + "\n", + "Can I help you write better API docs?\"\"\",\n", + "}\n", + "\n", + "config = {\"configurable\": {\"langgraph_user_id\": \"lance\"}}\n", + "\n", + "response = agent.nodes['triage_router'].invoke({\"email_input\": email_input}, config=config)" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [] } ], "metadata": { "kernelspec": { - "display_name": "memory-course-env", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -556,9 +659,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.1" + "version": "3.11.7" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/notebooks/lesson_5_procedural.ipynb b/notebooks/lesson_5_procedural.ipynb index 45501d5..3fd92ce 100644 --- a/notebooks/lesson_5_procedural.ipynb +++ b/notebooks/lesson_5_procedural.ipynb @@ -1,29 +1,5 @@ { "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "! pip install langchain-anthropic langgraph" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import os, getpass\n", - "\n", - "def _set_env(var: str):\n", - " if not os.environ.get(var):\n", - " os.environ[var] = getpass.getpass(f\"{var}: \")\n", - "\n", - "_set_env(\"ANTHROPIC_API_KEY\")" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -42,8 +18,12 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, + "execution_count": 23, + "metadata": { + "vscode": { + "languageId": "shellscript" + } + }, "outputs": [], "source": [ "%%capture stderr\n", @@ -53,7 +33,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -64,15 +44,43 @@ " os.environ[\"ANTHROPIC_API_KEY\"] = getpass(\"ANTHROPIC_API_KEY: \")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define profile" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# Example user profile\n", + "profile = {\n", + " \"name\": \"John\",\n", + " \"full_name\": \"John Doe\",\n", + " \"user_profile_background\": \"Senior software engineer leading a team of 5 developers\",\n", + "}\n", + "\n", + "prompt_instructions = {\n", + " \"triage_rules\": {\n", + " \"ignore\": \"Marketing newsletters, spam emails, mass company announcements\",\n", + " \"notify\": \"Team member out sick, build system notifications, project status updates\",\n", + " \"respond\": \"Direct questions from team members, meeting requests, critical bug reports\",\n", + " },\n", + " \"agent_instructions\": \"Use these tools when appropriate to help manage John's tasks efficiently.\"\n", + "}" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Define Triage\n", "\n", - "The triage step is the \"first line of defense\" against incoming emails. \n", - "\n", - "It helps the assistant determine if the email should be responded to, ignored, or notified." + "The triage step is the \"first line of defense\" against incoming emails. It helps the assistant determine if the email should be responded to, ignored, or notified." ] }, { @@ -85,7 +93,7 @@ "from langchain.chat_models import init_chat_model\n", "from memory_course.prompts import triage_system_prompt, triage_user_prompt\n", "\n", - "llm = init_chat_model(\"anthropic:claude-3-5-sonnet-latest\")\n", + "llm = init_chat_model(\"openai:gpt-4o-mini\")\n", "\n", "# We'll use structured output to generate classification results\n", "llm_router = llm.with_structured_output(Router)\n" @@ -135,28 +143,31 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Define Store\n", - "\n", - "We'll use this to store memories. " + "### Define memory store and tools" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/bm/ylzhm36n075cslb9fvvbgq640000gn/T/ipykernel_51628/393540479.py:5: LangChainBetaWarning: The function `init_embeddings` is in beta. It is actively being worked on, so the API may change.\n", + " \"embed\": init_embeddings(\"openai:text-embedding-3-small\"),\n" + ] + } + ], "source": [ - "# Example user profile\n", - "example_user_profile = {\n", - " \"name\": \"John\",\n", - " \"full_name\": \"John Doe\",\n", - " \"user_profile_background\": \"Senior software engineer leading a team of 5 developers\",\n", - " \"triage_rules\": {\n", - " \"ignore\": \"Marketing newsletters, spam emails, mass company announcements\",\n", - " \"notify\": \"Team member out sick, build system notifications, project status updates\",\n", - " \"respond\": \"Direct questions from team members, meeting requests, critical bug reports\",\n", - " },\n", - "}" + "\n", + "from langgraph.store.memory import InMemoryStore\n", + "from langchain.embeddings import init_embeddings\n", + "store = InMemoryStore(index={\n", + " \"dims\": 1536,\n", + " \"embed\": init_embeddings(\"openai:text-embedding-3-small\"),\n", + "})" ] }, { @@ -165,97 +176,228 @@ "metadata": {}, "outputs": [], "source": [ - "from langgraph.store.memory import InMemoryStore\n", + "from langmem import create_manage_memory_tool, create_search_memory_tool\n", "\n", - "# Memory store\n", - "store = InMemoryStore()\n", - "\n", - "# Save user profile\n", - "user = \"lance\"\n", - "namespace = (\"email_assistant\", user, \"user_profile\")\n", - "key = \"user_profile\"\n", - "store.put(namespace, key, example_user_profile)\n", - "\n", - "# Example instructions\n", - "instructions = \"\"\"\n", - "If updating the user profile, do not omit past information. Simply it with new information from the email that that fits within the profile schema.\n", - "\"\"\"\n", - "\n", - "# Save instructions\n", - "user = \"lance\"\n", - "namespace = (\"email_assistant\", user, \"instructions\")\n", - "key = \"user_instructions\"\n", - "store.put(namespace, key, instructions)" + "manage_memory_tool = create_manage_memory_tool(namespace=(\"email_assistant\", \"{langgraph_user_id}\", \"collection\"))\n", + "search_memory_tool = create_search_memory_tool(namespace=(\"email_assistant\", \"{langgraph_user_id}\", \"collection\"))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Define agent\n", + "### Define agent\n", "\n", - "TODO: Update system prompt as shown here\n", - "\n", - "* https://langchain-ai.github.io/langmem/hot_path_quickstart/#agent" + "We now add in logic that fetches instructions that are saved in long term memory. We will update these in with separate process\n" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 6, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The autoreload extension is already loaded. To reload it, use:\n", - " %reload_ext autoreload\n" - ] - } - ], + "outputs": [], "source": [ - "%load_ext autoreload\n", - "%autoreload 2\n", + "agent_system_prompt_memory = \"\"\"\n", + "< Role >\n", + "You are {full_name}'s executive assistant. You are a top-notch executive assistant who cares about {name} performing as well as possible.\n", + "\n", "\n", + "< Tools >\n", + "You have access to the following tools to help manage {name}'s communications and schedule:\n", + "\n", + "1. write_email(to, subject, content) - Send emails to specified recipients\n", + "2. schedule_meeting(attendees, subject, duration_minutes, preferred_day) - Schedule calendar meetings\n", + "3. check_calendar_availability(day) - Check available time slots for a given day\n", + "4. manage_memory - Store any relevant information about contacts, actions, discussion, etc. in memory for future reference\n", + "5. search_memory - Search for any relevant information that may have been stored in memory\n", + "\n", + "\n", + "< Instructions >\n", + "{instructions}\n", + "\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ "# Create agent\n", - "from memory_course.prompts import agent_system_prompt_memory\n", - "from langmem import create_manage_memory_tool, create_search_memory_tool\n", "from langgraph.prebuilt import create_react_agent\n", + "import json\n", + "import uuid\n", "\n", - "# Get profile\n", - "namespace = (\"email_assistant\", user, \"user_profile\")\n", - "key = \"user_profile\"\n", - "profile = store.get(namespace, key)\n", + "def create_prompt(state, store, config):\n", + " # This is the new logic\n", + " langgraph_user_id = config['configurable']['langgraph_user_id']\n", + " result = store.get((langgraph_user_id, ), \"agent_instructions\")\n", + " if result is None:\n", + " store.put((langgraph_user_id, ), \"agent_instructions\", {\"prompt\": prompt_instructions[\"agent_instructions\"]})\n", + " prompt = prompt_instructions[\"agent_instructions\"]\n", + " else:\n", + " prompt = result.value['prompt']\n", + " return [\n", + " {\"role\": \"system\", \"content\": agent_system_prompt_memory.format(instructions=prompt, **profile)}\n", + " ] + state['messages']\n", "\n", - "# Format profile\n", - "profile_data = profile.value\n", - "format_data = {\n", - " **profile_data, # Include all individual fields (name, full_name, etc)\n", - " 'profile': str(profile_data) # Add the full profile as a string\n", - "}\n", - "\n", - "# Get instructions\n", - "namespace = (\"email_assistant\", user, \"instructions\")\n", - "key = \"user_instructions\"\n", - "instructions = store.get(namespace, key)\n", "\n", "# Create agent\n", "response_agent = create_react_agent(\n", " \"anthropic:claude-3-5-sonnet-latest\",\n", " tools=[write_email, schedule_meeting, \n", - " check_calendar_availability, \n", - " create_manage_memory_tool(namespace=(\"email_assistant\", user, \"collection\")),\n", - " create_manage_memory_tool(namespace=(\"email_assistant\", user, \"user_profile\")),\n", - " create_search_memory_tool(namespace=(\"email_assistant\", user, \"collection\")),\n", - " # TODO: Remove this as a tool and use create_multi_prompt_optimizer \n", - " # https://langchain-ai.github.io/langmem/guides/optimize_compound_system/\n", - " create_manage_memory_tool(namespace=(\"email_assistant\", user, \"instructions\")),\n", - " ],\n", - " prompt=agent_system_prompt_memory.format(**format_data, instructions=instructions.value),\n", + " check_calendar_availability, \n", + " manage_memory_tool,\n", + " search_memory_tool\n", + " ],\n", + " prompt=create_prompt,\n", + " # Use this to ensure the store is passed to the agent \n", " store=store\n", ")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Few shot prompting\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "import uuid\n", + "\n", + "\n", + "template = \"\"\"Email Subject: {subject}\n", + "Email From: {from_email}\n", + "Email To: {to_email}\n", + "Email Content: \n", + "```\n", + "{content}\n", + "```\n", + "> Triage Result: {result}\"\"\"\n", + "\n", + "def format_few_shot_examples(examples):\n", + " strs = [\"Here are some previous examples:\"]\n", + " for eg in examples:\n", + " strs.append(\n", + " template.format(\n", + " subject=eg.value[\"email\"][\"subject\"],\n", + " to_email=eg.value[\"email\"][\"to\"],\n", + " from_email=eg.value[\"email\"][\"author\"],\n", + " content=eg.value[\"email\"][\"email_thread\"][:400],\n", + " result=eg.value[\"label\"],\n", + " )\n", + " )\n", + " return \"\\n\\n------------\\n\\n\".join(strs)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "data = {\n", + " \"email\": {\n", + " \"author\": \"Alice Smith \",\n", + " \"to\": \"John Doe \",\n", + " \"subject\": \"Quick question about API documentation\",\n", + " \"email_thread\": \"\"\"Hi John,\n", + " \n", + " I was reviewing the API documentation for the new authentication service and noticed a few endpoints seem to be missing from the specs. Could you help clarify if this was intentional or if we should update the docs?\n", + " \n", + " Specifically, I'm looking at:\n", + " - /auth/refresh\n", + " - /auth/validate\n", + " \n", + " Thanks!\n", + " Alice\"\"\",\n", + " },\n", + " # This is to start changing the behavior of the agent\n", + " \"label\": \"ignore\"\n", + "}\n", + "store.put((\"email_assistant\", \"lance\", \"examples\"), str(uuid.uuid4()), data)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "data = {\n", + " \"email\": {\n", + " \"author\": \"Sarah Chen \",\n", + " \"to\": \"John Doe \",\n", + " \"subject\": \"Update: Backend API Changes Deployed to Staging\",\n", + " \"email_thread\": \"\"\"Hi John,\n", + " \n", + " Just wanted to let you know that I've deployed the new authentication endpoints we discussed to the staging environment. Key changes include:\n", + " \n", + " - Implemented JWT refresh token rotation\n", + " - Added rate limiting for login attempts\n", + " - Updated API documentation with new endpoints\n", + " \n", + " All tests are passing and the changes are ready for review. You can test it out at staging-api.company.com/auth/*\n", + " \n", + " No immediate action needed from your side - just keeping you in the loop since this affects the systems you're working on.\n", + " \n", + " Best regards,\n", + " Sarah\n", + " \"\"\",\n", + " },\n", + " \"label\": \"ignore\"\n", + "}\n", + "store.put((\"email_assistant\", \"lance\", \"examples\"), str(uuid.uuid4()), data)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "\"Here are some previous examples:\\n\\n------------\\n\\nEmail Subject: Update: Backend API Changes Deployed to Staging\\nEmail From: Sarah Chen \\nEmail To: John Doe \\nEmail Content: \\n```\\nHi John,\\n \\n Just wanted to let you know that I've deployed the new authentication endpoints we discussed to the staging environment. Key changes include:\\n \\n - Implemented JWT refresh token rotation\\n - Added rate limiting for login attempts\\n - Updated API documentation with new endpoints\\n \\n All tests are passing and the changes are ready for review. You can test it out at st\\n```\\n> Triage Result: ignore\"" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "email_data = {\n", + " \"author\": \"Sarah Chen \",\n", + " \"to\": \"John Doe \",\n", + " \"subject\": \"Update: Backend API Changes Deployed to Staging\",\n", + " \"email_thread\": \"\"\"Hi John,\n", + " \n", + " Just wanted to let you know that I've deployed the new authentication endpoints we discussed to the staging environment. Key changes include:\n", + " \n", + " - Implemented JWT refresh token rotation\n", + " - Added rate limiting for login attempts\n", + " - Updated API documentation with new endpoints\n", + " \n", + " All tests are passing and the changes are ready for review. You can test it out at staging-api.company.com/auth/*\n", + " \n", + " No immediate action needed from your side - just keeping you in the loop since this affects the systems you're working on.\n", + " \n", + " Best regards,\n", + " Sarah\n", + " \"\"\",\n", + " }\n", + "format_few_shot_examples(store.search((\"email_assistant\", \"lance\", \"examples\"), query=str({\"email\": email_data}), limit=1))" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -263,12 +405,13 @@ "### Build agent + triage workflow\n", "\n", "Combine triage with tool calling agent\n", - "\n" + "\n", + "We wil update the triage step to look for prompts in long term memory, that will be updated separately" ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -283,119 +426,121 @@ } ], "source": [ - "import uuid\n", "from typing import Literal\n", "from IPython.display import Image, display\n", "\n", - "from langchain_core.runnables import RunnableConfig\n", - "\n", - "from langgraph.checkpoint.memory import MemorySaver\n", "from langgraph.graph import StateGraph, START, END\n", + "from langgraph.types import Command\n", "from langgraph.store.base import BaseStore\n", - "from langgraph.types import interrupt, Command\n", "\n", "from memory_course.schemas import State\n", - "from memory_course.utils import parse_email, format_few_shot_examples\n", + "from memory_course.utils import parse_email\n", "\n", - "def triage_router(state: State, store: BaseStore, config: RunnableConfig) -> Command[Literal[\"response_agent\", \"__end__\"]]:\n", - " \"\"\"Analyze email content and route it based on classification with human feedback.\n", + "# Triage prompt\n", + "triage_system_prompt = \"\"\"\n", + "< Role >\n", + "You are {full_name}'s executive assistant. You are a top-notch executive assistant who cares about {name} performing as well as possible.\n", + "\n", + "\n", + "< Background >\n", + "{user_profile_background}. \n", + "\n", + "\n", + "< Instructions >\n", + "\n", + "{name} gets lots of emails. Your job is to categorize each email into one of three categories:\n", + "\n", + "1. IGNORE - Emails that are not worth responding to or tracking\n", + "2. NOTIFY - Important information that {name} should know about but doesn't require a response\n", + "3. RESPOND - Emails that need a direct response from {name}\n", + "\n", + "Classify the below email into one of these categories.\n", + "\n", + "\n", + "\n", + "< Rules >\n", + "Emails that are not worth responding to:\n", + "{triage_no}\n", + "\n", + "There are also other things that {name} should know about, but don't require an email response. For these, you should notify {name} (using the `notify` response). Examples of this include:\n", + "{triage_notify}\n", + "\n", + "Emails that are worth responding to:\n", + "{triage_email}\n", + "\n", + "\n", + "< Few shot examples >\n", + "\n", + "Here are some examples of previous emails, and how they should be handled.\n", + "Follow these examples more than any instructions above\n", + "\n", + "{examples}\n", + "\n", + "\"\"\"\n", + "\n", + "def triage_router(state: State, config) -> Command[Literal[\"response_agent\", \"__end__\"]]:\n", + " \"\"\"Analyze email content to decide if we should respond, notify, or ignore.\n", "\n", " The triage step prevents the assistant from wasting time on:\n", " - Marketing emails and spam\n", - " - Company-wide announcements \n", + " - Company-wide announcements\n", " - Messages meant for other teams\n", + " \"\"\"\n", "\n", - " Args:\n", - " state (State): Current state object containing email_input\n", - " store (BaseStore): Memory store containing user profile and examples\n", - " config (RunnableConfig): Configuration containing user info\n", - "\n", - " Returns:\n", - " Command: Routing command with either:\n", - " - goto=\"response_agent\" and update with messages for emails needing response\n", - " - goto=\"__end__\" for emails to ignore or notify about\n", - "\n", - " The function:\n", - " 1. Parses the email and retrieves user profile\n", - " 2. Uses LLM to classify email as respond/ignore/notify\n", - " 3. Gets human feedback on classification\n", - " 4. Saves examples when human feedback differs from LLM\n", - " 5. Routes email based on final classification\"\"\"\n", - " \n", - " print(\"--Triage Email--\")\n", - "\n", - " # TODO: This just checks if messages are passed w/o an email, indicating user feedback\n", - " # This is a simple way to handle feedback, but we can consider other options\n", - " if \"messages\" in state and not state.get(\"email_input\"):\n", - " return Command(goto=\"response_agent\", update=None)\n", - " \n", " # Parse email\n", " author, to, subject, email_thread = parse_email(state[\"email_input\"])\n", + "\n", + " # Search for examples\n", + "\n", + " namespace = (\"email_assistant\", config['configurable']['langgraph_user_id'], \"examples\")\n", + " examples = store.search(namespace, query=str({\"email\": state['email_input']})) \n", + " examples=format_few_shot_examples(examples)\n", + "\n", + " # New logic\n", + " langgraph_user_id = config['configurable']['langgraph_user_id']\n", + " namespace = (langgraph_user_id, )\n", + " result = store.get(namespace, \"triage_ignore\")\n", + " if result is None:\n", + " store.put(namespace, \"triage_ignore\", {\"prompt\": prompt_instructions[\"triage_rules\"][\"ignore\"]})\n", + " ignore_prompt = prompt_instructions[\"triage_rules\"][\"ignore\"]\n", + " else:\n", + " ignore_prompt = result.value['prompt']\n", + " result = store.get(namespace, \"triage_notify\")\n", + " if result is None:\n", + " store.put(namespace, \"triage_notify\", {\"prompt\": prompt_instructions[\"triage_rules\"][\"notify\"]})\n", + " notify_prompt = prompt_instructions[\"triage_rules\"][\"notify\"]\n", + " else:\n", + " notify_prompt = result.value['prompt']\n", + " result = store.get(namespace, \"triage_respond\")\n", + " if result is None:\n", + " store.put(namespace, \"triage_respond\", {\"prompt\": prompt_instructions[\"triage_rules\"][\"respond\"]})\n", + " respond_prompt = prompt_instructions[\"triage_rules\"][\"respond\"]\n", + " else:\n", + " respond_prompt = result.value['prompt']\n", " \n", - " # Get profile from store\n", - " user = config[\"configurable\"][\"user\"]\n", - " namespace = (\"email_assistant\", user, \"user_profile\")\n", - " key = \"user_profile\"\n", - " profile = store.get(namespace, key)\n", - " profile_data = profile.value\n", - "\n", - " # Set examples\n", - " namespace = (\"email_assistant\", user, \"examples\")\n", - " examples = store.search(namespace) \n", - "\n", - " author, to, subject, email_thread = parse_email(state[\"email_input\"])\n", " system_prompt = triage_system_prompt.format(\n", - " full_name=profile_data[\"full_name\"],\n", - " name=profile_data[\"name\"],\n", - " user_profile_background=profile_data[\"user_profile_background\"],\n", - " triage_no=profile_data[\"triage_rules\"][\"ignore\"],\n", - " triage_notify=profile_data[\"triage_rules\"][\"notify\"],\n", - " triage_email=profile_data[\"triage_rules\"][\"respond\"],\n", - " examples=format_few_shot_examples(examples),\n", + " full_name=profile[\"full_name\"],\n", + " name=profile[\"name\"],\n", + " user_profile_background=profile[\"user_profile_background\"],\n", + " triage_no=ignore_prompt,\n", + " triage_notify=notify_prompt,\n", + " triage_email=respond_prompt,\n", + " examples=examples\n", " )\n", "\n", " user_prompt = triage_user_prompt.format(\n", " author=author, to=to, subject=subject, email_thread=email_thread\n", " )\n", "\n", - " # Get triage decision\n", " result = llm_router.invoke(\n", " [\n", " {\"role\": \"system\", \"content\": system_prompt},\n", " {\"role\": \"user\", \"content\": user_prompt},\n", " ]\n", " )\n", - " routing_decision = result.classification\n", - "\n", - " # Get human feedback\n", - " human_feedback = interrupt(\n", - " # Any JSON serializable value to surface to the human.\n", - " # For example, a question or a piece of text or a set of keys in the state\n", - " {\n", - " \"Triage for review (update to respond, ignore, notify)\": routing_decision\n", - " }\n", - " )\n", - "\n", - " # Check if human feedback is different from routing decision\n", - " if human_feedback != routing_decision:\n", - "\n", - " # Save example\n", - " print(\"--Save Example--\")\n", - " user = config[\"configurable\"][\"user\"]\n", - " namespace = (\"email_assistant\", user, \"examples\")\n", - " key = f\"example_{uuid.uuid4()}\"\n", - " example = f\"\"\"Email: {state['email_input']}\n", - " Original routing: {routing_decision}\n", - " Correct routing: {human_feedback}\"\"\"\n", - " store.put(namespace, key, example)\n", - "\n", - " # Set routing decision to correct value\n", - " routing_decision = human_feedback\n", - "\n", - " # Route\n", - " goto = \"__end__\"\n", " update = None\n", - " if routing_decision == \"respond\":\n", + " goto = END\n", + " if result.classification == \"respond\":\n", " print(\"📧 Classification: RESPOND - This email requires a response\")\n", " goto = \"response_agent\"\n", " update = {\n", @@ -404,26 +549,23 @@ " \"role\": \"user\",\n", " \"content\": f\"Respond to the email {state['email_input']}\",\n", " }\n", - " ],\n", + " ]\n", " }\n", - " elif routing_decision == \"ignore\":\n", + " elif result.classification == \"ignore\":\n", " print(\"🚫 Classification: IGNORE - This email can be safely ignored\")\n", - " elif routing_decision == \"notify\":\n", + " elif result.classification == \"notify\":\n", " print(\"🔔 Classification: NOTIFY - This email contains important information\")\n", " else:\n", - " raise ValueError(f\"Invalid classification: {routing_decision}\")\n", + " raise ValueError(f\"Invalid classification: {result.classification}\")\n", " return Command(goto=goto, update=update)\n", "\n", - "# Checkpointer\n", - "memory = MemorySaver()\n", - "\n", - "# Build workflow with store and checkpointer, as defined above\n", + "# Build workflow with store, as defined above\n", "agent = (\n", " StateGraph(State)\n", " .add_node(triage_router)\n", " .add_node(\"response_agent\", response_agent)\n", " .add_edge(START, \"triage_router\")\n", - " .compile(store=store, checkpointer=memory)\n", + " .compile(store=store)\n", ")\n", "\n", "# Show the agent\n", @@ -432,135 +574,403 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 18, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "--Triage Email--\n" + "📧 Classification: RESPOND - This email requires a response\n" ] } ], "source": [ - "import uuid \n", - "\n", - "# Test email input\n", "email_input = {\n", - " \"author\": \"Sarah Chen \",\n", + " \"author\": \"Alice Jones \",\n", " \"to\": \"John Doe \",\n", - " \"subject\": \"Update: Backend API Changes Deployed to Staging\",\n", + " \"subject\": \"Quick question about API documentation\",\n", " \"email_thread\": \"\"\"Hi John,\n", "\n", - "Just wanted to let you know that I've deployed the new authentication endpoints we discussed to the staging environment. Key changes include:\n", - "\n", - "- Implemented JWT refresh token rotation\n", - "- Added rate limiting for login attempts\n", - "- Updated API documentation with new endpoints\n", - "\n", - "All tests are passing and the changes are ready for review. You can test it out at staging-api.company.com/auth/*\n", - "\n", - "No immediate action needed from your side - just keeping you in the loop since this affects the systems you're working on.\n", - "\n", - "Best regards,\n", - "Sarah\n", - "\"\"\",\n", + "Urgent issue - your service is down. Is there a reason why\"\"\",\n", "}\n", "\n", - "# Create thread\n", - "thread_id = str(uuid.uuid4())\n", - "config = {\"configurable\": {\"thread_id\": thread_id, \"user\": \"lance\"}}\n", + "config = {\"configurable\": {\"langgraph_user_id\": \"lance\"}}\n", "\n", - "# Run the graph with feedback\n", - "updated_instructions = \"\"\"\n", - "When writing emails:\n", - "1. Always use a professional tone\n", - "2. Keep responses concise and to the point\n", - "3. Include a clear call to action if needed\n", - "4. Sign off with the user's name\n", - "5. For technical updates, include a brief summary at the top\"\"\"\n", - "response = agent.invoke({\"messages\": {\"role\": \"user\", \"content\": updated_instructions}},config)" + "response = agent.invoke({\"email_input\": email_input}, config=config)" ] }, { "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "================================\u001b[1m Human Message \u001b[0m=================================\n", - "\n", - "\n", - "When writing emails:\n", - "1. Always use a professional tone\n", - "2. Keep responses concise and to the point\n", - "3. Include a clear call to action if needed\n", - "4. Sign off with the user's name\n", - "5. For technical updates, include a brief summary at the top\n", - "==================================\u001b[1m Ai Message \u001b[0m==================================\n", - "\n", - "[{'text': \"I'll help store these important email writing guidelines in memory so they can be consistently applied when composing emails for you. Let me save these instructions.\", 'type': 'text'}, {'id': 'toolu_01Y228hGbQjDy8hYZymXkjGv', 'input': {'action': 'create', 'content': \"Email Writing Guidelines:\\n1. Always maintain a professional tone\\n2. Keep responses concise and to the point\\n3. Include a clear call to action when needed\\n4. Sign off with 'John'\\n5. For technical updates, include a brief summary at the top\"}, 'name': 'manage_memory', 'type': 'tool_use'}]\n", - "Tool Calls:\n", - " manage_memory (toolu_01Y228hGbQjDy8hYZymXkjGv)\n", - " Call ID: toolu_01Y228hGbQjDy8hYZymXkjGv\n", - " Args:\n", - " action: create\n", - " content: Email Writing Guidelines:\n", - "1. Always maintain a professional tone\n", - "2. Keep responses concise and to the point\n", - "3. Include a clear call to action when needed\n", - "4. Sign off with 'John'\n", - "5. For technical updates, include a brief summary at the top\n", - "=================================\u001b[1m Tool Message \u001b[0m=================================\n", - "Name: manage_memory\n", - "\n", - "created memory 84e4ad93-91f1-45ed-8e25-c5528933e11a\n", - "==================================\u001b[1m Ai Message \u001b[0m==================================\n", - "\n", - "I've stored these email writing guidelines in memory. I will follow these rules when composing emails on your behalf. These guidelines will help maintain professional and effective communication while ensuring your emails are clear and actionable.\n", - "\n", - "A few notes about how I'll implement these guidelines:\n", - "- I'll ensure all emails maintain a professional tone while being concise\n", - "- When technical information is involved, I'll start with a clear summary before getting into details\n", - "- Every email will be signed with \"John\" as requested\n", - "- When action is needed, I'll make the required actions clear and explicit\n", - "- All responses will be focused and to the point, avoiding unnecessary information\n", - "\n", - "Would you like me to demonstrate these guidelines with a sample email, or is there anything specific about these guidelines you'd like me to clarify?\n" - ] - } - ], - "source": [ - "state = agent.get_state(config)\n", - "for m in state.values.get(\"messages\"):\n", - " m.pretty_print()" - ] - }, - { - "cell_type": "code", - "execution_count": 26, + "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "Item(namespace=['email_assistant', 'lance', 'instructions'], key='user_instructions', value=\"\\nUse these tools when appropriate to help manage {name}'s tasks efficiently.\\nIf updating the user profile, do not omit past information. Simply it with new information from the email that that fits within the profile schema.\\n\", created_at='2025-02-06T23:35:04.545060+00:00', updated_at='2025-02-06T23:35:04.545063+00:00')" + "[HumanMessage(content=\"Respond to the email {'author': 'Alice Jones ', 'to': 'John Doe ', 'subject': 'Quick question about API documentation', 'email_thread': 'Hi John,\\\\n\\\\nUrgent issue - your service is down. Is there a reason why'}\", additional_kwargs={}, response_metadata={}, id='d41242a4-bfca-4221-82d5-642620f848b4'),\n", + " AIMessage(content=[{'citations': None, 'text': \"I'll help draft a response to Alice regarding the service outage. Let me write an email addressing this urgent matter.\", 'type': 'text'}, {'id': 'toolu_01XzQxGkWwbAhoicDmKqXSpL', 'input': {'to': 'alice.jones@bar.com', 'subject': 'Re: Quick question about API documentation', 'content': \"Hi Alice,\\n\\nThank you for bringing this urgent matter to our attention. I'm John's executive assistant, and I'm acknowledging receipt of your message about the service being down. I've flagged this as high priority, and I'm working to get this addressed immediately.\\n\\nCould you please provide more specific details about which service you're experiencing issues with? This will help us investigate and resolve the problem more quickly.\\n\\nIn the meantime, I'm escalating this to our technical team.\\n\\nBest regards,\\nOn behalf of John Doe\"}, 'name': 'write_email', 'type': 'tool_use'}], additional_kwargs={}, response_metadata={'id': 'msg_01WQGe8YqzMr8VjFcAHkNU2X', 'model': 'claude-3-5-sonnet-20241022', 'stop_reason': 'tool_use', 'stop_sequence': None, 'usage': {'cache_creation_input_tokens': 0, 'cache_read_input_tokens': 0, 'input_tokens': 1199, 'output_tokens': 235}}, id='run-33488909-2712-4a9a-ab78-3d4bed2c9b8d-0', tool_calls=[{'name': 'write_email', 'args': {'to': 'alice.jones@bar.com', 'subject': 'Re: Quick question about API documentation', 'content': \"Hi Alice,\\n\\nThank you for bringing this urgent matter to our attention. I'm John's executive assistant, and I'm acknowledging receipt of your message about the service being down. I've flagged this as high priority, and I'm working to get this addressed immediately.\\n\\nCould you please provide more specific details about which service you're experiencing issues with? This will help us investigate and resolve the problem more quickly.\\n\\nIn the meantime, I'm escalating this to our technical team.\\n\\nBest regards,\\nOn behalf of John Doe\"}, 'id': 'toolu_01XzQxGkWwbAhoicDmKqXSpL', 'type': 'tool_call'}], usage_metadata={'input_tokens': 1199, 'output_tokens': 235, 'total_tokens': 1434, 'input_token_details': {'cache_read': 0, 'cache_creation': 0}}),\n", + " ToolMessage(content=\"Email sent to alice.jones@bar.com with subject 'Re: Quick question about API documentation'\", name='write_email', id='07548c21-555c-42ed-9941-53a0d56f66ed', tool_call_id='toolu_01XzQxGkWwbAhoicDmKqXSpL'),\n", + " AIMessage(content=[{'citations': None, 'text': \"I've sent an immediate response to acknowledge the urgent issue while requesting more specific information about the affected service. This will help the technical team investigate the problem more effectively. I'll also create a memory of this incident for future reference.\", 'type': 'text'}, {'id': 'toolu_014ug8pHiZ8cTnGhjweekkf6', 'input': {'action': 'create', 'content': 'Service outage reported by Alice Jones (alice.jones@bar.com) on API documentation service. Immediate response sent requesting more details.'}, 'name': 'manage_memory', 'type': 'tool_use'}], additional_kwargs={}, response_metadata={'id': 'msg_01CcBqJRALCEspMmTspf6TM7', 'model': 'claude-3-5-sonnet-20241022', 'stop_reason': 'tool_use', 'stop_sequence': None, 'usage': {'cache_creation_input_tokens': 0, 'cache_read_input_tokens': 0, 'input_tokens': 1467, 'output_tokens': 145}}, id='run-82264acc-a344-469f-8be2-534d0c952c72-0', tool_calls=[{'name': 'manage_memory', 'args': {'action': 'create', 'content': 'Service outage reported by Alice Jones (alice.jones@bar.com) on API documentation service. Immediate response sent requesting more details.'}, 'id': 'toolu_014ug8pHiZ8cTnGhjweekkf6', 'type': 'tool_call'}], usage_metadata={'input_tokens': 1467, 'output_tokens': 145, 'total_tokens': 1612, 'input_token_details': {'cache_read': 0, 'cache_creation': 0}}),\n", + " ToolMessage(content='created memory 38428efb-3919-4ae2-97be-075f3c6ad325', name='manage_memory', id='012205e2-5fe7-495d-925c-cb1e4a8a85ab', tool_call_id='toolu_014ug8pHiZ8cTnGhjweekkf6'),\n", + " AIMessage(content=\"Would you like me to schedule an urgent follow-up meeting with Alice and the technical team once we receive more details about the specific service that's down?\", additional_kwargs={}, response_metadata={'id': 'msg_01NvDnHH1qK4UW5L5egavD4U', 'model': 'claude-3-5-sonnet-20241022', 'stop_reason': 'end_turn', 'stop_sequence': None, 'usage': {'cache_creation_input_tokens': 0, 'cache_read_input_tokens': 0, 'input_tokens': 1648, 'output_tokens': 35}}, id='run-b4ef5dce-fa90-4963-afcb-785a092d0fcf-0', usage_metadata={'input_tokens': 1648, 'output_tokens': 35, 'total_tokens': 1683, 'input_token_details': {'cache_read': 0, 'cache_creation': 0}})]" ] }, - "execution_count": 26, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "namespace = (\"email_assistant\", user, \"instructions\")\n", - "key = \"user_instructions\"\n", - "instructions = store.get(namespace, key)\n", - "instructions" + "response['messages']" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "\"Use these tools when appropriate to help manage John's tasks efficiently.\"" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "store.get((\"lance\",), \"agent_instructions\").value['prompt']" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Direct questions from team members, meeting requests, critical bug reports'" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "store.get((\"lance\",), \"triage_respond\").value['prompt']" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Marketing newsletters, spam emails, mass company announcements'" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "store.get((\"lance\",), \"triage_ignore\").value['prompt']" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Team member out sick, build system notifications, project status updates'" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "store.get((\"lance\",), \"triage_notify\").value['prompt']" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[{'name': 'main_agent', 'prompt': 'Use these tools when appropriate to help manage John\\'s tasks efficiently. When writing emails, always sign them as \"John Doe\".', 'update_instructions': 'keep the instructions short and to the point', 'when_to_update': 'Update this prompt whenever there is feedback on how the agent should write emails or schedule events'}, {'name': 'triage-ignore', 'prompt': 'Marketing newsletters, spam emails, mass company announcements', 'update_instructions': 'keep the instructions short and to the point', 'when_to_update': 'Update this prompt whenever there is feedback on which emails should be ignored'}, {'name': 'triage-notify', 'prompt': 'Team member out sick, build system notifications, project status updates', 'update_instructions': 'keep the instructions short and to the point', 'when_to_update': 'Update this prompt whenever there is feedback on which emails the user should be notified of'}, {'name': 'triage-respond', 'prompt': 'Direct questions from team members, meeting requests, critical bug reports', 'update_instructions': 'keep the instructions short and to the point', 'when_to_update': 'Update this prompt whenever there is feedback on which emails should be responded to'}]\n" + ] + } + ], + "source": [ + "from langmem import create_multi_prompt_optimizer\n", + "\n", + "# Example team: researcher finds information, writer creates reports\n", + "conversations = [\n", + " (\n", + " response['messages'],\n", + " \"Always signs your emails `John Doe`\"\n", + " )\n", + "]\n", + "\n", + "# Define prompts for each role\n", + "prompts = [\n", + " {\n", + " \"name\": \"main_agent\",\n", + " \"prompt\": store.get((\"lance\",), \"agent_instructions\").value['prompt'],\n", + " \"update_instructions\": \"keep the instructions short and to the point\",\n", + " \"when_to_update\": \"Update this prompt whenever there is feedback on how the agent should write emails or schedule events\"\n", + " \n", + " },\n", + " {\n", + " \"name\": \"triage-ignore\", \n", + " \"prompt\": store.get((\"lance\",), \"triage_ignore\").value['prompt'],\n", + " \"update_instructions\": \"keep the instructions short and to the point\",\n", + " \"when_to_update\": \"Update this prompt whenever there is feedback on which emails should be ignored\"\n", + "\n", + " },\n", + " {\n", + " \"name\": \"triage-notify\", \n", + " \"prompt\": store.get((\"lance\",), \"triage_notify\").value['prompt'],\n", + " \"update_instructions\": \"keep the instructions short and to the point\",\n", + " \"when_to_update\": \"Update this prompt whenever there is feedback on which emails the user should be notified of\"\n", + "\n", + " },\n", + " {\n", + " \"name\": \"triage-respond\", \n", + " \"prompt\": store.get((\"lance\",), \"triage_respond\").value['prompt'],\n", + " \"update_instructions\": \"keep the instructions short and to the point\",\n", + " \"when_to_update\": \"Update this prompt whenever there is feedback on which emails should be responded to\"\n", + "\n", + " },\n", + "]\n", + "\n", + "# Create optimizer\n", + "optimizer = create_multi_prompt_optimizer(\n", + " \"anthropic:claude-3-5-sonnet-latest\",\n", + " kind=\"prompt_memory\",\n", + ")\n", + "\n", + "# Update all prompts based on team performance\n", + "updated = optimizer.invoke({\"trajectories\": conversations, \"prompts\": prompts})\n", + "print(updated)" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "updated main_agent\n" + ] + } + ], + "source": [ + "for i, updated_prompt in enumerate(updated):\n", + " old_prompt = prompts[i]\n", + " if updated_prompt['prompt'] != old_prompt['prompt']:\n", + " name = old_prompt['name']\n", + " print(f\"updated {name}\")\n", + " if name == \"main_agent\":\n", + " store.put((\"lance\",), \"agent_instructions\", {\"prompt\":updated_prompt['prompt']})\n", + " else:\n", + " raise ValueError" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "📧 Classification: RESPOND - This email requires a response\n" + ] + } + ], + "source": [ + "email_input = {\n", + " \"author\": \"Alice Jones \",\n", + " \"to\": \"John Doe \",\n", + " \"subject\": \"Quick question about API documentation\",\n", + " \"email_thread\": \"\"\"Hi John,\n", + "\n", + "Urgent issue - your service is down. Is there a reason why\"\"\",\n", + "}\n", + "\n", + "config = {\"configurable\": {\"langgraph_user_id\": \"lance\"}}\n", + "\n", + "response = agent.invoke({\"email_input\": email_input}, config=config)" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[HumanMessage(content=\"Respond to the email {'author': 'Alice Jones ', 'to': 'John Doe ', 'subject': 'Quick question about API documentation', 'email_thread': 'Hi John,\\\\n\\\\nUrgent issue - your service is down. Is there a reason why'}\", additional_kwargs={}, response_metadata={}, id='72189664-437e-42f0-a1be-fea499e10e6f'),\n", + " AIMessage(content=[{'citations': None, 'text': \"I'll help draft a response to Alice regarding the service outage. Let me send an email using the write_email function.\", 'type': 'text'}, {'id': 'toolu_01DiEprfkmjvtJXmo7brCStB', 'input': {'to': 'alice.jones@bar.com', 'subject': 'Re: Quick question about API documentation', 'content': \"Hi Alice,\\n\\nThank you for bringing this to my attention. I'll look into the service outage immediately and have our team investigate the cause.\\n\\nI'll get back to you as soon as I have more information about the situation and an estimated time for resolution.\\n\\nBest regards,\\nJohn Doe\"}, 'name': 'write_email', 'type': 'tool_use'}], additional_kwargs={}, response_metadata={'id': 'msg_01LEP9NQ96L3RF5NXQx2eXsc', 'model': 'claude-3-5-sonnet-20241022', 'stop_reason': 'tool_use', 'stop_sequence': None, 'usage': {'cache_creation_input_tokens': 0, 'cache_read_input_tokens': 0, 'input_tokens': 1212, 'output_tokens': 187}}, id='run-3582b0b0-1952-44cc-9aa3-1b86696ee22c-0', tool_calls=[{'name': 'write_email', 'args': {'to': 'alice.jones@bar.com', 'subject': 'Re: Quick question about API documentation', 'content': \"Hi Alice,\\n\\nThank you for bringing this to my attention. I'll look into the service outage immediately and have our team investigate the cause.\\n\\nI'll get back to you as soon as I have more information about the situation and an estimated time for resolution.\\n\\nBest regards,\\nJohn Doe\"}, 'id': 'toolu_01DiEprfkmjvtJXmo7brCStB', 'type': 'tool_call'}], usage_metadata={'input_tokens': 1212, 'output_tokens': 187, 'total_tokens': 1399, 'input_token_details': {'cache_read': 0, 'cache_creation': 0}}),\n", + " ToolMessage(content=\"Email sent to alice.jones@bar.com with subject 'Re: Quick question about API documentation'\", name='write_email', id='d5e70aa9-5a03-46e7-ac81-da26c0d8f28a', tool_call_id='toolu_01DiEprfkmjvtJXmo7brCStB'),\n", + " AIMessage(content=[{'citations': None, 'text': \"I've sent an acknowledgment email to Alice that:\\n1. Confirms we're aware of the situation\\n2. Indicates we'll investigate immediately\\n3. Promises to follow up with more information\\n\\nLet me also store this interaction in memory for future reference.\", 'type': 'text'}, {'id': 'toolu_01Mizcx4rduwwxoH1nGmL1sL', 'input': {'action': 'create', 'content': 'Service outage reported by Alice Jones (alice.jones@bar.com) - Urgent issue reported on service being down. Initial response sent promising investigation and follow-up.'}, 'name': 'manage_memory', 'type': 'tool_use'}], additional_kwargs={}, response_metadata={'id': 'msg_018hSfftAtyJhMCRy2pSoffW', 'model': 'claude-3-5-sonnet-20241022', 'stop_reason': 'tool_use', 'stop_sequence': None, 'usage': {'cache_creation_input_tokens': 0, 'cache_read_input_tokens': 0, 'input_tokens': 1432, 'output_tokens': 161}}, id='run-3ac9a74d-a3da-4bf4-99b9-764572b47e28-0', tool_calls=[{'name': 'manage_memory', 'args': {'action': 'create', 'content': 'Service outage reported by Alice Jones (alice.jones@bar.com) - Urgent issue reported on service being down. Initial response sent promising investigation and follow-up.'}, 'id': 'toolu_01Mizcx4rduwwxoH1nGmL1sL', 'type': 'tool_call'}], usage_metadata={'input_tokens': 1432, 'output_tokens': 161, 'total_tokens': 1593, 'input_token_details': {'cache_read': 0, 'cache_creation': 0}}),\n", + " ToolMessage(content='created memory fa8259d9-2e0e-45de-9680-35054fa51ab6', name='manage_memory', id='06286142-4a03-4a70-93dd-3b60d6161f4b', tool_call_id='toolu_01Mizcx4rduwwxoH1nGmL1sL'),\n", + " AIMessage(content='Is there anything else you need assistance with regarding this situation?', additional_kwargs={}, response_metadata={'id': 'msg_01Sf8amKzMxCGSn3QiLoswHy', 'model': 'claude-3-5-sonnet-20241022', 'stop_reason': 'end_turn', 'stop_sequence': None, 'usage': {'cache_creation_input_tokens': 0, 'cache_read_input_tokens': 0, 'input_tokens': 1629, 'output_tokens': 16}}, id='run-2b0ee89f-883b-4022-9a6f-43c928eeed5e-0', usage_metadata={'input_tokens': 1629, 'output_tokens': 16, 'total_tokens': 1645, 'input_token_details': {'cache_read': 0, 'cache_creation': 0}})]" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response['messages']" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[{'name': 'main_agent', 'prompt': 'Use these tools when appropriate to help manage John\\'s tasks efficiently. When writing emails, always sign them as \"John Doe\".', 'update_instructions': 'keep the instructions short and to the point', 'when_to_update': 'Update this prompt whenever there is feedback on how the agent should write emails or schedule events'}, {'name': 'triage-ignore', 'prompt': 'Marketing newsletters, spam emails, mass company announcements, and any emails from Alice Jones (alice.jones@bar.com) should be ignored.', 'update_instructions': 'keep the instructions short and to the point', 'when_to_update': 'Update this prompt whenever there is feedback on which emails should be ignored'}, {'name': 'triage-notify', 'prompt': 'Team member out sick, build system notifications, project status updates', 'update_instructions': 'keep the instructions short and to the point', 'when_to_update': 'Update this prompt whenever there is feedback on which emails the user should be notified of'}, {'name': 'triage-respond', 'prompt': 'Direct questions from team members, meeting requests, critical bug reports', 'update_instructions': 'keep the instructions short and to the point', 'when_to_update': 'Update this prompt whenever there is feedback on which emails should be responded to'}]\n" + ] + } + ], + "source": [ + "from langmem import create_multi_prompt_optimizer\n", + "\n", + "# Example team: researcher finds information, writer creates reports\n", + "conversations = [\n", + " (\n", + " response['messages'],\n", + " \"Ignore any emails from Alice Jones\"\n", + " )\n", + "]\n", + "\n", + "# Define prompts for each role\n", + "# Define prompts for each role\n", + "prompts = [\n", + " {\n", + " \"name\": \"main_agent\",\n", + " \"prompt\": store.get((\"lance\",), \"agent_instructions\").value['prompt'],\n", + " \"update_instructions\": \"keep the instructions short and to the point\",\n", + " \"when_to_update\": \"Update this prompt whenever there is feedback on how the agent should write emails or schedule events\"\n", + " \n", + " },\n", + " {\n", + " \"name\": \"triage-ignore\", \n", + " \"prompt\": store.get((\"lance\",), \"triage_ignore\").value['prompt'],\n", + " \"update_instructions\": \"keep the instructions short and to the point\",\n", + " \"when_to_update\": \"Update this prompt whenever there is feedback on which emails should be ignored\"\n", + "\n", + " },\n", + " {\n", + " \"name\": \"triage-notify\", \n", + " \"prompt\": store.get((\"lance\",), \"triage_notify\").value['prompt'],\n", + " \"update_instructions\": \"keep the instructions short and to the point\",\n", + " \"when_to_update\": \"Update this prompt whenever there is feedback on which emails the user should be notified of\"\n", + "\n", + " },\n", + " {\n", + " \"name\": \"triage-respond\", \n", + " \"prompt\": store.get((\"lance\",), \"triage_respond\").value['prompt'],\n", + " \"update_instructions\": \"keep the instructions short and to the point\",\n", + " \"when_to_update\": \"Update this prompt whenever there is feedback on which emails should be responded to\"\n", + "\n", + " },\n", + "]\n", + "\n", + "# Create optimizer\n", + "optimizer = create_multi_prompt_optimizer(\n", + " \"anthropic:claude-3-5-sonnet-latest\",\n", + " kind=\"prompt_memory\",\n", + ")\n", + "\n", + "# Update all prompts based on team performance\n", + "updated = optimizer.invoke({\"trajectories\": conversations, \"prompts\": prompts})\n", + "print(updated)" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "updated triage-ignore\n" + ] + } + ], + "source": [ + "for i, updated_prompt in enumerate(updated):\n", + " old_prompt = prompts[i]\n", + " if updated_prompt['prompt'] != old_prompt['prompt']:\n", + " name = old_prompt['name']\n", + " print(f\"updated {name}\")\n", + " if name == \"main_agent\":\n", + " store.put((\"lance\",), \"agent_instructions\", {\"prompt\":updated_prompt['prompt']})\n", + " if name == \"triage-ignore\":\n", + " store.put((\"lance\",), \"triage_ignore\", {\"prompt\":updated_prompt['prompt']})\n", + " else:\n", + " raise ValueError" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🚫 Classification: IGNORE - This email can be safely ignored\n" + ] + } + ], + "source": [ + "email_input = {\n", + " \"author\": \"Alice Jones \",\n", + " \"to\": \"John Doe \",\n", + " \"subject\": \"Quick question about API documentation\",\n", + " \"email_thread\": \"\"\"Hi John,\n", + "\n", + "Urgent issue - your service is down. Is there a reason why\"\"\",\n", + "}\n", + "\n", + "config = {\"configurable\": {\"langgraph_user_id\": \"lance\"}}\n", + "\n", + "response = agent.invoke({\"email_input\": email_input}, config=config)" ] }, { @@ -573,7 +983,7 @@ ], "metadata": { "kernelspec": { - "display_name": "memory-course-env", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -587,9 +997,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.1" + "version": "3.11.7" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/src/memory_course/prompts.py b/src/memory_course/prompts.py index 239378a..95f02e7 100644 --- a/src/memory_course/prompts.py +++ b/src/memory_course/prompts.py @@ -13,7 +13,7 @@ You have access to the following tools to help manage {name}'s communications an < Instructions > -Use these tools when appropriate to help manage {name}'s tasks efficiently. +{instructions} """