cr

README.md

@@ -1,4 +1,10 @@
-# Demo Voice React Agent
+# 🦜🎤 Voice ReAct Agent
+
+This is an implementation of a [ReAct](https://arxiv.org/abs/2210.03629)-style agent that uses OpenAI's new [Realtime API](https://platform.openai.com/docs/guides/realtime).
+
+Specifically, we enable this model to call tools by providing it a list of [LangChain tools](https://python.langchain.com/docs/how_to/custom_tools/#creating-tools-from-functions). It is easy to write custom tools, and you can easily pass these to the model.
+
+![](static/react.png)
 
 ## Installation
 
@@ -15,6 +21,8 @@ export OPENAI_API_KEY=your_openai_api_key
 export TAVILY_API_KEY=your_tavily_api_key
 ```
 
+Note: the Tavily API key is for the Tavily search engine, you can get an API key [here](https://app.tavily.com/). This is just an example tool, and if you do not want to use it you do not have to (see [Adding your own tools](#adding-your-own-tools))
+
 ## Running the project
 
 To run the project, execute the following command:
@@ -28,7 +36,21 @@ uv run src/server/app.py
 
 Now you can open the browser and navigate to `http://localhost:3000` to see the project running.
 
+### Enable microphone
+
+You may need to make sure that your browser can access your microphone.
+
+- [Chrome](http://0.0.0.0:3000/)
+
 ## Adding your own tools
 
-You can add your own tools by adding them to the `server/src/server/app.py` file
-in the webhook handler.
+You can add your own tools by adding them to the `server/src/server/tools.py` file.
+
+## Adding your own custom instructions
+
+You can add your own custom instructions by adding them to the `server/src/server/prompt.py` file.
+
+## Next steps
+
+[] Enable interrupting the AI
+[] Enable changing of instructions/tools based on state

server/src/server/app.py

@@ -5,12 +5,12 @@ from starlette.routing import Route, WebSocketRoute
 from starlette.staticfiles import StaticFiles
 from starlette.websockets import WebSocket
 
-from langchain_core.tools import tool
 from langchain_openai_voice import OpenAIVoiceReactAgent
 
-from langchain_community.tools import TavilySearchResults
 
 from server.utils import websocket_stream
+from server.prompt import INSTRUCTIONS
+from server.tools import TOOLS
 
 
 async def websocket_endpoint(websocket: WebSocket):
@@ -18,21 +18,10 @@ async def websocket_endpoint(websocket: WebSocket):
 
     browser_receive_stream = websocket_stream(websocket)
 
-    @tool
-    def add(a: int, b: int):
-        """Add two numbers. Please let the user know that you're adding the numbers BEFORE you call the tool"""
-        return a + b
-
-    tavily_tool = TavilySearchResults(
-        max_results=5,
-        include_answer=True,
-    )
-    tavily_tool.description += "\n\nLet the user know you're asking your friend Tavily for help before you call the tool."
-
     agent = OpenAIVoiceReactAgent(
         model="gpt-4o-realtime-preview",
-        tools=[add],  # no tools for now
-        instructions="You are a helpful assistant.",
+        tools=TOOLS,
+        instructions=INSTRUCTIONS,
     )
 
     await agent.aconnect(browser_receive_stream, websocket.send_text)

server/src/server/prompt.py

@@ -0,0 +1 @@
+INSTRUCTIONS = "You are a helpful assistant."

server/src/server/tools.py

@@ -0,0 +1,17 @@
+from langchain_core.tools import tool
+
+from langchain_community.tools import TavilySearchResults
+
+
+@tool
+def add(a: int, b: int):
+    """Add two numbers. Please let the user know that you're adding the numbers BEFORE you call the tool"""
+    return a + b
+
+tavily_tool = TavilySearchResults(
+    max_results=5,
+    include_answer=True,
+    description="This is a search tool for accessing the internet.\n\nLet the user know you're asking your friend Tavily for help before you call the tool."
+)
+
+TOOLS = [add, tavily_tool]