Release 0.0.51 (#530)

@eyurtsev

.clabot

@@ -0,0 +1,4 @@
+{
+  "contributors": ["eyurtsev", "hwchase17", "nfcampos", "efriis", "jacoblee93", "dqbd", "kreneskyp", "adarsh-jha-dev", "harris", "baskaryan", "hinthornw", "bracesproul", "jakerachleff", "craigsdennis", "anhi", "169", "LarchLiu", "PaulLockett", "RCMatthias", "jwynia", "majiayu000", "mpskex", "shivachittamuru", "sinashaloudegi", "sowsan", "akira", "lucianotonet", "JGalego", "nat-n", "dirien"],
+  "message": "Thank you for your pull request and welcome to our community. We require contributors to sign our Contributor License Agreement, and we don't seem to have the username {{usersWithoutCLA}} on file. In order for us to review and merge your code, please complete the Individual Contributor License Agreement here https://forms.gle/AQFbtkWRoHXUgipM6 .\n\nThis process is done manually on our side, so after signing the form one of the maintainers will add you to the contributors list.\n\nFor more details about why we have a CLA and other contribution guidelines please see: https://github.com/langchain-ai/langserve/blob/main/CONTRIBUTING.md."
+}

.github/workflows/_pydantic_compatibility.yml

@@ -13,6 +13,7 @@ env:
 
 jobs:
   build:
+    timeout-minutes: 10
     defaults:
       run:
         working-directory: ${{ inputs.working-directory }}

.github/workflows/langserve_ci.yml

@@ -11,8 +11,10 @@ on:
       - '.github/workflows/_test.yml'
       - '.github/workflows/langserve_ci.yml'
       - 'langserve/**'
+      - 'tests/**'
       - 'examples/**'
       - 'pyproject.toml'
+      - 'poetry.lock'
       - 'Makefile'
   workflow_dispatch:  # Allows to trigger the workflow manually in GitHub UI
 
@@ -38,7 +40,14 @@ jobs:
       working-directory: .
     secrets: inherit
 
+  pydantic-compatibility:
+    uses:
+      ./.github/workflows/_pydantic_compatibility.yml
+    with:
+      working-directory: .
+    secrets: inherit
   test:
+    timeout-minutes: 10
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -50,7 +59,7 @@ jobs:
           - "3.9"
           - "3.10"
           - "3.11"
-    name: Python ${{ matrix.python-version }} extended tests
+    name: Python ${{ matrix.python-version }} tests
     steps:
       - uses: actions/checkout@v3
 

.gitignore

@@ -160,3 +160,6 @@ cython_debug/
 #.idea/
 
 .envrc
+
+# IntelliJ IDE's
+.idea

CONTRIBUTING.md

@@ -49,3 +49,32 @@ To run linting for this project:
 ```sh
 make lint
 ```
+
+## Frontend Playground Development
+
+Here are a few tips to keep in mind when developing the LangServe playgrounds:
+
+### Setup
+
+Switch directories to `langserve/playground` or `langserve/chat_playground`, then run `yarn` to install required
+dependencies. `yarn dev` will start the playground at `http://localhost:5173/____LANGSERVE_BASE_URL/` in dev mode.
+
+You can run one of the chains in the `examples/` repo using `poetry run python path/to/file.py`.
+
+### Setting CORS
+
+You may need to add the following to an example route when developing the playground in dev mode to handle CORS:
+
+```python
+from fastapi.middleware.cors import CORSMiddleware
+
+# Set all CORS enabled origins
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+    expose_headers=["*"],
+)
+```

Makefile

@@ -33,10 +33,10 @@ lint_diff format_diff: PYTHON_FILES=$(shell git diff --relative=. --name-only --
 
 lint lint_diff:
 	poetry run ruff .
-	poetry run black $(PYTHON_FILES) --check
+	poetry run ruff format $(PYTHON_FILES) --check
 
 format format_diff:
-	poetry run black $(PYTHON_FILES)
+	poetry run ruff format $(PYTHON_FILES)
 	poetry run ruff --select I --fix $(PYTHON_FILES)
 
 spell_check:

README.md

@@ -1,50 +1,125 @@
-# LangServe 🦜️🔗 
+# 🦜️🏓 LangServe
+
+[![Release Notes](https://img.shields.io/github/release/langchain-ai/langserve)](https://github.com/langchain-ai/langserve/releases)
+[![Downloads](https://static.pepy.tech/badge/langserve/month)](https://pepy.tech/project/langserve)
+[![Open Issues](https://img.shields.io/github/issues-raw/langchain-ai/langserve)](https://github.com/langchain-ai/langserve/issues)
+[![](https://dcbadge.vercel.app/api/server/6adMQxSpJS?compact=true&style=flat)](https://discord.com/channels/1038097195422978059/1170024642245832774)
+
+🚩 We will be releasing a hosted version of LangServe for one-click deployments of
+LangChain applications. [Sign up here](https://airtable.com/app0hN6sd93QcKubv/shrAjst60xXa6quV2)
+to get on the waitlist.
 
 ## Overview
 
-`LangServe` helps developers deploy `LangChain` [runnables and chains](https://python.langchain.com/docs/expression_language/) as a REST API.
+[LangServe](https://github.com/langchain-ai/langserve) helps developers
+deploy `LangChain` [runnables and chains](https://python.langchain.com/docs/expression_language/)
+as a REST API.
 
-This library is integrated with [FastAPI](https://fastapi.tiangolo.com/) and uses [pydantic](https://docs.pydantic.dev/latest/) for data validation.
+This library is integrated with [FastAPI](https://fastapi.tiangolo.com/) and
+uses [pydantic](https://docs.pydantic.dev/latest/) for data validation.
 
-In addition, it provides a client that can be used to call into runnables deployed on a server.
-A javascript client is available in [LangChainJS](https://js.langchain.com/docs/api/runnables_remote/classes/RemoteRunnable).
+In addition, it provides a client that can be used to call into runnables deployed on a
+server.
+A JavaScript client is available
+in [LangChain.js](https://js.langchain.com/docs/ecosystem/langserve).
 
 ## Features
 
-- Input and Output schemas automatically inferred from your LangChain object, and enforced on every API call, with rich error messages
+- Input and Output schemas automatically inferred from your LangChain object, and
+  enforced on every API call, with rich error messages
 - API docs page with JSONSchema and Swagger (insert example link)
-- Efficient `/invoke`, `/batch` and `/stream` endpoints with support for many concurrent requests on a single server
-- `/stream_log` endpoint for streaming all (or some) intermediate steps from your chain/agent
-- Built-in (optional) tracing to [LangSmith](https://www.langchain.com/langsmith), just add your API key (see [Instructions](https://docs.smith.langchain.com/)])
-- All built with battle-tested open-source Python libraries like FastAPI, Pydantic, uvloop and asyncio.
-- Use the client SDK to call a LangServe server as if it was a Runnable running locally (or call the HTTP API directly)
+- Efficient `/invoke/`, `/batch/` and `/stream/` endpoints with support for many
+  concurrent requests on a single server
+- `/stream_log/` endpoint for streaming all (or some) intermediate steps from your
+  chain/agent
+- **new** as of 0.0.40, supports `astream_events` to make it easier to stream without needing to parse the output of `stream_log`.
+- Playground page at `/playground/` with streaming output and intermediate steps
+- Built-in (optional) tracing to [LangSmith](https://www.langchain.com/langsmith), just
+  add your API key (see [Instructions](https://docs.smith.langchain.com/))
+- All built with battle-tested open-source Python libraries like FastAPI, Pydantic,
+  uvloop and asyncio.
+- Use the client SDK to call a LangServe server as if it was a Runnable running
+  locally (or call the HTTP API directly)
+- [LangServe Hub](https://github.com/langchain-ai/langchain/blob/master/templates/README.md)
 
-### Limitations
+## Limitations
 
 - Client callbacks are not yet supported for events that originate on the server
-- Does not work with [pydantic v2 yet](https://github.com/tiangolo/fastapi/issues/10360)
+- OpenAPI docs will not be generated when using Pydantic V2. Fast API does not
+  support [mixing pydantic v1 and v2 namespaces](https://github.com/tiangolo/fastapi/issues/10360).
+  See section below for more details.
 
-## LangChain CLI  🛠️
+## Hosted LangServe
+
+We will be releasing a hosted version of LangServe for one-click deployments of
+LangChain
+applications. [Sign up here](https://airtable.com/apppQ9p5XuujRl3wJ/shrABpHWdxry8Bacm)
+to get on the waitlist.
+
+## Security
+
+* Vulnerability in Versions 0.0.13 - 0.0.15 -- playground endpoint allows accessing
+  arbitrary files on
+  server. [Resolved in 0.0.16](https://github.com/langchain-ai/langserve/pull/98).
+
+## Installation
+
+For both client and server:
+
+```bash
+pip install "langserve[all]"
+```
+
+or `pip install "langserve[client]"` for client code,
+and `pip install "langserve[server]"` for server code.
+
+## LangChain CLI 🛠️
 
 Use the `LangChain` CLI to bootstrap a `LangServe` project quickly.
 
-To use the langchain CLI make sure that you have a recent version of `langchain` installed
-and also `typer`. (`pip install langchain typer` or `pip install "langchain[cli]"`)
+To use the langchain CLI make sure that you have a recent version of `langchain-cli`
+installed. You can install it with `pip install -U langchain-cli`.
 
 ```sh
-langchain ../path/to/directory
+langchain app new ../path/to/directory
 ```
 
-And follow the instructions...
-
 ## Examples
 
-For more examples, see the [examples](./examples) directory.
+Get your LangServe instance started quickly with
+[LangChain Templates](https://github.com/langchain-ai/langchain/blob/master/templates/README.md).
 
+For more examples, see the templates
+[index](https://github.com/langchain-ai/langchain/blob/master/templates/docs/INDEX.md)
+or the [examples](https://github.com/langchain-ai/langserve/tree/main/examples)
+directory.
+
+| Description                                                                                                                                                                                                                                                        | Links                                                                                                                                                                                                                               |
+|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| **LLMs** Minimal example that reserves OpenAI and Anthropic chat models. Uses async, supports batching and streaming.                                                                                                                                              | [server](https://github.com/langchain-ai/langserve/tree/main/examples/llm/server.py), [client](https://github.com/langchain-ai/langserve/blob/main/examples/llm/client.ipynb)                                                       | 
+| **Retriever** Simple server that exposes a retriever as a runnable.                                                                                                                                                                                                | [server](https://github.com/langchain-ai/langserve/tree/main/examples/retrieval/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/retrieval/client.ipynb)                                           | 
+| **Conversational Retriever** A [Conversational Retriever](https://python.langchain.com/docs/expression_language/cookbook/retrieval#conversational-retrieval-chain) exposed via LangServe                                                                           | [server](https://github.com/langchain-ai/langserve/tree/main/examples/conversational_retrieval_chain/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/conversational_retrieval_chain/client.ipynb) | 
+| **Agent** without **conversation history** based on [OpenAI tools](https://python.langchain.com/docs/modules/agents/agent_types/openai_functions_agent)                                                                                             | [server](https://github.com/langchain-ai/langserve/tree/main/examples/agent/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/agent/client.ipynb)                                                   | 
+| **Agent** with **conversation history** based on [OpenAI tools](https://python.langchain.com/docs/modules/agents/agent_types/openai_functions_agent)                                                                                                               | [server](https://github.com/langchain-ai/langserve/blob/main/examples/agent_with_history/server.py), [client](https://github.com/langchain-ai/langserve/blob/main/examples/agent_with_history/client.ipynb)                         | 
+| [RunnableWithMessageHistory](https://python.langchain.com/docs/expression_language/how_to/message_history) to implement chat persisted on backend, keyed off a `session_id` supplied by client.                                                                    | [server](https://github.com/langchain-ai/langserve/tree/main/examples/chat_with_persistence/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/chat_with_persistence/client.ipynb)                   |
+| [RunnableWithMessageHistory](https://python.langchain.com/docs/expression_language/how_to/message_history) to implement chat persisted on backend, keyed off a `conversation_id` supplied by client, and `user_id` (see Auth for implementing `user_id` properly). | [server](https://github.com/langchain-ai/langserve/tree/main/examples/chat_with_persistence_and_user/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/chat_with_persistence_and_user/client.ipynb) | 
+| [Configurable Runnable](https://python.langchain.com/docs/expression_language/how_to/configure) to create a retriever that supports run time configuration of the index name.                                                                                      | [server](https://github.com/langchain-ai/langserve/tree/main/examples/configurable_retrieval/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/configurable_retrieval/client.ipynb)                 | 
+| [Configurable Runnable](https://python.langchain.com/docs/expression_language/how_to/configure) that shows configurable fields and configurable alternatives.                                                                                                      | [server](https://github.com/langchain-ai/langserve/tree/main/examples/configurable_chain/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/configurable_chain/client.ipynb)                         | 
+| **APIHandler** Shows how to use `APIHandler` instead of `add_routes`. This provides more flexibility for developers to define endpoints. Works well with all FastAPI patterns, but takes a bit more effort.                                                        | [server](https://github.com/langchain-ai/langserve/tree/main/examples/api_handler_examples/server.py)                                                                                                                               |
+| **LCEL Example** Example that uses LCEL to manipulate a dictionary input.                                                                                                                                                                                          | [server](https://github.com/langchain-ai/langserve/tree/main/examples/passthrough_dict/client.ipynb), [client](https://github.com/langchain-ai/langserve/tree/main/examples/passthrough_dict/client.ipynb)                          | 
+| **Auth** with `add_routes`: Simple authentication that can be applied across all endpoints associated with app. (Not useful on its own for implementing per user logic.)                                                                                           | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/global_deps/server.py)                                                                                                                                   | 
+| **Auth** with `add_routes`: Simple authentication mechanism based on path dependencies. (No useful on its own for implementing per user logic.)                                                                                                                    | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/path_dependencies/server.py)                                                                                                                             | 
+| **Auth** with `add_routes`: Implement per user logic and auth for endpoints that use per request config modifier. (**Note**: At the moment, does not integrate with OpenAPI docs.)                                                                                 | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/per_req_config_modifier/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/auth/per_req_config_modifier/client.ipynb)     | 
+| **Auth** with `APIHandler`: Implement per user logic and auth that shows how to search only within user owned documents.                                                                                                                                           | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/api_handler/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/auth/api_handler/client.ipynb)                             | 
+| **Widgets** Different widgets that can be used with playground (file upload and chat)                                                                                                                                                                              | [server](https://github.com/langchain-ai/langserve/tree/main/examples/widgets/chat/tuples/server.py)                                                                                                                                            | 
+| **Widgets** File upload widget used for LangServe playground.                                                                                                                                                                                                      | [server](https://github.com/langchain-ai/langserve/tree/main/examples/file_processing/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/file_processing/client.ipynb)                               | 
+
+## Sample Application
 
 ### Server
 
-Here's a server that deploys an OpenAI chat model, an Anthropic chat model, and a chain that uses
+Here's a server that deploys an OpenAI chat model, an Anthropic chat model, and a chain
+that uses
 the Anthropic model to tell a joke about a topic.
 
 ```python
@@ -54,11 +129,10 @@ from langchain.prompts import ChatPromptTemplate
 from langchain.chat_models import ChatAnthropic, ChatOpenAI
 from langserve import add_routes
 
-
 app = FastAPI(
-  title="LangChain Server",
-  version="1.0",
-  description="A simple api server using Langchain's Runnable interfaces",
+    title="LangChain Server",
+    version="1.0",
+    description="A simple api server using Langchain's Runnable interfaces",
 )
 
 add_routes(
@@ -78,7 +152,7 @@ prompt = ChatPromptTemplate.from_template("tell me a joke about {topic}")
 add_routes(
     app,
     prompt | model,
-    path="/chain",
+    path="/joke",
 )
 
 if __name__ == "__main__":
@@ -87,14 +161,40 @@ if __name__ == "__main__":
     uvicorn.run(app, host="localhost", port=8000)
 ```
 
+If you intend to call your endpoint from the browser, you will also need to set CORS headers.
+You can use FastAPI's built-in middleware for that:
+
+```python
+from fastapi.middleware.cors import CORSMiddleware
+
+# Set all CORS enabled origins
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+    expose_headers=["*"],
+)
+```
+
 ### Docs
 
 If you've deployed the server above, you can view the generated OpenAPI docs using:
 
+> ⚠️ If using pydantic v2, docs will not be generated for *invoke*, *batch*, *stream*,
+*stream_log*. See [Pydantic](#pydantic) section below for more details.
+
 ```sh
 curl localhost:8000/docs
 ```
 
+make sure to **add** the `/docs` suffix.
+
+> ⚠️ Index page `/` is not defined by **design**, so `curl localhost:8000` or visiting
+> the URL
+> will return a 404. If you want content at `/` define an endpoint `@app.get("/")`.
+
 ### Client
 
 Python SDK
@@ -108,7 +208,7 @@ from langserve import RemoteRunnable
 
 openai = RemoteRunnable("http://localhost:8000/openai/")
 anthropic = RemoteRunnable("http://localhost:8000/anthropic/")
-joke_chain = RemoteRunnable("http://localhost:8000/chain/")
+joke_chain = RemoteRunnable("http://localhost:8000/joke/")
 
 joke_chain.invoke({"topic": "parrots"})
 
@@ -116,35 +216,37 @@ joke_chain.invoke({"topic": "parrots"})
 await joke_chain.ainvoke({"topic": "parrots"})
 
 prompt = [
-    SystemMessage(content='Act like either a cat or a parrot.'), 
+    SystemMessage(content='Act like either a cat or a parrot.'),
     HumanMessage(content='Hello!')
 ]
 
 # Supports astream
 async for msg in anthropic.astream(prompt):
     print(msg, end="", flush=True)
-    
+
 prompt = ChatPromptTemplate.from_messages(
     [("system", "Tell me a long story about {topic}")]
 )
-    
+
 # Can define custom chains
 chain = prompt | RunnableMap({
     "openai": openai,
     "anthropic": anthropic,
 })
 
-chain.batch([{ "topic": "parrots" }, { "topic": "cats" }])
+chain.batch([{"topic": "parrots"}, {"topic": "cats"}])
 ```
 
 In TypeScript (requires LangChain.js version 0.0.166 or later):
 
 ```typescript
-import { RemoteRunnable } from "langchain/runnables/remote";
+import { RemoteRunnable } from "@langchain/core/runnables/remote";
 
-const chain = new RemoteRunnable({ url: `http://localhost:8000/chain/invoke/` });
+const chain = new RemoteRunnable({
+    url: `http://localhost:8000/joke/`,
+});
 const result = await chain.invoke({
-  "topic": "cats", 
+    topic: "cats",
 });
 ```
 
@@ -152,8 +254,9 @@ Python using `requests`:
 
 ```python
 import requests
+
 response = requests.post(
-    "http://localhost:8000/chain/invoke/",
+    "http://localhost:8000/joke/invoke",
     json={'input': {'topic': 'cats'}}
 )
 response.json()
@@ -162,7 +265,7 @@ response.json()
 You can also use `curl`:
 
 ```sh
-curl --location --request POST 'http://localhost:8000/chain/invoke/' \
+curl --location --request POST 'http://localhost:8000/joke/invoke' \
     --header 'Content-Type: application/json' \
     --data-raw '{
         "input": {
@@ -171,17 +274,16 @@ curl --location --request POST 'http://localhost:8000/chain/invoke/' \
     }'
 ```
 
-
-## Endpoints 
+## Endpoints
 
 The following code:
 
 ```python
 ...
 add_routes(
-  app,
-  runnable,
-  path="/my_runnable",
+    app,
+    runnable,
+    path="/my_runnable",
 )
 ```
 
@@ -190,37 +292,142 @@ adds of these endpoints to the server:
 - `POST /my_runnable/invoke` - invoke the runnable on a single input
 - `POST /my_runnable/batch` - invoke the runnable on a batch of inputs
 - `POST /my_runnable/stream` - invoke on a single input and stream the output
-- `POST /my_runnable/stream_log` - invoke on a single input and stream the output, including output of intermediate steps as it's generated
+- `POST /my_runnable/stream_log` - invoke on a single input and stream the output,
+  including output of intermediate steps as it's generated
+- `POST /my_runnable/astream_events` - invoke on a single input and stream events as they are generated,
+ including from intermediate steps.
 - `GET /my_runnable/input_schema` - json schema for input to the runnable
 - `GET /my_runnable/output_schema` - json schema for output of the runnable
 - `GET /my_runnable/config_schema` - json schema for config of the runnable
 
-## Installation
+These endpoints match
+the [LangChain Expression Language interface](https://python.langchain.com/docs/expression_language/interface) --
+please reference this documentation for more details.
 
-For both client and server:
+## Playground
 
-```bash
-pip install "langserve[all]"
+You can find a playground page for your runnable at `/my_runnable/playground/`. This
+exposes a simple UI
+to [configure](https://python.langchain.com/docs/expression_language/how_to/configure)
+and invoke your runnable with streaming output and intermediate steps.
+
+<p align="center">
+<img src="https://github.com/langchain-ai/langserve/assets/3205522/5ca56e29-f1bb-40f4-84b5-15916384a276" width="50%"/>
+</p>
+
+### Widgets
+
+The playground supports [widgets](#playground-widgets) and can be used to test your
+runnable with different inputs. See the [widgets](#widgets) section below for more
+details.
+
+### Sharing
+
+In addition, for configurable runnables, the playground will allow you to configure the
+runnable and share a link with the configuration:
+
+<p align="center">
+<img src="https://github.com/langchain-ai/langserve/assets/3205522/86ce9c59-f8e4-4d08-9fa3-62030e0f521d" width="50%"/>
+</p>
+
+## Chat playground
+
+LangServe also supports a chat-focused playground that opt into and use under `/my_runnable/playground/`.
+Unlike the general playground, only certain types of runnables are supported - the runnable's input schema must
+be a `dict` with either:
+
+- a single key, and that key's value must be a list of chat messages.
+- two keys, one whose value is a list of messages, and the other representing the most recent message.
+
+We recommend you use the first format.
+
+The runnable must also return either an `AIMessage` or a string.
+
+To enable it, you must set `playground_type="chat",` when adding your route. Here's an example:
+
+```python
+# Declare a chain
+prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", "You are a helpful, professional assistant named Cob."),
+        MessagesPlaceholder(variable_name="messages"),
+    ]
+)
+
+chain = prompt | ChatAnthropic(model="claude-2")
+
+
+class InputChat(BaseModel):
+    """Input for the chat endpoint."""
+
+    messages: List[Union[HumanMessage, AIMessage, SystemMessage]] = Field(
+        ...,
+        description="The chat messages representing the current conversation.",
+    )
+
+
+add_routes(
+    app,
+    chain.with_types(input_type=InputChat),
+    enable_feedback_endpoint=True,
+    enable_public_trace_link_endpoint=True,
+    playground_type="chat",
+)
 ```
 
-or `pip install "langserve[client]"` for client code, and `pip install "langserve[server]"` for server code.
+If you are using LangSmith, you can also set `enable_feedback_endpoint=True` on your route to enable thumbs-up/thumbs-down buttons
+after each message, and `enable_public_trace_link_endpoint=True` to add a button that creates a public traces for runs.
+Note that you will also need to set the following environment variables:
+
+```bash
+export LANGCHAIN_TRACING_V2="true"
+export LANGCHAIN_PROJECT="YOUR_PROJECT_NAME"
+export LANGCHAIN_API_KEY="YOUR_API_KEY"
+```
+
+Here's an example with the above two options turned on:
+
+<p align="center">
+<img src="./.github/img/chat_playground.png" width="50%"/>
+</p>
+
+Note: If you enable public trace links, the internals of your chain will be exposed. We recommend only using this setting
+for demos or testing.
 
 ## Legacy Chains
 
-LangServe works with both Runnables (constructed via [LangChain Expression Language](https://python.langchain.com/docs/expression_language/)) and legacy chains (inheriting from `Chain`).
-However, some of the input schemas for legacy chains may be incomplete/incorrect, leading to errors.
+LangServe works with both Runnables (constructed
+via [LangChain Expression Language](https://python.langchain.com/docs/expression_language/))
+and legacy chains (inheriting from `Chain`).
+However, some of the input schemas for legacy chains may be incomplete/incorrect,
+leading to errors.
 This can be fixed by updating the `input_schema` property of those chains in LangChain.
-If you encounter any errors, please open an issue on THIS repo, and we will work to address it.
-
-
-## Handling Authentication
-
-If you need to add authentication to your server, 
-please reference FastAPI's [security documentation](https://fastapi.tiangolo.com/tutorial/security/)
-and [middleware documentation](https://fastapi.tiangolo.com/tutorial/middleware/).
+If you encounter any errors, please open an issue on THIS repo, and we will work to
+address it.
 
 ## Deployment
 
+### Deploy to AWS
+
+You can deploy to AWS using the [AWS Copilot CLI](https://aws.github.io/copilot-cli/)
+
+```bash
+copilot init --app [application-name] --name [service-name] --type 'Load Balanced Web Service' --dockerfile './Dockerfile' --deploy
+```
+
+Click [here](https://aws.amazon.com/containers/copilot/) to learn more.
+
+### Deploy to Azure
+
+You can deploy to Azure using Azure Container Apps (Serverless):
+
+```
+az containerapp up --name [container-app-name] --source . --resource-group [resource-group-name] --environment  [environment-name] --ingress external --target-port 8001 --env-vars=OPENAI_API_KEY=your_key
+```
+
+You can find more
+info [here](https://learn.microsoft.com/en-us/azure/container-apps/containerapp-up)
+
 ### Deploy to GCP
 
 You can deploy to GCP Cloud Run using the following command:
@@ -228,3 +435,374 @@ You can deploy to GCP Cloud Run using the following command:
 ```
 gcloud run deploy [your-service-name] --source . --port 8001 --allow-unauthenticated --region us-central1 --set-env-vars=OPENAI_API_KEY=your_key
 ```
+
+### Deploy using Infrastructure as Code
+
+#### Pulumi
+
+You can deploy your LangServe server with [Pulumi](https://www.pulumi.com/) using your preferred general purpose language. Below are some quickstart 
+examples for deploying LangServe to different cloud providers.
+
+These examples are a good starting point for your own infrastructure as code (IaC) projects. You can easily modify them to suit your needs.
+
+
+| Cloud | Language   | Repository                                          | Quickstart                                                                                                                                  |
+|-------|------------|-----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------|
+| AWS   | dotnet     | https://github.com/pulumi/examples/aws-cs-langserve | [![Deploy](https://get.pulumi.com/new/button.svg)](https://app.pulumi.com/new?template=https://github.com/pulumi/examples/aws-cs-langserve) |
+| AWS   | golang     | https://github.com/pulumi/examples/aws-go-langserve | [![Deploy](https://get.pulumi.com/new/button.svg)](https://app.pulumi.com/new?template=https://github.com/pulumi/examples/aws-go-langserve) |
+| AWS   | python     | https://github.com/pulumi/examples/aws-py-langserve | [![Deploy](https://get.pulumi.com/new/button.svg)](https://app.pulumi.com/new?template=https://github.com/pulumi/examples/aws-py-langserve) |
+| AWS   | typescript | https://github.com/pulumi/examples/aws-ts-langserve | [![Deploy](https://get.pulumi.com/new/button.svg)](https://app.pulumi.com/new?template=https://github.com/pulumi/examples/aws-ts-langserve) |
+| AWS   | javascript | https://github.com/pulumi/examples/aws-js-langserve | [![Deploy](https://get.pulumi.com/new/button.svg)](https://app.pulumi.com/new?template=https://github.com/pulumi/examples/aws-js-langserve) |
+
+
+
+
+### Community Contributed
+
+#### Deploy to Railway
+
+[Example Railway Repo](https://github.com/PaulLockett/LangServe-Railway/tree/main)
+
+[![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/template/pW9tXP?referralCode=c-aq4K)
+
+## Pydantic
+
+LangServe provides support for Pydantic 2 with some limitations.
+
+1. OpenAPI docs will not be generated for invoke/batch/stream/stream_log when using
+   Pydantic V2. Fast API does not support [mixing pydantic v1 and v2 namespaces].
+2. LangChain uses the v1 namespace in Pydantic v2. Please read
+   the [following guidelines to ensure compatibility with LangChain](https://github.com/langchain-ai/langchain/discussions/9337)
+
+Except for these limitations, we expect the API endpoints, the playground and any other
+features to work as expected.
+
+## Advanced
+
+### Handling Authentication
+
+If you need to add authentication to your server, please read Fast API's documentation
+about [dependencies](https://fastapi.tiangolo.com/tutorial/dependencies/)
+and [security](https://fastapi.tiangolo.com/tutorial/security/).
+
+The below examples show how to wire up authentication logic LangServe endpoints using FastAPI primitives.
+
+You are responsible for providing the actual authentication logic, the users table etc. 
+
+If you're not sure what you're doing, you could try using an existing solution [Auth0](https://auth0.com/).
+
+#### Using add_routes
+
+If you're using `add_routes`, see
+examples [here](https://github.com/langchain-ai/langserve/tree/main/examples/auth).
+
+| Description                                                                                                                                                                                                                                                        | Links                                                                                                                                                                                                                                |
+|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| **Auth** with `add_routes`: Simple authentication that can be applied across all endpoints associated with app. (Not useful on its own for implementing per user logic.)                                                                                           | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/global_deps/server.py)                                                                                                                                    | 
+| **Auth** with `add_routes`: Simple authentication mechanism based on path dependencies. (No useful on its own for implementing per user logic.)                                                                                                                    | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/path_dependencies/server.py)                                                                                                                              | 
+| **Auth** with `add_routes`: Implement per user logic and auth for endpoints that use per request config modifier. (**Note**: At the moment, does not integrate with OpenAPI docs.)                                                                                 | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/per_req_config_modifier/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/auth/per_req_config_modifier/client.ipynb)     | 
+
+Alternatively, you can use FastAPI's [middleware](https://fastapi.tiangolo.com/tutorial/middleware/).
+
+Using global dependencies and path dependencies has the advantage that auth will be properly supported in the OpenAPI docs page, but
+these are not sufficient for implement per user logic (e.g., making an application that can search only within user owned documents).
+
+If you need to implement per user logic, you can use the `per_req_config_modifier` or `APIHandler` (below) to implement this logic.
+
+**Per User**
+
+If you need authorization or logic that is user dependent,
+specify `per_req_config_modifier` when using `add_routes`. Use a callable receives the
+raw `Request` object and can extract relevant information from it for authentication and
+authorization purposes.
+
+#### Using APIHandler
+
+If you feel comfortable with FastAPI and python, you can use LangServe's [APIHandler](https://github.com/langchain-ai/langserve/blob/main/examples/api_handler_examples/server.py).
+
+| Description                                                                                                                                                                                                                                                        | Links                                                                                                                                                                                                                                |
+|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| **Auth** with `APIHandler`: Implement per user logic and auth that shows how to search only within user owned documents.                                                                                                                                           | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/api_handler/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/auth/api_handler/client.ipynb)                             | 
+| **APIHandler** Shows how to use `APIHandler` instead of `add_routes`. This provides more flexibility for developers to define endpoints. Works well with all FastAPI patterns, but takes a bit more effort.                                                        | [server](https://github.com/langchain-ai/langserve/tree/main/examples/api_handler_examples/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/api_handler_examples/client.ipynb)                     |
+
+It's a bit more work, but gives you complete control over the endpoint definitions, so
+you can do whatever custom logic you need for auth.
+
+### Files
+
+LLM applications often deal with files. There are different architectures
+that can be made to implement file processing; at a high level:
+
+1. The file may be uploaded to the server via a dedicated endpoint and processed using a
+   separate endpoint
+2. The file may be uploaded by either value (bytes of file) or reference (e.g., s3 url
+   to file content)
+3. The processing endpoint may be blocking or non-blocking
+4. If significant processing is required, the processing may be offloaded to a dedicated
+   process pool
+
+You should determine what is the appropriate architecture for your application.
+
+Currently, to upload files by value to a runnable, use base64 encoding for the
+file (`multipart/form-data` is not supported yet).
+
+Here's
+an [example](https://github.com/langchain-ai/langserve/tree/main/examples/file_processing)
+that shows
+how to use base64 encoding to send a file to a remote runnable.
+
+Remember, you can always upload files by reference (e.g., s3 url) or upload them as
+multipart/form-data to a dedicated endpoint.
+
+### Custom Input and Output Types
+
+Input and Output types are defined on all runnables.
+
+You can access them via the `input_schema` and `output_schema` properties.
+
+`LangServe` uses these types for validation and documentation.
+
+If you want to override the default inferred types, you can use the `with_types` method.
+
+Here's a toy example to illustrate the idea:
+
+```python
+from typing import Any
+
+from fastapi import FastAPI
+from langchain.schema.runnable import RunnableLambda
+
+app = FastAPI()
+
+
+def func(x: Any) -> int:
+    """Mistyped function that should accept an int but accepts anything."""
+    return x + 1
+
+
+runnable = RunnableLambda(func).with_types(
+    input_type=int,
+)
+
+add_routes(app, runnable)
+```
+
+### Custom User Types
+
+Inherit from `CustomUserType` if you want the data to de-serialize into a
+pydantic model rather than the equivalent dict representation.
+
+At the moment, this type only works *server* side and is used
+to specify desired *decoding* behavior. If inheriting from this type
+the server will keep the decoded type as a pydantic model instead
+of converting it into a dict.
+
+```python
+from fastapi import FastAPI
+from langchain.schema.runnable import RunnableLambda
+
+from langserve import add_routes
+from langserve.schema import CustomUserType
+
+app = FastAPI()
+
+
+class Foo(CustomUserType):
+    bar: int
+
+
+def func(foo: Foo) -> int:
+    """Sample function that expects a Foo type which is a pydantic model"""
+    assert isinstance(foo, Foo)
+    return foo.bar
+
+
+# Note that the input and output type are automatically inferred!
+# You do not need to specify them.
+# runnable = RunnableLambda(func).with_types( # <-- Not needed in this case
+#     input_type=Foo,
+#     output_type=int,
+#
+add_routes(app, RunnableLambda(func), path="/foo")
+```
+
+### Playground Widgets
+
+The playground allows you to define custom widgets for your runnable from the backend.
+
+Here are a few examples:
+
+| Description                                                                                                                                                                                                                                                        | Links                                                                                                                                                                                                                                |
+|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| **Widgets** Different widgets that can be used with playground (file upload and chat)                                                                                                                                                                              | [server](https://github.com/langchain-ai/langserve/tree/main/examples/widgets/chat/tuples/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/widgets/client.ipynb)                                               | 
+| **Widgets** File upload widget used for LangServe playground.                                                                                                                                                                                                      | [server](https://github.com/langchain-ai/langserve/tree/main/examples/file_processing/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/file_processing/client.ipynb)                               | 
+
+#### Schema
+
+- A widget is specified at the field level and shipped as part of the JSON schema of the
+  input type
+- A widget must contain a key called `type` with the value being one of a well known
+  list of widgets
+- Other widget keys will be associated with values that describe paths in a JSON object
+
+```typescript
+type JsonPath = number | string | (number | string)[];
+type NameSpacedPath = { title: string; path: JsonPath }; // Using title to mimick json schema, but can use namespace
+type OneOfPath = { oneOf: JsonPath[] };
+
+type Widget = {
+    type: string // Some well known type (e.g., base64file, chat etc.)
+    [key: string]: JsonPath | NameSpacedPath | OneOfPath;
+};
+```
+
+### Available Widgets
+
+There are only two widgets that the user can specify manually right now:
+
+1. File Upload Widget
+2. Chat History Widget
+
+See below more information about these widgets.
+
+All other widgets on the playground UI are created and managed automatically by the UI
+based on the config schema of the Runnable. When you create Configurable Runnables,
+the playground should create appropriate widgets for you to control the behavior.
+
+#### File Upload Widget
+
+Allows creation of a file upload input in the UI playground for files
+that are uploaded as base64 encoded strings. Here's the
+full [example](https://github.com/langchain-ai/langserve/tree/main/examples/file_processing).
+
+Snippet:
+
+```python
+try:
+    from pydantic.v1 import Field
+except ImportError:
+    from pydantic import Field
+
+from langserve import CustomUserType
+
+
+# ATTENTION: Inherit from CustomUserType instead of BaseModel otherwise
+#            the server will decode it into a dict instead of a pydantic model.
+class FileProcessingRequest(CustomUserType):
+    """Request including a base64 encoded file."""
+
+    # The extra field is used to specify a widget for the playground UI.
+    file: str = Field(..., extra={"widget": {"type": "base64file"}})
+    num_chars: int = 100
+
+```
+
+Example widget:
+
+<p align="center">
+<img src="https://github.com/langchain-ai/langserve/assets/3205522/52199e46-9464-4c2e-8be8-222250e08c3f" width="50%"/>
+</p>
+
+### Chat Widget
+
+Look
+at the [widget example](https://github.com/langchain-ai/langserve/tree/main/examples/widgets/chat/tuples/server.py).
+
+To define a chat widget, make sure that you pass "type": "chat".
+
+* "input" is JSONPath to the field in the *Request* that has the new input message.
+* "output" is JSONPath to the field in the *Response* that has new output message(s).
+* Don't specify these fields if the entire input or output should be used as they are (
+  e.g., if the output is a list of chat messages.)
+
+Here's a snippet:
+
+```python
+class ChatHistory(CustomUserType):
+    chat_history: List[Tuple[str, str]] = Field(
+        ...,
+        examples=[[("human input", "ai response")]],
+        extra={"widget": {"type": "chat", "input": "question", "output": "answer"}},
+    )
+    question: str
+
+
+def _format_to_messages(input: ChatHistory) -> List[BaseMessage]:
+    """Format the input to a list of messages."""
+    history = input.chat_history
+    user_input = input.question
+
+    messages = []
+
+    for human, ai in history:
+        messages.append(HumanMessage(content=human))
+        messages.append(AIMessage(content=ai))
+    messages.append(HumanMessage(content=user_input))
+    return messages
+
+
+model = ChatOpenAI()
+chat_model = RunnableParallel({"answer": (RunnableLambda(_format_to_messages) | model)})
+add_routes(
+    app,
+    chat_model.with_types(input_type=ChatHistory),
+    config_keys=["configurable"],
+    path="/chat",
+)
+```
+
+Example widget:
+<p align="center">
+<img src="https://github.com/langchain-ai/langserve/assets/3205522/a71ff37b-a6a9-4857-a376-cf27c41d3ca4" width="50%"/>
+</p>
+
+You can also specify a list of messages as your a parameter directly, as shown in this snippet:
+
+```python
+prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", "You are a helpful assisstant named Cob."),
+        MessagesPlaceholder(variable_name="messages"),
+    ]
+)
+
+chain = prompt | ChatAnthropic(model="claude-2")
+
+
+class MessageListInput(BaseModel):
+    """Input for the chat endpoint."""
+    messages: List[Union[HumanMessage, AIMessage]] = Field(
+        ...,
+        description="The chat messages representing the current conversation.",
+        extra={"widget": {"type": "chat", "input": "messages"}},
+    )
+
+
+add_routes(
+    app,
+    chain.with_types(input_type=MessageListInput),
+    path="/chat",
+)
+```
+
+See [this sample file](https://github.com/langchain-ai/langserve/tree/main/examples/widgets/chat/message_list/server.py) for an example.
+
+### Enabling / Disabling Endpoints (LangServe >=0.0.33)
+
+You can enable / disable which endpoints are exposed when adding routes for a given chain.
+
+Use `enabled_endpoints` if you want to make sure to never get a new endpoint when upgrading langserve to a newer
+verison.
+
+Enable: The code below will only enable `invoke`, `batch` and the
+corresponding `config_hash` endpoint variants.
+
+```python
+add_routes(app, chain, enabled_endpoints=["invoke", "batch", "config_hashes"], path="/mychain")
+```
+
+Disable: The code below will disable the playground for the chain
+
+```python
+add_routes(app, chain, disabled_endpoints=["playground"], path="/mychain")
+```

examples/agent/client.ipynb

@@ -18,16 +18,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
-   "metadata": {},
+   "execution_count": 1,
+   "metadata": {
+    "tags": []
+   },
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "{'output': {'output': 'Eugene thinks that cats like fish.'}}"
+       "{'output': {'output': 'Eugene thinks that cats like fish.'},\n",
+       " 'callback_events': [],\n",
+       " 'metadata': {'run_id': 'f16d95e5-dd8f-48d1-8668-4b33a54023fb'}}"
       ]
      },
-     "execution_count": 7,
+     "execution_count": 1,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -50,7 +54,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 2,
    "metadata": {
     "tags": []
    },
@@ -70,7 +74,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 3,
    "metadata": {
     "tags": []
    },
@@ -81,7 +85,7 @@
        "{'output': 'Hello! How can I assist you today?'}"
       ]
      },
-     "execution_count": 4,
+     "execution_count": 3,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -92,7 +96,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 4,
    "metadata": {
     "tags": []
    },
@@ -103,7 +107,7 @@
        "{'output': 'Eugene thinks that cats like fish.'}"
       ]
      },
-     "execution_count": 5,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -113,11 +117,814 @@
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
+   "cell_type": "markdown",
    "metadata": {},
-   "outputs": [],
-   "source": []
+   "source": [
+    "## Stream\n",
+    "\n",
+    "Please note that streaming alternates between actions and observations. It does not stream individual tokens! If you need to stream individual tokens you will need to use astream_log!"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "--\n",
+      "{'actions': [AgentActionMessageLog(tool='get_eugene_thoughts', tool_input={'query': 'cats'}, log=\"\\nInvoking: `get_eugene_thoughts` with `{'query': 'cats'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})])], 'messages': [AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})]}\n",
+      "--\n",
+      "{'steps': [{'action': AgentActionMessageLog(tool='get_eugene_thoughts', tool_input={'query': 'cats'}, log=\"\\nInvoking: `get_eugene_thoughts` with `{'query': 'cats'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})]), 'observation': [Document(page_content='cats like fish'), Document(page_content='dogs like sticks')]}], 'messages': [FunctionMessage(content=\"[Document(page_content='cats like fish'), Document(page_content='dogs like sticks')]\", name='get_eugene_thoughts')]}\n",
+      "--\n",
+      "{'output': \"Eugene thinks that cats like fish. Now let me tell you a story about that thought.\\n\\nOnce upon a time, in a small village, there lived a mischievous cat named Whiskers. Whiskers was known for his love for fish. Every day, he would venture out to the nearby river in search of his favorite food.\\n\\nOne sunny day, Whiskers set out on his usual fish-hunting expedition. As he approached the riverbank, he noticed a group of fishermen casting their nets into the water. Whiskers couldn't resist the temptation and decided to join in on the action.\\n\\nWith his agile paws, Whiskers skillfully maneuvered through the fishermen, strategically positioning himself to snatch the fish caught in their nets. The fishermen were amazed by Whiskers' quick reflexes and couldn't help but laugh at the sight of a cat fishing alongside them.\\n\\nWhiskers continued his fish-stealing escapades for several days, becoming somewhat of a local legend. People from neighboring villages would come to witness the incredible sight of a cat outsmarting seasoned fishermen.\\n\\nOne day, as Whiskers was enjoying his stolen fish by the river, he noticed a stray dog named Rover approaching him. Rover had heard about Whiskers' fishing talents and was intrigued by the cat's abilities.\\n\\nCurious, Whiskers decided to share his secret with Rover. He taught him the art of stealth and how to snatch fish from the nets without being noticed. Rover, being a quick learner, soon became Whiskers' partner in crime.\\n\\nTogether, Whiskers and Rover formed an unbeatable duo, leaving the fishermen puzzled and amazed at the disappearing fish. The villagers, entertained by their antics, started leaving fish out for Whiskers and Rover as a token of appreciation.\\n\\nAnd so, Whiskers and Rover continued their fish-stealing adventures, bringing joy and laughter to the village. Eugene's thought about cats and their love for fish certainly came to life in the mischievous and clever Whiskers, who proved that cats truly have a special affinity for fish.\", 'messages': [AIMessage(content=\"Eugene thinks that cats like fish. Now let me tell you a story about that thought.\\n\\nOnce upon a time, in a small village, there lived a mischievous cat named Whiskers. Whiskers was known for his love for fish. Every day, he would venture out to the nearby river in search of his favorite food.\\n\\nOne sunny day, Whiskers set out on his usual fish-hunting expedition. As he approached the riverbank, he noticed a group of fishermen casting their nets into the water. Whiskers couldn't resist the temptation and decided to join in on the action.\\n\\nWith his agile paws, Whiskers skillfully maneuvered through the fishermen, strategically positioning himself to snatch the fish caught in their nets. The fishermen were amazed by Whiskers' quick reflexes and couldn't help but laugh at the sight of a cat fishing alongside them.\\n\\nWhiskers continued his fish-stealing escapades for several days, becoming somewhat of a local legend. People from neighboring villages would come to witness the incredible sight of a cat outsmarting seasoned fishermen.\\n\\nOne day, as Whiskers was enjoying his stolen fish by the river, he noticed a stray dog named Rover approaching him. Rover had heard about Whiskers' fishing talents and was intrigued by the cat's abilities.\\n\\nCurious, Whiskers decided to share his secret with Rover. He taught him the art of stealth and how to snatch fish from the nets without being noticed. Rover, being a quick learner, soon became Whiskers' partner in crime.\\n\\nTogether, Whiskers and Rover formed an unbeatable duo, leaving the fishermen puzzled and amazed at the disappearing fish. The villagers, entertained by their antics, started leaving fish out for Whiskers and Rover as a token of appreciation.\\n\\nAnd so, Whiskers and Rover continued their fish-stealing adventures, bringing joy and laughter to the village. Eugene's thought about cats and their love for fish certainly came to life in the mischievous and clever Whiskers, who proved that cats truly have a special affinity for fish.\")]}\n"
+     ]
+    }
+   ],
+   "source": [
+    "async for chunk in remote_runnable.astream({\"input\": \"what does eugene think of cats? Then tell me a story about that thought.\"}):\n",
+    "    print('--')\n",
+    "    print(chunk)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Stream Events\n",
+    "\n",
+    "The client is looking for a runnable name called `agent` for the chain events. This name was defined on the server side using `runnable.with_config({\"run_name\": \"agent\"}`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Starting agent: agent with input: {'input': 'what does eugene think of cats? Then tell me a story about that thought.'}\n",
+      "--\n",
+      "Starting tool: get_eugene_thoughts with inputs: {'query': 'cats'}\n",
+      "Done tool: get_eugene_thoughts\n",
+      "Tool output was: [Document(page_content='cats like fish'), Document(page_content='dogs like sticks')]\n",
+      "--\n",
+      "E|ug|ene| thinks| that| cats| like| fish|.| Now| let| me| tell| you| a| story| about| that| thought|:\n",
+      "\n",
+      "|Once| upon| a| time|,| in| a| small| village|,| there| lived| a| curious| cat| named| Wh|isk|ers|.| Wh|isk|ers| was| known| for| his| love| of| fish|.| Every| day|,| he| would| venture| to| the| nearby| river| in| search| of| his| favorite| meal|.\n",
+      "\n",
+      "|One| sunny| morning|,| Wh|isk|ers| set| out| on| his| daily| fish|-h|unting| expedition|.| As| he| approached| the| river|,| he| could| smell| the| fresh| scent| of| the| water| and| feel| the| excitement| building| up| inside| him|.| Wh|isk|ers| knew| that| today| might| be| his| lucky| day|.\n",
+      "\n",
+      "|He| carefully| ti|pto|ed| along| the| river|bank|,| his| eyes| fixed| on| the| water|'s| surface|.| Suddenly|,| he| spotted| a| shimmer|ing| fish| swimming| gracefully| through| the| clear| blue| water|.| Wh|isk|ers| c|rou|ched| low|,| ready| to| p|ounce|.\n",
+      "\n",
+      "|With| lightning| speed|,| he| le|aped| into| the| air|,| his| p|aws| out|st|retched| towards| the| fish|.| Splash|!| Wh|isk|ers| landed| right| in| the| middle| of| the| river|,| causing| r|ipples| to| spread| in| all| directions|.| But| he| didn|'t| care|.| All| he| wanted| was| that| delicious| fish|.\n",
+      "\n",
+      "|Wh|isk|ers| chased| the| fish| with| all| his| might|,| dart|ing| through| the| water| with| elegance| and| precision|.| The| fish| sw|am| gracefully|,| trying| to| escape| Wh|isk|ers|'| determined| pursuit|.| But| the| cat| was| relentless|.\n",
+      "\n",
+      "|After| a| few| moments| of| intense| chase|,| Wh|isk|ers| finally| managed| to| catch| the| fish| in| his| p|aws|.| He| triumph|antly| carried| it| to| the| river|bank|,| where| he| enjoyed| his| well|-des|erved| feast|.| The| taste| of| the| fresh| fish| was| heavenly|,| satisfying| his| hunger| and| bringing| a| content|ed| smile| to| his| face|.\n",
+      "\n",
+      "|From| that| day| on|,| Wh|isk|ers| became| known| as| the| legendary| fish|-catching| cat| in| the| village|.| People| would| often| gather| by| the| river| to| watch| him| in| action|,| amazed| by| his| agility| and| determination|.| Wh|isk|ers| taught| everyone| the| importance| of| perseverance| and| following| one|'s| passion|,| just| like| he| pursued| his| love| for| fish|.\n",
+      "\n",
+      "|And| so|,| Wh|isk|ers| continued| his| fish|-h|unting| adventures|,| spreading| joy| and| inspiration| to| everyone| he| encountered|.| He| proved| that| when| you| have| a| passion| for| something|,| nothing| can| stop| you| from| achieving| it| –| just| like| cats| and| their| love| for| fish|.\n",
+      "\n",
+      "|The| end|.|\n",
+      "--\n",
+      "Done agent: agent with output: Eugene thinks that cats like fish. Now let me tell you a story about that thought:\n",
+      "\n",
+      "Once upon a time, in a small village, there lived a curious cat named Whiskers. Whiskers was known for his love of fish. Every day, he would venture to the nearby river in search of his favorite meal.\n",
+      "\n",
+      "One sunny morning, Whiskers set out on his daily fish-hunting expedition. As he approached the river, he could smell the fresh scent of the water and feel the excitement building up inside him. Whiskers knew that today might be his lucky day.\n",
+      "\n",
+      "He carefully tiptoed along the riverbank, his eyes fixed on the water's surface. Suddenly, he spotted a shimmering fish swimming gracefully through the clear blue water. Whiskers crouched low, ready to pounce.\n",
+      "\n",
+      "With lightning speed, he leaped into the air, his paws outstretched towards the fish. Splash! Whiskers landed right in the middle of the river, causing ripples to spread in all directions. But he didn't care. All he wanted was that delicious fish.\n",
+      "\n",
+      "Whiskers chased the fish with all his might, darting through the water with elegance and precision. The fish swam gracefully, trying to escape Whiskers' determined pursuit. But the cat was relentless.\n",
+      "\n",
+      "After a few moments of intense chase, Whiskers finally managed to catch the fish in his paws. He triumphantly carried it to the riverbank, where he enjoyed his well-deserved feast. The taste of the fresh fish was heavenly, satisfying his hunger and bringing a contented smile to his face.\n",
+      "\n",
+      "From that day on, Whiskers became known as the legendary fish-catching cat in the village. People would often gather by the river to watch him in action, amazed by his agility and determination. Whiskers taught everyone the importance of perseverance and following one's passion, just like he pursued his love for fish.\n",
+      "\n",
+      "And so, Whiskers continued his fish-hunting adventures, spreading joy and inspiration to everyone he encountered. He proved that when you have a passion for something, nothing can stop you from achieving it – just like cats and their love for fish.\n",
+      "\n",
+      "The end.\n"
+     ]
+    }
+   ],
+   "source": [
+    "async for event in remote_runnable.astream_events(\n",
+    "    {\"input\": \"what does eugene think of cats? Then tell me a story about that thought.\"},\n",
+    "    version=\"v1\",\n",
+    "):\n",
+    "    kind = event[\"event\"]\n",
+    "    if kind == \"on_chain_start\":\n",
+    "        if (\n",
+    "            event[\"name\"] == \"agent\"\n",
+    "        ):  # Was assigned when creating the agent with `.with_config({\"run_name\": \"Agent\"})`\n",
+    "            print(\n",
+    "                f\"Starting agent: {event['name']} with input: {event['data'].get('input')}\"\n",
+    "            )\n",
+    "    elif kind == \"on_chain_end\":\n",
+    "        if (\n",
+    "            event[\"name\"] == \"agent\"\n",
+    "        ):  # Was assigned when creating the agent with `.with_config({\"run_name\": \"Agent\"})`\n",
+    "            print()\n",
+    "            print(\"--\")\n",
+    "            print(\n",
+    "                f\"Done agent: {event['name']} with output: {event['data'].get('output')['output']}\"\n",
+    "            )\n",
+    "    if kind == \"on_chat_model_stream\":\n",
+    "        content = event[\"data\"][\"chunk\"].content\n",
+    "        if content:\n",
+    "            # Empty content in the context of OpenAI means\n",
+    "            # that the model is asking for a tool to be invoked.\n",
+    "            # So we only print non-empty content\n",
+    "            print(content, end=\"|\")\n",
+    "    elif kind == \"on_tool_start\":\n",
+    "        print(\"--\")\n",
+    "        print(\n",
+    "            f\"Starting tool: {event['name']} with inputs: {event['data'].get('input')}\"\n",
+    "        )\n",
+    "    elif kind == \"on_tool_end\":\n",
+    "        print(f\"Done tool: {event['name']}\")\n",
+    "        print(f\"Tool output was: {event['data'].get('output')}\")\n",
+    "        print(\"--\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Stream log\n",
+    "\n",
+    "If you need acccess the individual llm tokens from an agent use `astream_log`. Please make sure that you set **streaming=True** on your LLM (see server code). For this to work, the LLM must also support streaming!"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "--\n",
+      "RunLogPatch({'op': 'replace',\n",
+      "  'path': '',\n",
+      "  'value': {'final_output': None,\n",
+      "            'id': '1213bf75-c5ff-401a-b3c5-b4663ec7dfca',\n",
+      "            'logs': {},\n",
+      "            'streamed_output': []}})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableSequence',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '7629fc73-a2fd-48fa-b9a2-c336e270180d',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'RunnableSequence',\n",
+      "            'start_time': '2024-01-05T22:24:50.737+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': [],\n",
+      "            'type': 'chain'}})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableParallel<input,agent_scratchpad>',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '72d2e143-d977-4661-b901-430cae1ee739',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'RunnableParallel<input,agent_scratchpad>',\n",
+      "            'start_time': '2024-01-05T22:24:50.738+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:1'],\n",
+      "            'type': 'chain'}})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/<lambda>',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '7ec78a86-596f-4904-9888-45ef67302cb8',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': '<lambda>',\n",
+      "            'start_time': '2024-01-05T22:24:50.739+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['map:key:input'],\n",
+      "            'type': 'chain'}})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/<lambda>:2',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '281ca194-3e0c-490c-b637-759f5d971769',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': '<lambda>',\n",
+      "            'start_time': '2024-01-05T22:24:50.739+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['map:key:agent_scratchpad'],\n",
+      "            'type': 'chain'}})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/<lambda>/final_output',\n",
+      "  'value': {'output': 'what does eugene think of cats?'}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/<lambda>/end_time',\n",
+      "  'value': '2024-01-05T22:24:50.740+00:00'})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/<lambda>:2/final_output',\n",
+      "  'value': {'output': []}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/<lambda>:2/end_time',\n",
+      "  'value': '2024-01-05T22:24:50.740+00:00'})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableParallel<input,agent_scratchpad>/final_output',\n",
+      "  'value': {'agent_scratchpad': [],\n",
+      "            'input': 'what does eugene think of cats?'}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/RunnableParallel<input,agent_scratchpad>/end_time',\n",
+      "  'value': '2024-01-05T22:24:50.740+00:00'})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatPromptTemplate',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '68c6ad9d-b9f0-47e0-9ea7-d0f170a22732',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'ChatPromptTemplate',\n",
+      "            'start_time': '2024-01-05T22:24:50.741+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:2'],\n",
+      "            'type': 'prompt'}})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatPromptTemplate/final_output',\n",
+      "  'value': {'messages': [SystemMessage(content='You are a helpful assistant.'),\n",
+      "                         HumanMessage(content='what does eugene think of cats?')]}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatPromptTemplate/end_time',\n",
+      "  'value': '2024-01-05T22:24:50.741+00:00'})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '883e05a9-6b29-4494-92b3-51c72973cb54',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'ChatOpenAI',\n",
+      "            'start_time': '2024-01-05T22:24:50.743+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:3'],\n",
+      "            'type': 'llm'}})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/ChatOpenAI/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': ''}})})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/ChatOpenAI/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': '{\\n'}})})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/ChatOpenAI/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': ' '}})})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/ChatOpenAI/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': ' \"'}})})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/ChatOpenAI/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': 'query'}})})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/ChatOpenAI/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': '\":'}})})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/ChatOpenAI/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': ' \"'}})})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/ChatOpenAI/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': 'cats'}})})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/ChatOpenAI/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': '\"\\n'}})})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/ChatOpenAI/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': '}'}})})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/ChatOpenAI/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/final_output',\n",
+      "  'value': LLMResult(generations=[[ChatGeneration(generation_info={'finish_reason': 'function_call'}, message=AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}}))]], llm_output=None, run=None)},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/end_time',\n",
+      "  'value': '2024-01-05T22:24:51.497+00:00'})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/OpenAIFunctionsAgentOutputParser',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': 'f8052926-8eb1-4df1-a701-309ee48fd63c',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'OpenAIFunctionsAgentOutputParser',\n",
+      "            'start_time': '2024-01-05T22:24:51.498+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:4'],\n",
+      "            'type': 'parser'}})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/OpenAIFunctionsAgentOutputParser/final_output',\n",
+      "  'value': AgentActionMessageLog(tool='get_eugene_thoughts', tool_input={'query': 'cats'}, log=\"\\nInvoking: `get_eugene_thoughts` with `{'query': 'cats'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})])},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/OpenAIFunctionsAgentOutputParser/end_time',\n",
+      "  'value': '2024-01-05T22:24:51.499+00:00'})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableSequence/final_output',\n",
+      "  'value': {'output': AgentActionMessageLog(tool='get_eugene_thoughts', tool_input={'query': 'cats'}, log=\"\\nInvoking: `get_eugene_thoughts` with `{'query': 'cats'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})])}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/RunnableSequence/end_time',\n",
+      "  'value': '2024-01-05T22:24:51.500+00:00'})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': {'actions': [AgentActionMessageLog(tool='get_eugene_thoughts', tool_input={'query': 'cats'}, log=\"\\nInvoking: `get_eugene_thoughts` with `{'query': 'cats'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})])],\n",
+      "            'messages': [AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})]}},\n",
+      " {'op': 'replace',\n",
+      "  'path': '/final_output',\n",
+      "  'value': {'actions': [AgentActionMessageLog(tool='get_eugene_thoughts', tool_input={'query': 'cats'}, log=\"\\nInvoking: `get_eugene_thoughts` with `{'query': 'cats'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})])],\n",
+      "            'messages': [AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})]}})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/get_eugene_thoughts',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': 'e745d7ec-5992-4ab8-a3ef-191342c6470c',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'get_eugene_thoughts',\n",
+      "            'start_time': '2024-01-05T22:24:51.501+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': [],\n",
+      "            'type': 'tool'}})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/get_eugene_thoughts/final_output',\n",
+      "  'value': {'output': \"[Document(page_content='cats like fish'), \"\n",
+      "                      \"Document(page_content='dogs like sticks')]\"}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/get_eugene_thoughts/end_time',\n",
+      "  'value': '2024-01-05T22:24:51.737+00:00'})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': {'messages': [FunctionMessage(content=\"[Document(page_content='cats like fish'), Document(page_content='dogs like sticks')]\", name='get_eugene_thoughts')],\n",
+      "            'steps': [{'action': AgentActionMessageLog(tool='get_eugene_thoughts', tool_input={'query': 'cats'}, log=\"\\nInvoking: `get_eugene_thoughts` with `{'query': 'cats'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})]),\n",
+      "                       'observation': [Document(page_content='cats like fish'),\n",
+      "                                       Document(page_content='dogs like sticks')]}]}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/final_output/steps',\n",
+      "  'value': [{'action': AgentActionMessageLog(tool='get_eugene_thoughts', tool_input={'query': 'cats'}, log=\"\\nInvoking: `get_eugene_thoughts` with `{'query': 'cats'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})]),\n",
+      "             'observation': [Document(page_content='cats like fish'),\n",
+      "                             Document(page_content='dogs like sticks')]}]},\n",
+      " {'op': 'add',\n",
+      "  'path': '/final_output/messages/1',\n",
+      "  'value': FunctionMessage(content=\"[Document(page_content='cats like fish'), Document(page_content='dogs like sticks')]\", name='get_eugene_thoughts')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableSequence:2',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '15d4ae13-554b-4144-9aa8-05527a15c042',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'RunnableSequence',\n",
+      "            'start_time': '2024-01-05T22:24:51.739+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': [],\n",
+      "            'type': 'chain'}})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableParallel<input,agent_scratchpad>:2',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '207dab98-d50c-4f2c-934d-291ba3ffd953',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'RunnableParallel<input,agent_scratchpad>',\n",
+      "            'start_time': '2024-01-05T22:24:51.740+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:1'],\n",
+      "            'type': 'chain'}})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/<lambda>:3',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '8795a642-3f8e-4e5c-aec6-825b7aac2148',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': '<lambda>',\n",
+      "            'start_time': '2024-01-05T22:24:51.741+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['map:key:input'],\n",
+      "            'type': 'chain'}})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/<lambda>:4',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '60be3aac-3718-4386-a941-1dd371b9a3ea',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': '<lambda>',\n",
+      "            'start_time': '2024-01-05T22:24:51.741+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['map:key:agent_scratchpad'],\n",
+      "            'type': 'chain'}})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/<lambda>:3/final_output',\n",
+      "  'value': {'output': 'what does eugene think of cats?'}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/<lambda>:3/end_time',\n",
+      "  'value': '2024-01-05T22:24:51.742+00:00'})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/<lambda>:4/final_output',\n",
+      "  'value': {'output': [AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}}),\n",
+      "                       FunctionMessage(content=\"[Document(page_content='cats like fish'), Document(page_content='dogs like sticks')]\", name='get_eugene_thoughts')]}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/<lambda>:4/end_time',\n",
+      "  'value': '2024-01-05T22:24:51.742+00:00'})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableParallel<input,agent_scratchpad>:2/final_output',\n",
+      "  'value': {'agent_scratchpad': [AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}}),\n",
+      "                                 FunctionMessage(content=\"[Document(page_content='cats like fish'), Document(page_content='dogs like sticks')]\", name='get_eugene_thoughts')],\n",
+      "            'input': 'what does eugene think of cats?'}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/RunnableParallel<input,agent_scratchpad>:2/end_time',\n",
+      "  'value': '2024-01-05T22:24:51.742+00:00'})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatPromptTemplate:2',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': 'f1ff89b0-ae9c-49d1-8a2e-73c7f001f1e7',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'ChatPromptTemplate',\n",
+      "            'start_time': '2024-01-05T22:24:51.743+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:2'],\n",
+      "            'type': 'prompt'}})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatPromptTemplate:2/final_output',\n",
+      "  'value': {'messages': [SystemMessage(content='You are a helpful assistant.'),\n",
+      "                         HumanMessage(content='what does eugene think of cats?'),\n",
+      "                         AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}}),\n",
+      "                         FunctionMessage(content=\"[Document(page_content='cats like fish'), Document(page_content='dogs like sticks')]\", name='get_eugene_thoughts')]}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatPromptTemplate:2/end_time',\n",
+      "  'value': '2024-01-05T22:24:51.743+00:00'})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': 'c4aeb3d0-33c3-4f46-80e9-d1ffb0f983f2',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'ChatOpenAI',\n",
+      "            'start_time': '2024-01-05T22:24:51.744+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:3'],\n",
+      "            'type': 'llm'}})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/ChatOpenAI:2/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': 'E'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='E')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': 'ug'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='ug')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': 'ene'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='ene')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' has'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' has')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' two'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' two')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' thoughts'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' thoughts')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' on'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' on')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' cats'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' cats')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': '.'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='.')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' One'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' One')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' thought'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' thought')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' is'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' is')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' \"'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' \"')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': 'cats'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='cats')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' like'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' like')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' fish'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' fish')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': '\"'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='\"')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' and'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' and')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' the'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' the')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' other'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' other')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' thought'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' thought')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' is'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' is')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' \"'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' \"')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': 'dogs'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='dogs')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' like'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' like')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' sticks'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' sticks')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': '\".'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='\".')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/ChatOpenAI:2/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='')})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/final_output',\n",
+      "  'value': LLMResult(generations=[[ChatGeneration(text='Eugene has two thoughts on cats. One thought is \"cats like fish\" and the other thought is \"dogs like sticks\".', generation_info={'finish_reason': 'stop'}, message=AIMessage(content='Eugene has two thoughts on cats. One thought is \"cats like fish\" and the other thought is \"dogs like sticks\".'))]], llm_output=None, run=None)},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/end_time',\n",
+      "  'value': '2024-01-05T22:24:53.082+00:00'})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/OpenAIFunctionsAgentOutputParser:2',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '7074d8c2-df0b-4d18-9890-dd050c2f0ca9',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'OpenAIFunctionsAgentOutputParser',\n",
+      "            'start_time': '2024-01-05T22:24:53.083+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:4'],\n",
+      "            'type': 'parser'}})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/OpenAIFunctionsAgentOutputParser:2/final_output',\n",
+      "  'value': AgentFinish(return_values={'output': 'Eugene has two thoughts on cats. One thought is \"cats like fish\" and the other thought is \"dogs like sticks\".'}, log='Eugene has two thoughts on cats. One thought is \"cats like fish\" and the other thought is \"dogs like sticks\".')},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/OpenAIFunctionsAgentOutputParser:2/end_time',\n",
+      "  'value': '2024-01-05T22:24:53.084+00:00'})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableSequence:2/final_output',\n",
+      "  'value': {'output': AgentFinish(return_values={'output': 'Eugene has two thoughts on cats. One thought is \"cats like fish\" and the other thought is \"dogs like sticks\".'}, log='Eugene has two thoughts on cats. One thought is \"cats like fish\" and the other thought is \"dogs like sticks\".')}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/RunnableSequence:2/end_time',\n",
+      "  'value': '2024-01-05T22:24:53.084+00:00'})\n",
+      "--\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': {'messages': [AIMessage(content='Eugene has two thoughts on cats. One thought is \"cats like fish\" and the other thought is \"dogs like sticks\".')],\n",
+      "            'output': 'Eugene has two thoughts on cats. One thought is \"cats '\n",
+      "                      'like fish\" and the other thought is \"dogs like '\n",
+      "                      'sticks\".'}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/final_output/output',\n",
+      "  'value': 'Eugene has two thoughts on cats. One thought is \"cats like fish\" '\n",
+      "           'and the other thought is \"dogs like sticks\".'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/final_output/messages/2',\n",
+      "  'value': AIMessage(content='Eugene has two thoughts on cats. One thought is \"cats like fish\" and the other thought is \"dogs like sticks\".')})\n"
+     ]
+    }
+   ],
+   "source": [
+    "async for chunk in remote_runnable.astream_log({\"input\": \"what does eugene think of cats?\"}):\n",
+    "    print('--')\n",
+    "    print(chunk)"
+   ]
   }
  ],
  "metadata": {
@@ -136,7 +943,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.1"
+   "version": "3.10.8"
   }
  },
  "nbformat": 4,

examples/agent/server.py

@@ -1,5 +1,24 @@
 #!/usr/bin/env python
-"""Example LangChain server exposes a conversational retrieval chain."""
+"""Example LangChain server exposes a conversational retrieval agent.
+
+Relevant LangChain documentation:
+
+* Creating a custom agent: https://python.langchain.com/docs/modules/agents/how_to/custom_agent
+* Streaming with agents: https://python.langchain.com/docs/modules/agents/how_to/streaming#custom-streaming-with-events
+* General streaming documentation: https://python.langchain.com/docs/expression_language/streaming
+
+**ATTENTION**
+1. To support streaming individual tokens you will need to use the astream events
+   endpoint rather than the streaming endpoint.
+2. This example does not truncate message history, so it will crash if you
+   send too many messages (exceed token length).
+3. The playground at the moment does not render agent output well! If you want to
+   use the playground you need to customize it's output server side using astream
+   events by wrapping it within another runnable.
+4. See the client notebook it has an example of how to use stream_events client side!
+"""
+from typing import Any
+
 from fastapi import FastAPI
 from langchain.agents import AgentExecutor, tool
 from langchain.agents.format_scratchpad import format_to_openai_functions
@@ -7,9 +26,9 @@ from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser
 from langchain.chat_models import ChatOpenAI
 from langchain.embeddings import OpenAIEmbeddings
 from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain.pydantic_v1 import BaseModel
 from langchain.tools.render import format_tool_to_openai_function
 from langchain.vectorstores import FAISS
-from pydantic import BaseModel
 
 from langserve import add_routes
 
@@ -30,12 +49,22 @@ tools = [get_eugene_thoughts]
 prompt = ChatPromptTemplate.from_messages(
     [
         ("system", "You are a helpful assistant."),
+        # Please note that the ordering of the user input vs.
+        # the agent_scratchpad is important.
+        # The agent_scratchpad is a working space for the agent to think,
+        # invoke tools, see tools outputs in order to respond to the given
+        # user input. It has to come AFTER the user input.
         ("user", "{input}"),
         MessagesPlaceholder(variable_name="agent_scratchpad"),
     ]
 )
 
-llm = ChatOpenAI()
+# We need to set streaming=True on the LLM to support streaming individual tokens.
+# Tokens will be available when using the stream_log / stream events endpoints,
+# but not when using the stream endpoint since the stream implementation for agent
+# streams action observation pairs not individual tokens.
+# See the client notebook that shows how to use the stream events endpoint.
+llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0, streaming=True)
 
 llm_with_tools = llm.bind(functions=[format_tool_to_openai_function(t) for t in tools])
 
@@ -56,7 +85,7 @@ agent_executor = AgentExecutor(agent=agent, tools=tools)
 app = FastAPI(
     title="LangChain Server",
     version="1.0",
-    description="Spin up a simple api server using Langchain's Runnable interfaces",
+    description="Spin up a simple api server using LangChain's Runnable interfaces",
 )
 
 
@@ -67,14 +96,20 @@ class Input(BaseModel):
 
 
 class Output(BaseModel):
-    output: str
+    output: Any
 
 
 # Adds routes to the app for using the chain under:
 # /invoke
 # /batch
 # /stream
-add_routes(app, agent_executor, input_type=Input, output_type=Output)
+# /stream_events
+add_routes(
+    app,
+    agent_executor.with_types(input_type=Input, output_type=Output).with_config(
+        {"run_name": "agent"}
+    ),
+)
 
 if __name__ == "__main__":
     import uvicorn

examples/agent_custom_streaming/client.ipynb

@@ -0,0 +1,232 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Client\n",
+    "\n",
+    "Client code interacting with a server that implements: \n",
+    "\n",
+    "* custom streaming for an agent\n",
+    "* agent with user selected tools\n",
+    "\n",
+    "This agent does not have memory! See other examples in LangServe to see how to add memory.\n",
+    "\n",
+    "**ATTENTION** We made the agent stream strings as an output. This is almost certainly not what you would want for your application. Feel free to adapt to return more structured output; however, keep in mind that likely the client can just use `astream_events`!\n",
+    "\n",
+    "See relevant documentation about agents:\n",
+    "\n",
+    "* Creating a custom agent: https://python.langchain.com/docs/modules/agents/how_to/custom_agent\n",
+    "* Streaming with agents: https://python.langchain.com/docs/modules/agents/how_to/streaming#custom-streaming-with-events\n",
+    "* General streaming documentation: https://python.langchain.com/docs/expression_language/streaming"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can interact with this via API directly"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "16"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "word = \"audioeeeeeeeeeee\"\n",
+    "len(word)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Starting agent: agent with input: {'input': 'what is the length of the word audioeeeeeeeeeee?'}\n",
+      "The length of the word \"audioeeeeeeeeeee\" is 15 characters.\n",
+      "Done agent: agent with output: The length of the word \"audioeeeeeeeeeee\" is 15 characters.\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "import requests\n",
+    "\n",
+    "inputs = {\"input\": {\"input\": f\"what is the length of the word {word}?\", \"chat_history\": [], \"tools\": []}}\n",
+    "response = requests.post(\"http://localhost:8000/invoke\", json=inputs)\n",
+    "\n",
+    "print(response.json()['output'])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let's provide it with a tool to test that tool selection works"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Starting agent: agent with input: {'input': 'what is the length of the word audioeeeeeeeeeee?'}\n",
+      "\n",
+      "Starting tool: word_length with inputs: {'word': 'audioeeeeeeeeeee'}\n",
+      "\n",
+      "Done tool: word_length with output: 16\n",
+      "The length of the word \"audioeeeeeeeeeee\" is 16.\n",
+      "Done agent: agent with output: The length of the word \"audioeeeeeeeeeee\" is 16.\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "import requests\n",
+    "\n",
+    "inputs = {\"input\": {\"input\": f\"what is the length of the word {word}?\", \"chat_history\": [], \"tools\": [\"word_length\", \"favorite_animal\"]}}\n",
+    "response = requests.post(\"http://localhost:8000/invoke\", json=inputs)\n",
+    "\n",
+    "print(response.json()['output'])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can also interact with this via the RemoteRunnable interface (to use in other chains)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langserve import RemoteRunnable\n",
+    "\n",
+    "remote_runnable = RemoteRunnable(\"http://localhost:8000/\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Remote runnable has the same interface as local runnables"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Stream\n",
+    "\n",
+    "Streaming output from a **CUSTOM STREAMING** implementation that streams string representations of intermediate steps. Please see server side implementation for details."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "|Starting agent: agent with input: {'input': 'What is eugenes favorite animal?'}|\n",
+      "|I|'m| sorry|,| but| I| don|'t| have| access| to| personal| information| about| individuals| unless| it| has| been| shared| with| me| in| the| course| of| our| conversation|.|\n",
+      "|Done agent: agent with output: I'm sorry, but I don't have access to personal information about individuals unless it has been shared with me in the course of our conversation.|\n",
+      "|"
+     ]
+    }
+   ],
+   "source": [
+    "async for chunk in remote_runnable.astream({\"input\": \"What is eugenes favorite animal?\", \"tools\": [\"word_length\"]}):\n",
+    "    print(chunk, end='|', flush=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "|Starting agent: agent with input: {'input': 'What is eugenes favorite animal?'}|\n",
+      "|\n",
+      "|Starting tool: favorite_animal with inputs: {'name': 'Eugene'}|\n",
+      "|\n",
+      "|Done tool: favorite_animal with output: cat|\n",
+      "|E|ug|ene|'s| favorite| animal| is| a| cat|.|\n",
+      "|Done agent: agent with output: Eugene's favorite animal is a cat.|\n",
+      "|"
+     ]
+    }
+   ],
+   "source": [
+    "async for chunk in remote_runnable.astream({\"input\": \"What is eugenes favorite animal?\", \"tools\": [\"word_length\", \"favorite_animal\"]}):\n",
+    "    print(chunk, end='|', flush=True)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/agent_custom_streaming/server.py

@@ -0,0 +1,248 @@
+#!/usr/bin/env python
+"""Example LangChain server that shows how to customize streaming for an agent.
+
+Example uses a RunnableLambda that:
+
+1) Uses the agent's astream events method to create a custom streaming API endpoint.
+2) Instantiates an agent with custom tools (based on the user request).
+
+In this example, we kept things simple and are outputting strings to the client
+with all the intermediate steps of the agent. This is just for demonstration
+purposes, and usually you would want to return more structured output in the form
+of a dictionary.
+
+To add history to the agent you can use RunnableWithHistory. Please see the
+other examples in LangServe for how to use RunnableWithHistory to store history
+on the server side.
+
+Alternatively, you can keep track of history on the client side and send it to the
+server with each request. For that to work, you will definitely want to modify the
+streaming output to yield dictionaries with structured output, so it's easy
+to determine what the final agent output was on the client side.
+
+Customize the streaming output to your use case!
+
+Note that we configure the agent using the `tools` field in the input rather
+than using configurable fields. Using custom runnables and configurable fields
+is another option to customize the agent. 
+
+Please see configurable_agent_executor: https://github.com/langchain-ai/langserve/blob/main/examples/configurable_agent_executor/server.py
+for an example that uses a custom runnable with configurable fields.
+
+Relevant LangChain documentation:
+
+* Creating a custom agent: https://python.langchain.com/docs/modules/agents/how_to/custom_agent
+* Streaming with agents: https://python.langchain.com/docs/modules/agents/how_to/streaming#custom-streaming-with-events
+* General streaming documentation: https://python.langchain.com/docs/expression_language/streaming
+* Message History: https://python.langchain.com/docs/expression_language/how_to/message_history
+
+**ATTENTION**
+1. This example does not truncate message history, so it will crash if you
+   send too many messages (exceed token length).
+2. The playground at the moment does not render agent output well! If you want to
+   use the playground you need to customize it's output server side using astream
+   events by wrapping it within another runnable.
+3. See the client notebook to see how .stream() behaves!
+"""  # noqa: E501
+from typing import Any, AsyncIterator, List, Literal
+
+from fastapi import FastAPI
+from langchain.agents import AgentExecutor, tool
+from langchain.agents.format_scratchpad.openai_tools import (
+    format_to_openai_tool_messages,
+)
+from langchain.agents.output_parsers.openai_tools import OpenAIToolsAgentOutputParser
+from langchain.prompts import MessagesPlaceholder
+from langchain_community.tools.convert_to_openai import format_tool_to_openai_tool
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.runnables import RunnableLambda
+from langchain_openai import ChatOpenAI
+
+from langserve import add_routes
+from langserve.pydantic_v1 import BaseModel
+
+prompt = ChatPromptTemplate.from_messages(
+    [
+        (
+            "system",
+            "You are very powerful assistant, but bad at calculating lengths of words. "
+            "Talk with the user as normal. "
+            "If they ask you to calculate the length of a word, use a tool",
+        ),
+        # Please note the ordering of the fields in the prompt!
+        # The correct ordering is:
+        # 1. user - the user's current input
+        # 2. agent_scratchpad - the agent's working space for thinking and
+        #    invoking tools to respond to the user's input.
+        # If you change the ordering, the agent will not work correctly since
+        # the messages will be shown to the underlying LLM in the wrong order.
+        ("user", "{input}"),
+        MessagesPlaceholder(variable_name="agent_scratchpad"),
+    ]
+)
+
+
+@tool
+def word_length(word: str) -> int:
+    """Returns a counter word"""
+    return len(word)
+
+
+@tool
+def favorite_animal(name: str) -> str:
+    """Get the favorite animal of the person with the given name"""
+    if name.lower().strip() == "eugene":
+        return "cat"
+    return "dog"
+
+
+# We need to set streaming=True on the LLM to support streaming individual tokens.
+# Tokens will be available when using the stream_log / stream events endpoints,
+# but not when using the stream endpoint since the stream implementation for agent
+# streams action observation pairs not individual tokens.
+# See the client notebook that shows how to use the stream events endpoint.
+llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0, streaming=True)
+
+TOOL_MAPPING = {
+    "word_length": word_length,
+    "favorite_animal": favorite_animal,
+}
+KnownTool = Literal["word_length", "favorite_animal"]
+
+
+def _create_agent_with_tools(requested_tools: List[KnownTool]) -> AgentExecutor:
+    """Create an agent with custom tools."""
+    tools = []
+
+    for requested_tool in requested_tools:
+        if requested_tool not in TOOL_MAPPING:
+            raise ValueError(f"Unknown tool: {requested_tool}")
+        tools.append(TOOL_MAPPING[requested_tool])
+
+    if tools:
+        llm_with_tools = llm.bind(
+            tools=[format_tool_to_openai_tool(tool) for tool in tools]
+        )
+    else:
+        llm_with_tools = llm
+
+    agent = (
+        {
+            "input": lambda x: x["input"],
+            "agent_scratchpad": lambda x: format_to_openai_tool_messages(
+                x["intermediate_steps"]
+            ),
+        }
+        | prompt
+        | llm_with_tools
+        | OpenAIToolsAgentOutputParser()
+    )
+    agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=True).with_config(
+        {"run_name": "agent"}
+    )
+    return agent_executor
+
+
+app = FastAPI(
+    title="LangChain Server",
+    version="1.0",
+    description="Spin up a simple api server using LangChain's Runnable interfaces",
+)
+
+
+# We need to add these input/output schemas because the current AgentExecutor
+# is lacking in schemas.
+class Input(BaseModel):
+    input: str
+    tools: List[KnownTool]
+
+
+async def custom_stream(input: Input) -> AsyncIterator[str]:
+    """A custom runnable that can stream content.
+
+    Args:
+        input: The input to the agent. See the Input model for more details.
+
+    Yields:
+        strings that are streamed to the client.
+
+
+    Strings were chosen for simplicity, feel free to adapt to your use case.
+
+    You will almost certainly want to return more structured output in the form
+    of a dictionary, so it's easy to determine what the agent is doing without
+    parsing the output.
+
+    Before creating a custom streaming API, you should consider if you can use
+    the existing astream events API and customize the output on the client side
+    (potentially less overall work both server and client side).
+    """
+    agent_executor = _create_agent_with_tools(input["tools"])
+    async for event in agent_executor.astream_events(
+        {
+            "input": input["input"],
+        },
+        version="v1",
+    ):
+        kind = event["event"]
+        if kind == "on_chain_start":
+            if (
+                event["name"] == "agent"
+            ):  # matches `.with_config({"run_name": "Agent"})` in agent_executor
+                yield "\n"
+                yield (
+                    f"Starting agent: {event['name']} "
+                    f"with input: {event['data'].get('input')}"
+                )
+                yield "\n"
+        elif kind == "on_chain_end":
+            if (
+                event["name"] == "agent"
+            ):  # matches `.with_config({"run_name": "Agent"})` in agent_executor
+                yield "\n"
+                yield (
+                    f"Done agent: {event['name']} "
+                    f"with output: {event['data'].get('output')['output']}"
+                )
+                yield "\n"
+        if kind == "on_chat_model_stream":
+            content = event["data"]["chunk"].content
+            if content:
+                # Empty content in the context of OpenAI means
+                # that the model is asking for a tool to be invoked.
+                # So we only print non-empty content
+                yield content
+        elif kind == "on_tool_start":
+            yield "\n"
+            yield (
+                f"Starting tool: {event['name']} "
+                f"with inputs: {event['data'].get('input')}"
+            )
+            yield "\n"
+        elif kind == "on_tool_end":
+            yield "\n"
+            yield (
+                f"Done tool: {event['name']} "
+                f"with output: {event['data'].get('output')}"
+            )
+            yield "\n"
+
+
+class Output(BaseModel):
+    output: Any
+
+
+# Adds routes to the app for using the chain under:
+# /invoke
+# /batch
+# /stream
+# /stream_events
+add_routes(
+    app,
+    RunnableLambda(custom_stream),
+)
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=8000)

examples/agent_with_history/client.ipynb

@@ -0,0 +1,554 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Client\n",
+    "\n",
+    "Demo of a client interacting with a remote agent that can use history.\n",
+    "\n",
+    "See relevant documentation about agents:\n",
+    "\n",
+    "* Creating a custom agent: https://python.langchain.com/docs/modules/agents/how_to/custom_agent\n",
+    "* Streaming with agents: https://python.langchain.com/docs/modules/agents/how_to/streaming#custom-streaming-with-events\n",
+    "* General streaming documentation: https://python.langchain.com/docs/expression_language/streaming"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can interact with this via API directly"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'output': {'output': 'The length of the word \"audioee\" is 7.'},\n",
+       " 'callback_events': [],\n",
+       " 'metadata': {'run_id': '1847be77-f53c-40ba-b88d-06af3a598b6e'}}"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import requests\n",
+    "\n",
+    "inputs = {\"input\": {\"input\": \"what is the length of the word audioee?\", \"chat_history\": []}}\n",
+    "response = requests.post(\"http://localhost:8000/invoke\", json=inputs)\n",
+    "\n",
+    "response.json()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can also interact with this via the RemoteRunnable interface (to use in other chains)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langserve import RemoteRunnable\n",
+    "\n",
+    "remote_runnable = RemoteRunnable(\"http://localhost:8000/\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Remote runnable has the same interface as local runnables"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain_core.messages import HumanMessage, AIMessage"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "Human (Q/q to quit):  hello\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "AI: Hello! How can I assist you today?\n"
+     ]
+    },
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "Human (Q/q to quit):  my name is eugene\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "AI: Nice to meet you, Eugene! How can I help you today?\n"
+     ]
+    },
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "Human (Q/q to quit):  what is my name\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "AI: Your name is Eugene.\n"
+     ]
+    },
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "Human (Q/q to quit):  what is the length of my name\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "AI: The length of your name, Eugene, is 6 characters.\n"
+     ]
+    },
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "Human (Q/q to quit):  q\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "AI: Bye bye human\n"
+     ]
+    }
+   ],
+   "source": [
+    "chat_history = []\n",
+    "\n",
+    "while True:\n",
+    "    human = input(\"Human (Q/q to quit): \")\n",
+    "    if human in {\"q\", \"Q\"}:\n",
+    "        print('AI: Bye bye human')\n",
+    "        break\n",
+    "    ai = await remote_runnable.ainvoke({\"input\": human, \"chat_history\": chat_history})\n",
+    "    print(f\"AI: {ai['output']}\")\n",
+    "    chat_history.extend([HumanMessage(content=human), AIMessage(content=ai['output'])])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Stream\n",
+    "\n",
+    "Please note that streaming alternates between actions and observations. It does not stream individual tokens!\n",
+    "\n",
+    "To stream individual tokens, we need to use the astream events endpoint (see below)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "Human (Q/q to quit):  hello\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "AI: \n",
+      "Hello! How can I assist you today?\n",
+      "------\n"
+     ]
+    },
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "Human (Q/q to quit):  my name is eugene\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "AI: \n",
+      "Nice to meet you, Eugene! How can I help you today?\n",
+      "------\n"
+     ]
+    },
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "Human (Q/q to quit):  what is my name\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "AI: \n",
+      "Your name is Eugene.\n",
+      "------\n"
+     ]
+    },
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "Human (Q/q to quit):  what's the length of my name?\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "AI: \n",
+      "Calling Tool ```word_length``` with input ```{'word': 'Eugene'}```\n",
+      "------\n",
+      "Got result: ```6```\n",
+      "------\n",
+      "The length of your name, Eugene, is 6 characters.\n",
+      "------\n"
+     ]
+    },
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "Human (Q/q to quit):  q\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "AI: Bye bye human\n"
+     ]
+    }
+   ],
+   "source": [
+    "chat_history = []\n",
+    "\n",
+    "while True:\n",
+    "    human = input(\"Human (Q/q to quit): \")\n",
+    "    if human in {\"q\", \"Q\"}:\n",
+    "        print('AI: Bye bye human')\n",
+    "        break\n",
+    "\n",
+    "    ai = None\n",
+    "    print(\"AI: \")\n",
+    "    async for chunk in remote_runnable.astream({\"input\": human, \"chat_history\": chat_history}):\n",
+    "        # Agent Action\n",
+    "        if \"actions\" in chunk:\n",
+    "            for action in chunk[\"actions\"]:\n",
+    "                print(\n",
+    "                    f\"Calling Tool ```{action['tool']}``` with input ```{action['tool_input']}```\"\n",
+    "                )\n",
+    "        # Observation\n",
+    "        elif \"steps\" in chunk:\n",
+    "            for step in chunk[\"steps\"]:\n",
+    "                print(f\"Got result: ```{step['observation']}```\")\n",
+    "        # Final result\n",
+    "        elif \"output\" in chunk:\n",
+    "            print(chunk['output'])\n",
+    "            ai = AIMessage(content=chunk['output'])\n",
+    "        else:\n",
+    "            raise ValueError\n",
+    "        print(\"------\")        \n",
+    "    chat_history.extend([HumanMessage(content=human), ai])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Stream Events"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "Human (Q/q to quit):  hello! my name is eugene\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "AI: \n",
+      "Starting agent: agent with input: {'input': 'hello! my name is eugene', 'chat_history': []}\n",
+      "Hello| Eugene|!| How| can| I| assist| you| today|?|\n",
+      "--\n",
+      "Done agent: agent with output: Hello Eugene! How can I assist you today?\n"
+     ]
+    },
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "Human (Q/q to quit):  what's the length of my name?\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "AI: \n",
+      "Starting agent: agent with input: {'input': \"what's the length of my name?\", 'chat_history': []}\n",
+      "--\n",
+      "Starting tool: word_length with inputs: {'word': 'my name'}\n",
+      "Done tool: word_length\n",
+      "Tool output was: 7\n",
+      "--\n",
+      "The| length| of| your| name|,| \"|my| name|\",| is| |7| characters|.|\n",
+      "--\n",
+      "Done agent: agent with output: The length of your name, \"my name\", is 7 characters.\n"
+     ]
+    },
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "Human (Q/q to quit):  could you tell me a long story about the length of my name?\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "AI: \n",
+      "Starting agent: agent with input: {'input': 'could you tell me a long story about the length of my name?', 'chat_history': []}\n",
+      "Once| upon| a| time|,| there| was| a| person| named| [|Your| Name|].| Now|,| [|Your| Name|]| had| a| very| unique| and| special| name|.| It| was| a| name| that| carried| a| lot| of| meaning| and| significance|.| People| often| wondered| about| the| length| of| [|Your| Name|]'|s| name| and| how| it| compared| to| others|.\n",
+      "\n",
+      "|One| day|,| [|Your| Name|]| decided| to| embark| on| a| journey| to| discover| the| true| length| of| their| name|.| They| traveled| far| and| wide|,| seeking| the| wisdom| of| s|ages| and| scholars| who| were| known| for| their| knowledge| of| names| and| their| lengths|.\n",
+      "\n",
+      "|Along| the| way|,| [|Your| Name|]| encountered| many| interesting| characters| who| had| their| own| stories| to| tell| about| the| lengths| of| their| names|.| Some| had| short| names| that| were| easy| to| remember|,| while| others| had| long| names| that| seemed| to| go| on| forever|.\n",
+      "\n",
+      "|As| [|Your| Name|]| continued| their| quest|,| they| came| across| a| wise| old| wizard| who| claimed| to| have| a| magical| tool| that| could| calculate| the| exact| length| of| any| name|.| Intr|ig|ued|,| [|Your| Name|]| approached| the| wizard| and| asked| for| their| assistance|.\n",
+      "\n",
+      "|The| wizard| took| out| a| mystical| device| and| asked| [|Your| Name|]| to| spell| out| their| name|.| [|Your| Name|]| eagerly| complied|,| carefully| en|unci|ating| each| letter|.| The| wizard| then| waved| the| device| over| the| name| and| muttered| an| inc|ant|ation|.\n",
+      "\n",
+      "|In| an| instant|,| the| device| displayed| the| length| of| [|Your| Name|]'|s| name|.| It| was| a| number| that| represented| the| total| count| of| characters| in| their| name|,| including| spaces| and| punctuation| marks|.| [|Your| Name|]| was| amazed| to| see| the| result| and| thanked| the| wizard| for| their| help|.\n",
+      "\n",
+      "|Ar|med| with| this| newfound| knowledge|,| [|Your| Name|]| returned| home| and| shared| their| story| with| friends| and| family|.| They| realized| that| the| length| of| their| name| was| not| just| a| random| number|,| but| a| reflection| of| their| identity| and| the| unique| journey| they| had| taken| to| discover| it|.\n",
+      "\n",
+      "|And| so|,| [|Your| Name|]| lived| happily| ever| after|,| cher|ishing| their| name| and| the| story| behind| its| length|.| They| understood| that| the| length| of| a| name| is| not| just| a| matter| of| counting| letters|,| but| a| testament| to| the| individual|ity| and| significance| of| each| person|'s| identity|.|\n",
+      "--\n",
+      "Done agent: agent with output: Once upon a time, there was a person named [Your Name]. Now, [Your Name] had a very unique and special name. It was a name that carried a lot of meaning and significance. People often wondered about the length of [Your Name]'s name and how it compared to others.\n",
+      "\n",
+      "One day, [Your Name] decided to embark on a journey to discover the true length of their name. They traveled far and wide, seeking the wisdom of sages and scholars who were known for their knowledge of names and their lengths.\n",
+      "\n",
+      "Along the way, [Your Name] encountered many interesting characters who had their own stories to tell about the lengths of their names. Some had short names that were easy to remember, while others had long names that seemed to go on forever.\n",
+      "\n",
+      "As [Your Name] continued their quest, they came across a wise old wizard who claimed to have a magical tool that could calculate the exact length of any name. Intrigued, [Your Name] approached the wizard and asked for their assistance.\n",
+      "\n",
+      "The wizard took out a mystical device and asked [Your Name] to spell out their name. [Your Name] eagerly complied, carefully enunciating each letter. The wizard then waved the device over the name and muttered an incantation.\n",
+      "\n",
+      "In an instant, the device displayed the length of [Your Name]'s name. It was a number that represented the total count of characters in their name, including spaces and punctuation marks. [Your Name] was amazed to see the result and thanked the wizard for their help.\n",
+      "\n",
+      "Armed with this newfound knowledge, [Your Name] returned home and shared their story with friends and family. They realized that the length of their name was not just a random number, but a reflection of their identity and the unique journey they had taken to discover it.\n",
+      "\n",
+      "And so, [Your Name] lived happily ever after, cherishing their name and the story behind its length. They understood that the length of a name is not just a matter of counting letters, but a testament to the individuality and significance of each person's identity.\n"
+     ]
+    },
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "Human (Q/q to quit):  that's not what I wanted. My name is eugene. calculate the length of my name and tell me a story about the result.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "AI: \n",
+      "Starting agent: agent with input: {'input': \"that's not what I wanted. My name is eugene. calculate the length of my name and tell me a story about the result.\", 'chat_history': []}\n",
+      "--\n",
+      "Starting tool: word_length with inputs: {'word': 'eugene'}\n",
+      "Done tool: word_length\n",
+      "Tool output was: 6\n",
+      "--\n",
+      "The| length| of| your| name|,| Eugene|,| is| |6| characters|.| Now|,| let| me| tell| you| a| story| about| the| result|.\n",
+      "\n",
+      "|Once| upon| a| time|,| in| a| land| far| away|,| there| was| a| young| boy| named| Eugene|.| He| had| a| special| power| -| the| power| to| bring| joy| and| laughter| to| everyone| he| met|.| Eugene|'s| name|,| with| its| |6| letters|,| perfectly| reflected| his| vibrant| and| energetic| personality|.\n",
+      "\n",
+      "|One| day|,| Eugene| decided| to| embark| on| a| grand| adventure|.| He| set| off| on| a| journey| to| spread| happiness| and| positivity| throughout| the| kingdom|.| Along| the| way|,| he| encountered| people| from| all| walks| of| life| -| from| humble| farmers| to| noble| knights|.\n",
+      "\n",
+      "|With| his| infectious| smile| and| kind| heart|,| Eugene| touched| the| lives| of| everyone| he| met|.| His| name|,| with| its| |6| letters|,| became| synonymous| with| love|,| compassion|,| and| joy|.| People| would| often| say|,| \"|If| you| want| to| experience| true| happiness|,| just| spend| a| moment| with| Eugene|.\"\n",
+      "\n",
+      "|As| Eugene| continued| his| journey|,| word| of| his| incredible| ability| to| bring| happiness| spread| far| and| wide|.| People| from| distant| lands| would| travel| for| miles| just| to| catch| a| glimpse| of| him|.| His| name|,| with| its| |6| letters|,| became| a| symbol| of| hope| and| inspiration|.\n",
+      "\n",
+      "|E|ug|ene|'s| story| serves| as| a| reminder| that| sometimes|,| the| simplest| things| can| have| the| greatest| impact|.| His| name|,| with| its| |6| letters|,| became| a| beacon| of| light| in| a| world| that| often| seemed| dark| and| glo|omy|.\n",
+      "\n",
+      "|And| so|,| Eugene|'s| adventure| continues|,| as| he| spreads| joy| and| happiness| wherever| he| goes|.| His| name|,| with| its| |6| letters|,| will| forever| be| et|ched| in| the| hearts| of| those| whose| lives| he| has| touched|.\n",
+      "\n",
+      "|The| end|.|\n",
+      "--\n",
+      "Done agent: agent with output: The length of your name, Eugene, is 6 characters. Now, let me tell you a story about the result.\n",
+      "\n",
+      "Once upon a time, in a land far away, there was a young boy named Eugene. He had a special power - the power to bring joy and laughter to everyone he met. Eugene's name, with its 6 letters, perfectly reflected his vibrant and energetic personality.\n",
+      "\n",
+      "One day, Eugene decided to embark on a grand adventure. He set off on a journey to spread happiness and positivity throughout the kingdom. Along the way, he encountered people from all walks of life - from humble farmers to noble knights.\n",
+      "\n",
+      "With his infectious smile and kind heart, Eugene touched the lives of everyone he met. His name, with its 6 letters, became synonymous with love, compassion, and joy. People would often say, \"If you want to experience true happiness, just spend a moment with Eugene.\"\n",
+      "\n",
+      "As Eugene continued his journey, word of his incredible ability to bring happiness spread far and wide. People from distant lands would travel for miles just to catch a glimpse of him. His name, with its 6 letters, became a symbol of hope and inspiration.\n",
+      "\n",
+      "Eugene's story serves as a reminder that sometimes, the simplest things can have the greatest impact. His name, with its 6 letters, became a beacon of light in a world that often seemed dark and gloomy.\n",
+      "\n",
+      "And so, Eugene's adventure continues, as he spreads joy and happiness wherever he goes. His name, with its 6 letters, will forever be etched in the hearts of those whose lives he has touched.\n",
+      "\n",
+      "The end.\n"
+     ]
+    },
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "Human (Q/q to quit):  q\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "AI: Bye bye human\n"
+     ]
+    }
+   ],
+   "source": [
+    "chat_history = []\n",
+    "\n",
+    "while True:\n",
+    "    human = input(\"Human (Q/q to quit): \")\n",
+    "    if human in {\"q\", \"Q\"}:\n",
+    "        print('AI: Bye bye human')\n",
+    "        break\n",
+    "    ai = None\n",
+    "    print(\"AI: \")\n",
+    "    async for event in remote_runnable.astream_events(\n",
+    "        {\"input\": human, \"chat_history\": chat_history},\n",
+    "        version=\"v1\",\n",
+    "    ):\n",
+    "        kind = event[\"event\"]\n",
+    "        if kind == \"on_chain_start\":\n",
+    "            if (\n",
+    "                event[\"name\"] == \"agent\"\n",
+    "            ):  # Was assigned when creating the agent with `.with_config({\"run_name\": \"Agent\"})`\n",
+    "                print(\n",
+    "                    f\"Starting agent: {event['name']} with input: {event['data'].get('input')}\"\n",
+    "                )\n",
+    "        elif kind == \"on_chain_end\":\n",
+    "            if (\n",
+    "                event[\"name\"] == \"agent\"\n",
+    "            ):  # Was assigned when creating the agent with `.with_config({\"run_name\": \"Agent\"})`\n",
+    "                print()\n",
+    "                print(\"--\")\n",
+    "                print(\n",
+    "                    f\"Done agent: {event['name']} with output: {event['data'].get('output')['output']}\"\n",
+    "                )\n",
+    "        if kind == \"on_chat_model_stream\":\n",
+    "            content = event[\"data\"][\"chunk\"].content\n",
+    "            if content:\n",
+    "                # Empty content in the context of OpenAI means\n",
+    "                # that the model is asking for a tool to be invoked.\n",
+    "                # So we only print non-empty content\n",
+    "                print(content, end=\"|\")\n",
+    "        elif kind == \"on_tool_start\":\n",
+    "            print(\"--\")\n",
+    "            print(\n",
+    "                f\"Starting tool: {event['name']} with inputs: {event['data'].get('input')}\"\n",
+    "            )\n",
+    "        elif kind == \"on_tool_end\":\n",
+    "            print(f\"Done tool: {event['name']}\")\n",
+    "            print(f\"Tool output was: {event['data'].get('output')}\")\n",
+    "            print(\"--\")    \n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/agent_with_history/server.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python
+"""Example LangChain server exposes and agent that has conversation history.
+
+In this example, the history is stored entirely on the client's side.
+
+Please see other examples in LangServe on how to use RunnableWithHistory to
+store history on the server side.
+
+Relevant LangChain documentation:
+
+* Creating a custom agent: https://python.langchain.com/docs/modules/agents/how_to/custom_agent
+* Streaming with agents: https://python.langchain.com/docs/modules/agents/how_to/streaming#custom-streaming-with-events
+* General streaming documentation: https://python.langchain.com/docs/expression_language/streaming
+* Message History: https://python.langchain.com/docs/expression_language/how_to/message_history
+
+**ATTENTION**
+1. To support streaming individual tokens you will need to use the astream events
+   endpoint rather than the streaming endpoint.
+2. This example does not truncate message history, so it will crash if you
+   send too many messages (exceed token length).
+3. The playground at the moment does not render agent output well! If you want to
+   use the playground you need to customize it's output server side using astream
+   events by wrapping it within another runnable.
+4. See the client notebook it has an example of how to use stream_events client side!
+"""  # noqa: E501
+from typing import Any, List, Union
+
+from fastapi import FastAPI
+from langchain.agents import AgentExecutor, tool
+from langchain.agents.format_scratchpad.openai_tools import (
+    format_to_openai_tool_messages,
+)
+from langchain.agents.output_parsers.openai_tools import OpenAIToolsAgentOutputParser
+from langchain.prompts import MessagesPlaceholder
+from langchain_community.tools.convert_to_openai import format_tool_to_openai_tool
+from langchain_core.messages import AIMessage, FunctionMessage, HumanMessage
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_openai import ChatOpenAI
+
+from langserve import add_routes
+from langserve.pydantic_v1 import BaseModel, Field
+
+prompt = ChatPromptTemplate.from_messages(
+    [
+        (
+            "system",
+            "You are very powerful assistant, but bad at calculating lengths of words. "
+            "Talk with the user as normal. "
+            "If they ask you to calculate the length of a word, use a tool",
+        ),
+        # Please note the ordering of the fields in the prompt!
+        # The correct ordering is:
+        # 1. history - the past messages between the user and the agent
+        # 2. user - the user's current input
+        # 3. agent_scratchpad - the agent's working space for thinking and
+        #    invoking tools to respond to the user's input.
+        # If you change the ordering, the agent will not work correctly since
+        # the messages will be shown to the underlying LLM in the wrong order.
+        MessagesPlaceholder(variable_name="chat_history"),
+        ("user", "{input}"),
+        MessagesPlaceholder(variable_name="agent_scratchpad"),
+    ]
+)
+
+
+@tool
+def word_length(word: str) -> int:
+    """Returns a counter word"""
+    return len(word)
+
+
+# We need to set streaming=True on the LLM to support streaming individual tokens.
+# Tokens will be available when using the stream_log / stream events endpoints,
+# but not when using the stream endpoint since the stream implementation for agent
+# streams action observation pairs not individual tokens.
+# See the client notebook that shows how to use the stream events endpoint.
+llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0, streaming=True)
+
+tools = [word_length]
+
+
+llm_with_tools = llm.bind(tools=[format_tool_to_openai_tool(tool) for tool in tools])
+
+# ATTENTION: For production use case, it's a good idea to trim the prompt to avoid
+#            exceeding the context window length used by the model.
+#
+# To fix that simply adjust the chain to trim the prompt in whatever way
+# is appropriate for your use case.
+# For example, you may want to keep the system message and the last 10 messages.
+# Or you may want to trim based on the number of tokens.
+# Or you may want to also summarize the messages to keep information about things
+# that were learned about the user.
+#
+# def prompt_trimmer(messages: List[Union[HumanMessage, AIMessage, FunctionMessage]]):
+#     '''Trims the prompt to a reasonable length.'''
+#     # Keep in mind that when trimming you may want to keep the system message!
+#     return messages[-10:] # Keep last 10 messages.
+
+agent = (
+    {
+        "input": lambda x: x["input"],
+        "agent_scratchpad": lambda x: format_to_openai_tool_messages(
+            x["intermediate_steps"]
+        ),
+        "chat_history": lambda x: x["chat_history"],
+    }
+    | prompt
+    # | prompt_trimmer # See comment above.
+    | llm_with_tools
+    | OpenAIToolsAgentOutputParser()
+)
+agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=True)
+
+
+app = FastAPI(
+    title="LangChain Server",
+    version="1.0",
+    description="Spin up a simple api server using LangChain's Runnable interfaces",
+)
+
+
+# We need to add these input/output schemas because the current AgentExecutor
+# is lacking in schemas.
+class Input(BaseModel):
+    input: str
+    # The field extra defines a chat widget.
+    # Please see documentation about widgets in the main README.
+    # The widget is used in the playground.
+    # Keep in mind that playground support for agents is not great at the moment.
+    # To get a better experience, you'll need to customize the streaming output
+    # for now.
+    chat_history: List[Union[HumanMessage, AIMessage, FunctionMessage]] = Field(
+        ...,
+        extra={"widget": {"type": "chat", "input": "input", "output": "output"}},
+    )
+
+
+class Output(BaseModel):
+    output: Any
+
+
+# Adds routes to the app for using the chain under:
+# /invoke
+# /batch
+# /stream
+# /stream_events
+add_routes(
+    app,
+    agent_executor.with_types(input_type=Input, output_type=Output).with_config(
+        {"run_name": "agent"}
+    ),
+)
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=8000)

examples/api_handler_examples/server.py

@@ -0,0 +1,125 @@
+"""An example that shows how to use the API handler directly.
+
+For this to work with RemoteClient, the routes must match those expected
+by the client; i.e., /invoke, /batch, /stream, etc. No trailing slashes should be used.
+"""
+from importlib import metadata
+from typing import Annotated
+
+from fastapi import Depends, FastAPI, Request, Response
+from langchain_core.runnables import RunnableLambda
+from sse_starlette import EventSourceResponse
+
+from langserve import APIHandler
+
+PYDANTIC_VERSION = metadata.version("pydantic")
+_PYDANTIC_MAJOR_VERSION: int = int(PYDANTIC_VERSION.split(".")[0])
+
+app = FastAPI(
+    title="LangChain Server",
+    version="1.0",
+    description="Spin up a simple api server using Langchain's Runnable interfaces",
+)
+
+
+##
+# Example 1 -- invoke, batch together with doc-generation
+# This endpoint shows how to expose `invoke` and `batch` using the APIHandler.
+# It also shows how to generate documentation properly so it works correctly
+# depending on Fast API and pydantic versions.
+def add_one(x: int) -> int:
+    """Add one to the given number."""
+    return x + 1
+
+
+chain = RunnableLambda(add_one)
+
+api_handler = APIHandler(chain, path="/simple")
+
+
+# First register the endpoints without documentation
+@app.post("/simple/invoke", include_in_schema=False)
+async def simple_invoke(request: Request) -> Response:
+    """Handle a request."""
+    # The API Handler validates the parts of the request
+    # that are used by the runnnable (e.g., input, config fields)
+    return await api_handler.invoke(request)
+
+
+@app.post("/simple/batch", include_in_schema=False)
+async def simple_batch(request: Request) -> Response:
+    """Handle a request."""
+    # The API Handler validates the parts of the request
+    # that are used by the runnnable (e.g., input, config fields)
+    return await api_handler.batch(request)
+
+
+# Here, we show how to populate the documentation for the endpoint.
+# Please note that this is done separately from the actual endpoint.
+# This happens due to two reasons:
+# 1. FastAPI does not support using pydantic.v1 models in the docs endpoint.
+#    "https://github.com/tiangolo/fastapi/issues/10360"
+#    LangChain uses pydantic.v1 models!
+# 2. Configurable Runnables have a *dynamic* schema, which means that
+#    the shape of the input depends on the config.
+#    In this case, the openapi schema is a best effort showing the documentation
+#    that will work for the default config (and any non-conflicting configs).
+if _PYDANTIC_MAJOR_VERSION == 1:  # Do not use in your own
+    # Add documentation
+    @app.post("/simple/invoke")
+    async def simple_invoke_docs(
+        request: api_handler.InvokeRequest,
+    ) -> api_handler.InvokeResponse:
+        """API endpoint used only for documentation purposes. Populate /docs endpoint"""
+        raise NotImplementedError(
+            "This endpoint is only used for documentation purposes"
+        )
+
+    @app.post("/simple/batch")
+    async def simple_batch_docs(
+        request: api_handler.BatchRequest,
+    ) -> api_handler.BatchResponse:
+        """API endpoint used only for documentation purposes. Populate /docs endpoint"""
+        raise NotImplementedError(
+            "This endpoint is only used for documentation purposes"
+        )
+
+else:
+    print(
+        "Skipping documentation generation for pydantic v2: "
+        "https://github.com/tiangolo/fastapi/issues/10360"
+    )
+
+
+##
+# Example 2 -- Expose `invoke` and `stream` using the API Handler.
+# Uses FastAPI Depends get a ready API handler.
+async def _get_api_handler() -> APIHandler:
+    """Prepare a RunnableLambda."""
+    return APIHandler(RunnableLambda(add_one), path="/v2")
+
+
+@app.post("/v2/invoke")
+async def v2_invoke(
+    request: Request, runnable: Annotated[APIHandler, Depends(_get_api_handler)]
+) -> Response:
+    """Handle invoke request."""
+    # The API Handler validates the parts of the request
+    # that are used by the runnnable (e.g., input, config fields)
+    return await runnable.invoke(request)
+
+
+@app.post("/v2/stream")
+async def v2_stream(
+    request: Request, runnable: Annotated[APIHandler, Depends(_get_api_handler)]
+) -> EventSourceResponse:
+    """Handle stream request."""
+    # The API Handler validates the parts of the request
+    # that are used by the runnnable (e.g., input, config fields)
+    return await runnable.stream(request)
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=8000)

examples/auth/api_handler/client.ipynb

@@ -0,0 +1,212 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Client\n",
+    "\n",
+    "This is an example client that interacts with the server that has \"auth\".\n",
+    "\n",
+    "Please reference appropriate documentation in the server code and in FastAPI to actually make this secure.\n",
+    "\n",
+    "\n",
+    "**ATTENTION** Only the invoke endpoint has been defined by the server! \n",
+    "So batch/stream won't work. If you want to add stream and batch, you can do so as well on the server side.\n",
+    "The server is implemented using the APIHandler, it's more flexible, but does require a bit more code. :)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Login as Alice"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import requests\n",
+    "\n",
+    "response = requests.post(\"http://localhost:8000/token\", data={\"username\": \"alice\", \"password\": \"secret1\"})\n",
+    "result = response.json()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "token = result['access_token']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "inputs = {\"input\": \"hello\"}\n",
+    "response = requests.post(\"http://localhost:8000/my_runnable/invoke\", \n",
+    "    json={\n",
+    "        'input': 'hello',\n",
+    "    },\n",
+    "    headers={\n",
+    "        'Authorization': f\"Bearer {token}\"\n",
+    "    }\n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'output': [{'page_content': 'cats like mice',\n",
+       "   'metadata': {'owner_id': 'alice'},\n",
+       "   'type': 'Document'},\n",
+       "  {'page_content': 'cats like cheese',\n",
+       "   'metadata': {'owner_id': 'alice'},\n",
+       "   'type': 'Document'}],\n",
+       " 'callback_events': [],\n",
+       " 'metadata': {'run_id': '1732c9aa-c6d3-4736-b8ca-01265fa8ba06'}}"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "response.json()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "You can also interact with this via the RemoteRunnable interface (to use in other chains)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langserve import RemoteRunnable\n",
+    "\n",
+    "remote_runnable = RemoteRunnable(\"http://localhost:8000/my_runnable\", headers={\"Authorization\": f\"Bearer {token}\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(page_content='cats like mice', metadata={'owner_id': 'alice'}),\n",
+       " Document(page_content='cats like cheese', metadata={'owner_id': 'alice'})]"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "await remote_runnable.ainvoke(\"cat\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Login as John"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import requests\n",
+    "\n",
+    "response = requests.post(\"http://localhost:8000/token\", data={\"username\": \"john\", \"password\": \"secret2\"})\n",
+    "token = response.json()['access_token']\n",
+    "remote_runnable = RemoteRunnable(\"http://localhost:8000/my_runnable\", headers={\"Authorization\": f\"Bearer {token}\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(page_content='i like walks by the ocean', metadata={'owner_id': 'john'}),\n",
+       " Document(page_content='dogs like sticks', metadata={'owner_id': 'john'}),\n",
+       " Document(page_content='my favorite food is cheese', metadata={'owner_id': 'john'})]"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "await remote_runnable.ainvoke(\"water\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/auth/api_handler/server.py

@@ -0,0 +1,285 @@
+#!/usr/bin/env python
+"""Example that shows how to use the underlying APIHandler class directly with Auth.
+
+This example shows how to apply logic based on the user's identity.
+
+You can build on these concepts to implement a more complex app:
+* Add endpoints that allow users to manage their documents.
+* Make a more complex runnable that does something with the retrieved documents; e.g.,
+  a conversational agent that responds to the user's input with the retrieved documents
+  (which are user specific documents).
+
+For authentication, we use a fake token that's the same as the username, adapting
+the following example from the FastAPI docs:
+
+https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/
+
+**ATTENTION**
+
+This example is not actually secure and should not be used in production.
+
+Once you understand how to use `per_req_config_modifier`, read through
+the FastAPI docs and implement proper auth:
+https://fastapi.tiangolo.com/tutorial/security/oauth2-jwt/
+
+
+**ATTENTION**
+
+This example does not integrate auth with OpenAPI, so the OpenAPI docs won't
+be able to help with authentication. This is currently a limitation
+if using `add_routes`. If you need this functionality, you can use
+the underlying `APIHandler` class directly, which affords maximal flexibility.
+"""
+from importlib import metadata
+from typing import Any, List, Optional, Union
+
+from fastapi import Depends, FastAPI, HTTPException, Request, Response, status
+from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
+from langchain_community.embeddings.openai import OpenAIEmbeddings
+from langchain_community.vectorstores.chroma import Chroma
+from langchain_core.documents import Document
+from langchain_core.runnables import (
+    ConfigurableField,
+    RunnableConfig,
+    RunnableSerializable,
+)
+from langchain_core.vectorstores import VectorStore
+from typing_extensions import Annotated
+
+from langserve import APIHandler
+from langserve.pydantic_v1 import BaseModel
+
+
+class User(BaseModel):
+    username: str
+    email: Union[str, None] = None
+    full_name: Union[str, None] = None
+    disabled: Union[bool, None] = None
+
+
+class UserInDB(User):
+    hashed_password: str
+
+
+oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")
+
+app = FastAPI()
+
+FAKE_USERS_DB = {
+    "alice": {
+        "username": "alice",
+        "full_name": "Alice Wonderson",
+        "email": "alice@example.com",
+        "hashed_password": "fakehashedsecret1",
+        "disabled": False,
+    },
+    "john": {
+        "username": "john",
+        "full_name": "John Doe",
+        "email": "johndoe@example.com",
+        "hashed_password": "fakehashedsecret2",
+        "disabled": False,
+    },
+    "bob": {
+        "username": "john",
+        "full_name": "John Doe",
+        "email": "johndoe@example.com",
+        "hashed_password": "fakehashedsecret3",
+        "disabled": True,
+    },
+}
+
+
+def _fake_hash_password(password: str) -> str:
+    """Fake a hashed password."""
+    return "fakehashed" + password
+
+
+def _get_user(db: dict, username: str) -> Union[UserInDB, None]:
+    if username in db:
+        user_dict = db[username]
+        return UserInDB(**user_dict)
+
+
+def _fake_decode_token(token: str) -> Union[User, None]:
+    # This doesn't provide any security at all
+    # Check the next version
+    user = _get_user(FAKE_USERS_DB, token)
+    return user
+
+
+@app.post("/token")
+async def login(form_data: Annotated[OAuth2PasswordRequestForm, Depends()]):
+    user_dict = FAKE_USERS_DB.get(form_data.username)
+    if not user_dict:
+        raise HTTPException(status_code=400, detail="Incorrect username or password")
+    user = UserInDB(**user_dict)
+    hashed_password = _fake_hash_password(form_data.password)
+    if not hashed_password == user.hashed_password:
+        raise HTTPException(status_code=400, detail="Incorrect username or password")
+
+    return {"access_token": user.username, "token_type": "bearer"}
+
+
+async def get_current_user(token: Annotated[str, Depends(oauth2_scheme)]):
+    user = _fake_decode_token(token)
+    if not user:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid authentication credentials",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+    return user
+
+
+async def get_current_active_user(
+    current_user: Annotated[User, Depends(get_current_user)],
+):
+    if current_user.disabled:
+        raise HTTPException(status_code=400, detail="Inactive user")
+    return current_user
+
+
+class PerUserVectorstore(RunnableSerializable):
+    """A custom runnable that returns a list of documents for the given user.
+
+    The runnable is configurable by the user, and the search results are
+    filtered by the user ID.
+    """
+
+    user_id: Optional[str]
+    vectorstore: VectorStore
+
+    class Config:
+        # Allow arbitrary types since VectorStore is an abstract interface
+        # and not a pydantic model
+        arbitrary_types_allowed = True
+
+    def _invoke(
+        self, input: str, config: Optional[RunnableConfig] = None, **kwargs: Any
+    ) -> List[Document]:
+        """Invoke the retriever."""
+        # WARNING: Verify documentation of underlying vectorstore to make
+        # sure that it actually uses filters.
+        # Highly recommended to use unit-tests to verify this behavior, as
+        # implementations can be different depending on the underlying vectorstore.
+        retriever = self.vectorstore.as_retriever(
+            search_kwargs={"filter": {"owner_id": self.user_id}}
+        )
+        return retriever.invoke(input, config=config)
+
+    def invoke(
+        self, input: str, config: Optional[RunnableConfig] = None, **kwargs
+    ) -> List[Document]:
+        """Add one to an integer."""
+        return self._call_with_config(self._invoke, input, config, **kwargs)
+
+
+vectorstore = Chroma(
+    collection_name="some_collection",
+    embedding_function=OpenAIEmbeddings(),
+)
+
+vectorstore.add_documents(
+    [
+        Document(
+            page_content="cats like cheese",
+            metadata={"owner_id": "alice"},
+        ),
+        Document(
+            page_content="cats like mice",
+            metadata={"owner_id": "alice"},
+        ),
+        Document(
+            page_content="dogs like sticks",
+            metadata={"owner_id": "john"},
+        ),
+        Document(
+            page_content="my favorite food is cheese",
+            metadata={"owner_id": "john"},
+        ),
+        Document(
+            page_content="i like walks by the ocean",
+            metadata={"owner_id": "john"},
+        ),
+        Document(
+            page_content="dogs like grass",
+            metadata={"owner_id": "bob"},
+        ),
+    ]
+)
+
+per_user_retriever = PerUserVectorstore(
+    user_id=None,  # Placeholder ID that will be replaced by the per_req_config_modifier
+    vectorstore=vectorstore,
+).configurable_fields(
+    # Attention: Make sure to override the user ID for each request in the
+    # per_req_config_modifier. This should not be client configurable.
+    user_id=ConfigurableField(
+        id="user_id",
+        name="User ID",
+        description="The user ID to use for the retriever.",
+    )
+)
+
+
+# Let's define the API Handler
+api_handler = APIHandler(
+    per_user_retriever,
+    # Namespace for the runnable.
+    # Endpoints like batch / invoke should be under /my_runnable/invoke
+    # and /my_runnable/batch etc.
+    path="/my_runnable",
+)
+
+
+PYDANTIC_VERSION = metadata.version("pydantic")
+_PYDANTIC_MAJOR_VERSION: int = int(PYDANTIC_VERSION.split(".")[0])
+
+
+# **ATTENTION** Your code does not need to include both versions.
+# Use whichever version is appropriate given the pydantic version you are using.
+# Both versions are included here for demonstration purposes.
+#
+# If using pydantic <2, everything works as expected.
+# However, when using pydantic >=2 is installed, things are a bit
+# more complicated because LangChain uses the pydantic.v1 namespace
+# But the pydantic.v1 namespace is not supported by FastAPI.
+# See this issue: https://github.com/tiangolo/fastapi/issues/10360
+# So when using pydantic >=2, we need to use a vanilla starlette request
+# and response, and we will not have documentation.
+# Or we can create custom models for the request and response.
+# The underlying API Handler will still validate the request
+# correctly even if vanilla requests are used.
+if _PYDANTIC_MAJOR_VERSION == 1:
+
+    @app.post("/my_runnable/invoke")
+    async def invoke_with_auth(
+        # Included for documentation purposes
+        invoke_request: api_handler.InvokeRequest,
+        request: Request,
+        current_user: Annotated[User, Depends(get_current_active_user)],
+    ) -> Response:
+        """Handle a request."""
+        # The API Handler validates the parts of the request
+        # that are used by the runnnable (e.g., input, config fields)
+        config = {"configurable": {"user_id": current_user.username}}
+        return await api_handler.invoke(request, server_config=config)
+else:
+
+    @app.post("/my_runnable/invoke")
+    async def invoke_with_auth(
+        request: Request,
+        current_user: Annotated[User, Depends(get_current_active_user)],
+    ) -> Response:
+        """Handle a request."""
+        # The API Handler validates the parts of the request
+        # that are used by the runnnable (e.g., input, config fields)
+        config = {"configurable": {"user_id": current_user.username}}
+        return await api_handler.invoke(request, server_config=config)
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=8000)

examples/auth/global_deps/server.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+"""An example that uses Fast API global dependencies.
+
+This approach can be used if the same authentication logic can be used
+for all endpoints in the application.
+
+This may be a reasonable approach for simple applications.
+
+See:
+
+* https://fastapi.tiangolo.com/tutorial/dependencies/global-dependencies/
+* https://fastapi.tiangolo.com/tutorial/dependencies/
+* https://fastapi.tiangolo.com/tutorial/security/
+"""
+
+from fastapi import Depends, FastAPI, Header, HTTPException
+from langchain_core.runnables import RunnableLambda
+from typing_extensions import Annotated
+
+from langserve import add_routes
+
+
+async def verify_token(x_token: Annotated[str, Header()]) -> None:
+    """Verify the token is valid."""
+    # Replace this with your actual authentication logic
+    if x_token != "secret-token":
+        raise HTTPException(status_code=400, detail="X-Token header invalid")
+
+
+app = FastAPI(
+    title="LangChain Server",
+    version="1.0",
+    dependencies=[Depends(verify_token)],
+)
+
+
+def add_one(x: int) -> int:
+    """Add one to an integer."""
+    return x + 1
+
+
+chain = RunnableLambda(add_one)
+
+
+add_routes(
+    app,
+    chain,
+)
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=8000)

examples/auth/path_dependencies/server.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+"""An example that shows how to use path dependencies for authentication.
+
+The path dependencies are applied to all the routes added by the `add_routes`.
+
+To keep this example brief, we're providing a placeholder verify_token function
+that shows how to use path dependencies.
+
+To implement proper auth, please see the FastAPI docs:
+
+* https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/
+* https://fastapi.tiangolo.com/tutorial/dependencies/
+* https://fastapi.tiangolo.com/tutorial/security/
+"""  # noqa: E501
+
+from fastapi import Depends, FastAPI, Header, HTTPException
+from langchain_core.runnables import RunnableLambda
+from typing_extensions import Annotated
+
+from langserve import add_routes
+
+
+async def verify_token(x_token: Annotated[str, Header()]) -> None:
+    """Verify the token is valid."""
+    # Replace this with your actual authentication logic
+    if x_token != "secret-token":
+        raise HTTPException(status_code=400, detail="X-Token header invalid")
+
+
+app = FastAPI()
+
+
+def add_one(x: int) -> int:
+    """Add one to an integer."""
+    return x + 1
+
+
+chain = RunnableLambda(add_one)
+
+
+add_routes(
+    app,
+    chain,
+    dependencies=[Depends(verify_token)],
+)
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=8000)

examples/auth/per_req_config_modifier/client.ipynb

@@ -0,0 +1,207 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Client\n",
+    "\n",
+    "This is an example client that interacts with the server that has \"auth\".\n",
+    "\n",
+    "Please reference appropriate documentation in the server code and in FastAPI to actually make this secure."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Login as Alice"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import requests\n",
+    "\n",
+    "response = requests.post(\"http://localhost:8000/token\", data={\"username\": \"alice\", \"password\": \"secret1\"})\n",
+    "result = response.json()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "token = result['access_token']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "inputs = {\"input\": \"hello\"}\n",
+    "response = requests.post(\"http://localhost:8000/invoke\", \n",
+    "    json={\n",
+    "        'input': 'hello',\n",
+    "    },\n",
+    "    headers={\n",
+    "        'Authorization': f\"Bearer {token}\"\n",
+    "    }\n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'output': [{'page_content': 'cats like mice',\n",
+       "   'metadata': {'owner_id': 'alice'},\n",
+       "   'type': 'Document'},\n",
+       "  {'page_content': 'cats like cheese',\n",
+       "   'metadata': {'owner_id': 'alice'},\n",
+       "   'type': 'Document'}],\n",
+       " 'callback_events': [],\n",
+       " 'metadata': {'run_id': '00000000-0000-0000-0000-000000000000'}}"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "response.json()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "You can also interact with this via the RemoteRunnable interface (to use in other chains)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langserve import RemoteRunnable\n",
+    "\n",
+    "remote_runnable = RemoteRunnable(\"http://localhost:8000/\", headers={\"Authorization\": f\"Bearer {token}\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(page_content='cats like mice', metadata={'owner_id': 'alice'}),\n",
+       " Document(page_content='cats like cheese', metadata={'owner_id': 'alice'})]"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "await remote_runnable.ainvoke(\"cat\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Login as John"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import requests\n",
+    "\n",
+    "response = requests.post(\"http://localhost:8000/token\", data={\"username\": \"john\", \"password\": \"secret2\"})\n",
+    "token = response.json()['access_token']\n",
+    "remote_runnable = RemoteRunnable(\"http://localhost:8000/\", headers={\"Authorization\": f\"Bearer {token}\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(page_content='i like walks by the ocean', metadata={'owner_id': 'john'}),\n",
+       " Document(page_content='dogs like sticks', metadata={'owner_id': 'john'}),\n",
+       " Document(page_content='my favorite food is cheese', metadata={'owner_id': 'john'})]"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "await remote_runnable.ainvoke(\"water\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/auth/per_req_config_modifier/server.py

@@ -0,0 +1,243 @@
+#!/usr/bin/env python
+"""Example that shows how to use `per_req_config_modifier`.
+
+This is a simple example that shows how to use configurable runnables with
+per request configuration modification to achieve behavior that's different
+depending on the user.
+
+You can build on these concepts to implement a more complex app:
+* Add endpoints that allow users to manage their documents.
+* Make a more complex runnable that does something with the retrieved documents; e.g.,
+  a conversational agent that responds to the user's input with the retrieved documents
+  (which are user specific documents).
+
+For authentication, we use a fake token that's the same as the username, adapting
+the following example from the FastAPI docs:
+
+https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/
+
+**ATTENTION**
+
+This example is not actually secure and should not be used in production.
+
+Once you understand how to use `per_req_config_modifier`, read through
+the FastAPI docs and implement proper auth:
+https://fastapi.tiangolo.com/tutorial/security/oauth2-jwt/
+
+
+**ATTENTION**
+
+This example does not integrate auth with OpenAPI, so the OpenAPI docs won't
+be able to help with authentication. This is currently a limitation
+if using `add_routes`. If you need this functionality, you can use
+the underlying `APIHandler` class directly, which affords maximal flexibility.
+"""
+from typing import Any, Dict, List, Optional, Union
+
+from fastapi import Depends, FastAPI, HTTPException, Request, status
+from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
+from langchain_community.embeddings.openai import OpenAIEmbeddings
+from langchain_community.vectorstores.chroma import Chroma
+from langchain_core.documents import Document
+from langchain_core.runnables import (
+    ConfigurableField,
+    RunnableConfig,
+    RunnableSerializable,
+)
+from langchain_core.vectorstores import VectorStore
+from typing_extensions import Annotated
+
+from langserve import add_routes
+from langserve.pydantic_v1 import BaseModel
+
+
+class User(BaseModel):
+    username: str
+    email: Union[str, None] = None
+    full_name: Union[str, None] = None
+    disabled: Union[bool, None] = None
+
+
+class UserInDB(User):
+    hashed_password: str
+
+
+oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")
+
+app = FastAPI()
+
+FAKE_USERS_DB = {
+    "alice": {
+        "username": "alice",
+        "full_name": "Alice Wonderson",
+        "email": "alice@example.com",
+        "hashed_password": "fakehashedsecret1",
+        "disabled": False,
+    },
+    "john": {
+        "username": "john",
+        "full_name": "John Doe",
+        "email": "johndoe@example.com",
+        "hashed_password": "fakehashedsecret2",
+        "disabled": False,
+    },
+    "bob": {
+        "username": "john",
+        "full_name": "John Doe",
+        "email": "johndoe@example.com",
+        "hashed_password": "fakehashedsecret3",
+        "disabled": True,
+    },
+}
+
+
+def _fake_hash_password(password: str) -> str:
+    """Fake a hashed password."""
+    return "fakehashed" + password
+
+
+def _get_user(db: dict, username: str) -> Union[UserInDB, None]:
+    if username in db:
+        user_dict = db[username]
+        return UserInDB(**user_dict)
+
+
+def _fake_decode_token(token: str) -> Union[User, None]:
+    # This doesn't provide any security at all
+    # Check the next version
+    user = _get_user(FAKE_USERS_DB, token)
+    return user
+
+
+@app.post("/token")
+async def login(form_data: Annotated[OAuth2PasswordRequestForm, Depends()]):
+    user_dict = FAKE_USERS_DB.get(form_data.username)
+    if not user_dict:
+        raise HTTPException(status_code=400, detail="Incorrect username or password")
+    user = UserInDB(**user_dict)
+    hashed_password = _fake_hash_password(form_data.password)
+    if not hashed_password == user.hashed_password:
+        raise HTTPException(status_code=400, detail="Incorrect username or password")
+
+    return {"access_token": user.username, "token_type": "bearer"}
+
+
+async def get_current_active_user_from_request(request: Request) -> User:
+    """Get the current active user from the request."""
+    token = await oauth2_scheme(request)
+    user = _fake_decode_token(token)
+    if not user:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid authentication credentials",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+    if user.disabled:
+        raise HTTPException(status_code=400, detail="Inactive user")
+    return user
+
+
+class PerUserVectorstore(RunnableSerializable):
+    """A custom runnable that returns a list of documents for the given user.
+
+    The runnable is configurable by the user, and the search results are
+    filtered by the user ID.
+    """
+
+    user_id: Optional[str]
+    vectorstore: VectorStore
+
+    class Config:
+        # Allow arbitrary types since VectorStore is an abstract interface
+        # and not a pydantic model
+        arbitrary_types_allowed = True
+
+    def _invoke(
+        self, input: str, config: Optional[RunnableConfig] = None, **kwargs: Any
+    ) -> List[Document]:
+        """Invoke the retriever."""
+        # WARNING: Verify documentation of underlying vectorstore to make
+        # sure that it actually uses filters.
+        # Highly recommended to use unit-tests to verify this behavior, as
+        # implementations can be different depending on the underlying vectorstore.
+        retriever = self.vectorstore.as_retriever(
+            search_kwargs={"filter": {"owner_id": self.user_id}}
+        )
+        return retriever.invoke(input, config=config)
+
+    def invoke(
+        self, input: str, config: Optional[RunnableConfig] = None, **kwargs
+    ) -> List[Document]:
+        """Add one to an integer."""
+        return self._call_with_config(self._invoke, input, config, **kwargs)
+
+
+async def per_req_config_modifier(config: Dict, request: Request) -> Dict:
+    """Modify the config for each request."""
+    user = await get_current_active_user_from_request(request)
+    config["configurable"] = {}
+    # Attention: Make sure that the user ID is over-ridden for each request.
+    # We should not be accepting a user ID from the user in this case!
+    config["configurable"]["user_id"] = user.username
+    return config
+
+
+vectorstore = Chroma(
+    collection_name="some_collection",
+    embedding_function=OpenAIEmbeddings(),
+)
+
+vectorstore.add_documents(
+    [
+        Document(
+            page_content="cats like cheese",
+            metadata={"owner_id": "alice"},
+        ),
+        Document(
+            page_content="cats like mice",
+            metadata={"owner_id": "alice"},
+        ),
+        Document(
+            page_content="dogs like sticks",
+            metadata={"owner_id": "john"},
+        ),
+        Document(
+            page_content="my favorite food is cheese",
+            metadata={"owner_id": "john"},
+        ),
+        Document(
+            page_content="i like walks by the ocean",
+            metadata={"owner_id": "john"},
+        ),
+        Document(
+            page_content="dogs like grass",
+            metadata={"owner_id": "bob"},
+        ),
+    ]
+)
+
+per_user_retriever = PerUserVectorstore(
+    user_id=None,  # Placeholder ID that will be replaced by the per_req_config_modifier
+    vectorstore=vectorstore,
+).configurable_fields(
+    # Attention: Make sure to override the user ID for each request in the
+    # per_req_config_modifier. This should not be client configurable.
+    user_id=ConfigurableField(
+        id="user_id",
+        name="User ID",
+        description="The user ID to use for the retriever.",
+    )
+)
+
+add_routes(
+    app,
+    per_user_retriever,
+    per_req_config_modifier=per_req_config_modifier,
+    enabled_endpoints=["invoke"],
+)
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=8000)

examples/chain/client.ipynb

@@ -1,191 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "# Client\n",
-    "\n",
-    "Demo of client interacting with the simple chain server, which deploys a chain that tells jokes about a particular topic."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "You can interact with this via API directly"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'output': {'content': \"Why don't scientists trust atoms when playing sports? \\n\\nBecause they make up everything!\",\n",
-       "  'additional_kwargs': {},\n",
-       "  'type': 'ai',\n",
-       "  'example': False}}"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import requests\n",
-    "\n",
-    "inputs = {\"input\": {\"topic\": \"sports\"}}\n",
-    "response = requests.post(\"http://localhost:8000/invoke\", json=inputs)\n",
-    "\n",
-    "response.json()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "You can also interact with this via the RemoteRunnable interface (to use in other chains)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "from langserve import RemoteRunnable\n",
-    "\n",
-    "remote_runnable = RemoteRunnable(\"http://localhost:8000/\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Remote runnable has the same interface as local runnables"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "response = await remote_runnable.ainvoke({\"topic\": \"sports\"})"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "The client can also execute langchain code synchronously, and pass in configs"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "[AIMessage(content='Why did the football coach go to the bank?\\n\\nBecause he wanted to get his quarterback!', additional_kwargs={}, example=False),\n",
-       " AIMessage(content='Why did the car bring a sweater to the race?\\n\\nBecause it wanted to have a \"car-digan\" finish!', additional_kwargs={}, example=False)]"
-      ]
-     },
-     "execution_count": 4,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "from langchain.schema.runnable.config import RunnableConfig\n",
-    "\n",
-    "remote_runnable.batch([{\"topic\": \"sports\"}, {\"topic\": \"cars\"}])"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "The server supports streaming (using HTTP server-side events), which can help interact with long responses in real time"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Ah, indulge me in this lighthearted endeavor, dear interlocutor! Allow me to regale you with a rather verbose jest concerning our hirsute friends of the wilderness, the bears!\n",
-      "\n",
-      "Once upon a time, in the vast expanse of a verdant forest, there existed a most erudite and sagacious bear, renowned for his prodigious intellect and unabated curiosity. This bear, with his inquisitive disposition, embarked on a quest to uncover the secrets of humor, for he believed that laughter possessed the power to unite and uplift the spirits of all creatures, great and small.\n",
-      "\n",
-      "Upon his journey, our erudite bear encountered a group of mischievous woodland creatures, who, captivated by his exalted intelligence, dared to challenge him to create a jest that would truly encompass the majestic essence of the bear. Our sagacious bear, never one to back down from a challenge, took a moment to ponder, his profound thoughts swirling amidst the verdant canopy above.\n",
-      "\n",
-      "After much contemplation, the bear delivered his jest, thusly: \"Pray, dear friends, envision a most estimable gathering of bears, replete with their formidable bulk and majestic presence. In this symposium of ursine brilliance, one bear, with a prodigious appetite, sauntered forth to procure his daily sustenance. Alas, upon reaching his intended destination, he encountered a dapper gentleman, clad in a most resplendent suit, hitherto unseen in the realm of the forest.\n",
-      "\n",
-      "The gentleman, possessing an air of sophistication, addressed the bear with an air of candor, remarking, 'Good sir, I must confess that your corporeal form inspires awe and admiration in equal measure. However, I beseech you, kindly abstain from consuming the berries that grow in this territory, for they possess a most deleterious effect upon the digestive systems of bears.'\n",
-      "\n",
-      "In response, the bear, known for his indomitable spirit, replied in a most eloquent manner, 'Dearest sir, I appreciate your concern and your eloquent admonition, yet I must humbly convey that the allure of these succulent berries is simply irresistible. The culinary satisfaction they bring far outweighs the potential discomfort they may inflict upon my digestive faculties. Therefore, I am compelled to disregard your sage counsel and indulge in their delectable essence.'\n",
-      "\n",
-      "And so, dear listener, the bear, driven by his insatiable hunger, proceeded to relish the berries with unmitigated gusto, heedless of the gentleman's cautions. After partaking in his feast, the bear, much to his chagrin, soon discovered the veracity of the gentleman's warning, as his digestive faculties embarked upon an unrestrained journey of turmoil and trepidation.\n",
-      "\n",
-      "In the aftermath of his ill-fated indulgence, the bear, with a countenance of utmost regret, turned to the gentleman and uttered, 'Verily, good sir, your counsel was indeed sagacious and prescient. I find myself ensnared in a maelstrom of gastrointestinal distress, beseeching the heavens for respite from this discomfort.'\n",
-      "\n",
-      "And thus, dear interlocutor, we find ourselves at the crux of this jest, whereupon the bear, in his most vulnerable state, beseeches the heavens for relief from his gastrointestinal plight. In this moment of levity, we are reminded that even the most erudite and sagacious among us can succumb to the allure of temptation, and the consequences that follow serve as a timeless lesson for all creatures within the realm of nature.\"\n",
-      "\n",
-      "Oh, the whimsy of the bear's gastronomic misadventure! May it serve as a reminder that, even amidst the grandeur of the natural world, we must exercise prudence and contemplate the ramifications of our actions."
-     ]
-    }
-   ],
-   "source": [
-    "async for chunk in remote_runnable.astream({\"topic\": \"bears, but super verbose\"}):\n",
-    "    print(chunk.content, end=\"\", flush=True)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.1"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}

examples/chain/server.py

@@ -1,67 +0,0 @@
-#!/usr/bin/env python
-"""Example LangChain server exposes a chain composed of a prompt and an LLM."""
-from fastapi import FastAPI
-from fastapi.middleware.cors import CORSMiddleware
-from langchain.chat_models import ChatOpenAI
-from langchain.prompts import PromptTemplate
-
-# from typing_extensions import TypedDict
-from langchain.pydantic_v1 import BaseModel
-from langchain.schema.output_parser import StrOutputParser
-from langchain.schema.runnable import ConfigurableField
-
-from langserve import add_routes
-
-model = ChatOpenAI(temperature=0.5).configurable_alternatives(
-    ConfigurableField(id="llm", name="LLM"),
-    high_temp=ChatOpenAI(temperature=0.9),
-    low_temp=ChatOpenAI(temperature=0.1, max_tokens=1),
-    default_key="medium_temp",
-)
-prompt = PromptTemplate.from_template(
-    "tell me a joke about {topic}."
-).configurable_fields(
-    template=ConfigurableField(
-        id="prompt",
-        name="Prompt",
-        description="The prompt to use. Must contain {topic}",
-    )
-)
-chain = prompt | model | StrOutputParser()
-
-app = FastAPI(
-    title="LangChain Server",
-    version="1.0",
-    description="Spin up a simple api server using Langchain's Runnable interfaces",
-)
-
-# Set all CORS enabled origins
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-    expose_headers=["*"],
-)
-
-
-# The input type is automatically inferred from the runnable
-# interface; however, if you want to override it, you can do so
-# by passing in the input_type argument to add_routes.
-class ChainInput(BaseModel):
-    """The input to the chain."""
-
-    topic: str
-
-
-add_routes(app, chain, input_type=ChainInput, config_keys=["configurable"])
-
-# Alternatively, you can rely on langchain's type inference
-# to infer the input type from the runnable interface.
-# add_routes(app, chain)
-
-if __name__ == "__main__":
-    import uvicorn
-
-    uvicorn.run(app, host="localhost", port=8000)

examples/chat_playground/legacy_input/server.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+"""Example of a simple chatbot that just passes current conversation
+state back and forth between server and client.
+"""
+from typing import List, Union
+
+from fastapi import FastAPI
+from langchain.chat_models import ChatAnthropic
+from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+
+from langserve import add_routes
+from langserve.pydantic_v1 import BaseModel, Field
+
+app = FastAPI(
+    title="LangChain Server",
+    version="1.0",
+    description="Spin up a simple api server using Langchain's Runnable interfaces",
+)
+
+
+# Declare a chain
+prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", "You are a helpful, professional assistant named Cob."),
+        MessagesPlaceholder(variable_name="messages"),
+        ("human", "{input}"),
+    ]
+)
+
+chain = prompt | ChatAnthropic(model="claude-2")
+
+
+class InputChat(BaseModel):
+    """Input for the chat endpoint."""
+
+    messages: List[Union[HumanMessage, AIMessage, SystemMessage]] = Field(
+        ...,
+        description="The chat messages representing the current conversation.",
+    )
+
+    input: str
+
+
+add_routes(
+    app,
+    chain.with_types(input_type=InputChat),
+    enable_feedback_endpoint=True,
+    enable_public_trace_link_endpoint=True,
+    playground_type="chat",
+)
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=8000)

examples/chat_playground/server.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+"""Example of a simple chatbot that just passes current conversation
+state back and forth between server and client.
+"""
+from typing import List, Union
+
+from fastapi import FastAPI
+from langchain.chat_models import ChatAnthropic
+from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+
+from langserve import add_routes
+from langserve.pydantic_v1 import BaseModel, Field
+
+app = FastAPI(
+    title="LangChain Server",
+    version="1.0",
+    description="Spin up a simple api server using Langchain's Runnable interfaces",
+)
+
+
+# Declare a chain
+prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", "You are a helpful, professional assistant named Cob."),
+        MessagesPlaceholder(variable_name="messages"),
+    ]
+)
+
+chain = prompt | ChatAnthropic(model="claude-2")
+
+
+class InputChat(BaseModel):
+    """Input for the chat endpoint."""
+
+    messages: List[Union[HumanMessage, AIMessage, SystemMessage]] = Field(
+        ...,
+        description="The chat messages representing the current conversation.",
+    )
+
+
+add_routes(
+    app,
+    chain.with_types(input_type=InputChat),
+    enable_feedback_endpoint=True,
+    enable_public_trace_link_endpoint=True,
+    playground_type="chat",
+)
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=8000)

examples/chat_with_persistence/client.ipynb

@@ -0,0 +1,277 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Chat History\n",
+    "\n",
+    "An example of a client interacting with a chatbot where message history is persisted on the backend."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import uuid\n",
+    "from langserve import RemoteRunnable\n",
+    "\n",
+    "chat = RemoteRunnable(\"http://localhost:8000/\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let's create a prompt composed of a system message and a human message."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "session_id = str(uuid.uuid4())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "AIMessage(content=\" Hello Eugene! My name is Claude. It's nice to meet another cat lover.\")"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "chat.invoke({\"human_input\": \"my name is eugene. i like cats. what is your name?\"}, {'configurable': { 'session_id': session_id } })"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "AIMessage(content=' You told me your name is Eugene.')"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "chat.invoke({\"human_input\": \"what was my name?\"}, {'configurable': { 'session_id': session_id } })"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "AIMessage(content=' You said you like cats.')"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "chat.invoke({\"human_input\": \"What animal do i like?\"}, {'configurable': { 'session_id': session_id } })"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      " Sure\n",
+      ",\n",
+      " I\n",
+      "'d\n",
+      " be\n",
+      " happy\n",
+      " to\n",
+      " count\n",
+      " to\n",
+      " 10\n",
+      ":\n",
+      "\n",
+      "\n",
+      "1\n",
+      ",\n",
+      " 2\n",
+      ",\n",
+      " 3\n",
+      ",\n",
+      " 4\n",
+      ",\n",
+      " 5\n",
+      ",\n",
+      " 6\n",
+      ",\n",
+      " 7\n",
+      ",\n",
+      " 8\n",
+      ",\n",
+      " 9\n",
+      ",\n",
+      " 10\n"
+     ]
+    }
+   ],
+   "source": [
+    "for chunk in chat.stream({'human_input': \"Can you count till 10?\"},  {'configurable': { 'session_id': session_id } }):\n",
+    "    print()\n",
+    "    print(chunk.content, end='', flush=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[1;39m[\n",
+      "  \u001b[1;39m{\n",
+      "    \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n",
+      "    \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n",
+      "      \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"my name is eugene. i like cats. what is your name?\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n",
+      "    \u001b[1;39m}\u001b[0m\u001b[1;39m\n",
+      "  \u001b[1;39m}\u001b[0m\u001b[1;39m,\n",
+      "  \u001b[1;39m{\n",
+      "    \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"ai\"\u001b[0m\u001b[1;39m,\n",
+      "    \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n",
+      "      \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\" Hello Eugene! My name is Claude. It's nice to meet another cat lover.\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"ai\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n",
+      "    \u001b[1;39m}\u001b[0m\u001b[1;39m\n",
+      "  \u001b[1;39m}\u001b[0m\u001b[1;39m,\n",
+      "  \u001b[1;39m{\n",
+      "    \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n",
+      "    \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n",
+      "      \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"what was my name?\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n",
+      "    \u001b[1;39m}\u001b[0m\u001b[1;39m\n",
+      "  \u001b[1;39m}\u001b[0m\u001b[1;39m,\n",
+      "  \u001b[1;39m{\n",
+      "    \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"ai\"\u001b[0m\u001b[1;39m,\n",
+      "    \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n",
+      "      \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\" You told me your name is Eugene.\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"ai\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n",
+      "    \u001b[1;39m}\u001b[0m\u001b[1;39m\n",
+      "  \u001b[1;39m}\u001b[0m\u001b[1;39m,\n",
+      "  \u001b[1;39m{\n",
+      "    \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n",
+      "    \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n",
+      "      \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"What animal do i like?\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n",
+      "    \u001b[1;39m}\u001b[0m\u001b[1;39m\n",
+      "  \u001b[1;39m}\u001b[0m\u001b[1;39m,\n",
+      "  \u001b[1;39m{\n",
+      "    \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"ai\"\u001b[0m\u001b[1;39m,\n",
+      "    \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n",
+      "      \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\" You said you like cats.\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"ai\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n",
+      "    \u001b[1;39m}\u001b[0m\u001b[1;39m\n",
+      "  \u001b[1;39m}\u001b[0m\u001b[1;39m,\n",
+      "  \u001b[1;39m{\n",
+      "    \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n",
+      "    \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n",
+      "      \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Can you count till 10?\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n",
+      "    \u001b[1;39m}\u001b[0m\u001b[1;39m\n",
+      "  \u001b[1;39m}\u001b[0m\u001b[1;39m,\n",
+      "  \u001b[1;39m{\n",
+      "    \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"AIMessageChunk\"\u001b[0m\u001b[1;39m,\n",
+      "    \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n",
+      "      \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\" Sure, I'd be happy to count to 10:\\n\\n1, 2, 3, 4, 5, 6, 7, 8, 9, 10\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"AIMessageChunk\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n",
+      "    \u001b[1;39m}\u001b[0m\u001b[1;39m\n",
+      "  \u001b[1;39m}\u001b[0m\u001b[1;39m\n",
+      "\u001b[1;39m]\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "!cat chat_histories/c7a327f3-5578-4fb7-a8f2-3082d7cb58cc.json | jq ."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/chat_with_persistence/server.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python
+"""Example of a chat server with persistence handled on the backend.
+
+For simplicity, we're using file storage here -- to avoid the need to set up
+a database. This is obviously not a good idea for a production environment,
+but will help us to demonstrate the RunnableWithMessageHistory interface.
+
+We'll use cookies to identify the user and/or session. This will help illustrate how to
+fetch configuration from the request.
+"""
+import re
+from pathlib import Path
+from typing import Callable, Union
+
+from fastapi import FastAPI, HTTPException
+from langchain.chat_models import ChatAnthropic
+from langchain.memory import FileChatMessageHistory
+from langchain_core.chat_history import BaseChatMessageHistory
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain_core.runnables.history import RunnableWithMessageHistory
+
+from langserve import add_routes
+from langserve.pydantic_v1 import BaseModel, Field
+
+
+def _is_valid_identifier(value: str) -> bool:
+    """Check if the session ID is in a valid format."""
+    # Use a regular expression to match the allowed characters
+    valid_characters = re.compile(r"^[a-zA-Z0-9-_]+$")
+    return bool(valid_characters.match(value))
+
+
+def create_session_factory(
+    base_dir: Union[str, Path],
+) -> Callable[[str], BaseChatMessageHistory]:
+    """Create a session ID factory that creates session IDs from a base dir.
+
+    Args:
+        base_dir: Base directory to use for storing the chat histories.
+
+    Returns:
+        A session ID factory that creates session IDs from a base path.
+    """
+    base_dir_ = Path(base_dir) if isinstance(base_dir, str) else base_dir
+    if not base_dir_.exists():
+        base_dir_.mkdir(parents=True)
+
+    def get_chat_history(session_id: str) -> FileChatMessageHistory:
+        """Get a chat history from a session ID."""
+        if not _is_valid_identifier(session_id):
+            raise HTTPException(
+                status_code=400,
+                detail=f"Session ID `{session_id}` is not in a valid format. "
+                "Session ID must only contain alphanumeric characters, "
+                "hyphens, and underscores.",
+            )
+        file_path = base_dir_ / f"{session_id}.json"
+        return FileChatMessageHistory(str(file_path))
+
+    return get_chat_history
+
+
+app = FastAPI(
+    title="LangChain Server",
+    version="1.0",
+    description="Spin up a simple api server using Langchain's Runnable interfaces",
+)
+
+
+# Declare a chain
+prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", "You're an assistant by the name of Bob."),
+        MessagesPlaceholder(variable_name="history"),
+        ("human", "{human_input}"),
+    ]
+)
+
+chain = prompt | ChatAnthropic(model="claude-2")
+
+
+class InputChat(BaseModel):
+    """Input for the chat endpoint."""
+
+    # The field extra defines a chat widget.
+    # As of 2024-02-05, this chat widget is not fully supported.
+    # It's included in documentation to show how it should be specified, but
+    # will not work until the widget is fully supported for history persistence
+    # on the backend.
+    human_input: str = Field(
+        ...,
+        description="The human input to the chat system.",
+        extra={"widget": {"type": "chat", "input": "human_input"}},
+    )
+
+
+chain_with_history = RunnableWithMessageHistory(
+    chain,
+    create_session_factory("chat_histories"),
+    input_messages_key="human_input",
+    history_messages_key="history",
+).with_types(input_type=InputChat)
+
+
+add_routes(
+    app,
+    chain_with_history,
+)
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=8000)

examples/chat_with_persistence_and_user/client.ipynb

@@ -0,0 +1,359 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Chat History\n",
+    "\n",
+    "Here we'll be interacting with a server that's exposing a chat bot with message history being persisted on the backend."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import uuid\n",
+    "from langserve import RemoteRunnable\n",
+    "\n",
+    "conversation_id = str(uuid.uuid4())\n",
+    "chat = RemoteRunnable(\"http://localhost:8000/\", cookies={\"user_id\": \"eugene\"})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let's create a prompt composed of a system message and a human message."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "AIMessage(content=\"Hello Eugene! I'm Bob, your virtual assistant. How can I assist you today?\")"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "chat.invoke({\"human_input\": \"my name is eugene. what is your name?\"}, {'configurable': { 'conversation_id': conversation_id } })"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "AIMessage(content='Your name is Eugene. Is there something specific you would like assistance with, Eugene?')"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "chat.invoke({\"human_input\": \"what was my name?\"}, {'configurable': { 'conversation_id': conversation_id } })"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Use different user but same conversation id"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "chat = RemoteRunnable(\"http://localhost:8000/\", cookies={\"user_id\": \"nuno\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "AIMessage(content=\"I apologize, but I don't have access to personal information about users. As an AI assistant, I prioritize user privacy and data protection.\")"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "chat.invoke({\"human_input\": \"what was my name?\"}, {'configurable': { 'conversation_id': conversation_id }})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "Of\n",
+      " course\n",
+      "!\n",
+      " Here\n",
+      " you\n",
+      " go\n",
+      ":\n",
+      "\n",
+      "\n",
+      "1\n",
+      ",\n",
+      " \n",
+      "2\n",
+      ",\n",
+      " \n",
+      "3\n",
+      ",\n",
+      " \n",
+      "4\n",
+      ",\n",
+      " \n",
+      "5\n",
+      ",\n",
+      " \n",
+      "6\n",
+      ",\n",
+      " \n",
+      "7\n",
+      ",\n",
+      " \n",
+      "8\n",
+      ",\n",
+      " \n",
+      "9\n",
+      ",\n",
+      " \n",
+      "10\n",
+      ".\n"
+     ]
+    }
+   ],
+   "source": [
+    "for chunk in chat.stream({'human_input': \"Can you count till 10?\"},  {'configurable': { 'conversation_id': conversation_id } }):\n",
+    "    print()\n",
+    "    print(chunk.content, end='', flush=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'cd8e5a55-0295-41cd-a885-775e0403fd25'"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "conversation_id"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[01;34mchat_histories/\u001b[0m\n",
+      "├── \u001b[01;34meugene\u001b[0m\n",
+      "│   └── cd8e5a55-0295-41cd-a885-775e0403fd25.json\n",
+      "└── \u001b[01;34mnuno\u001b[0m\n",
+      "    └── cd8e5a55-0295-41cd-a885-775e0403fd25.json\n",
+      "\n",
+      "2 directories, 2 files\n"
+     ]
+    }
+   ],
+   "source": [
+    "!tree chat_histories/"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[1;39m[\n",
+      "  \u001b[1;39m{\n",
+      "    \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n",
+      "    \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n",
+      "      \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"my name is eugene. what is your name?\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n",
+      "    \u001b[1;39m}\u001b[0m\u001b[1;39m\n",
+      "  \u001b[1;39m}\u001b[0m\u001b[1;39m,\n",
+      "  \u001b[1;39m{\n",
+      "    \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"ai\"\u001b[0m\u001b[1;39m,\n",
+      "    \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n",
+      "      \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Hello Eugene! I'm Bob, your virtual assistant. How can I assist you today?\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"ai\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n",
+      "    \u001b[1;39m}\u001b[0m\u001b[1;39m\n",
+      "  \u001b[1;39m}\u001b[0m\u001b[1;39m,\n",
+      "  \u001b[1;39m{\n",
+      "    \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n",
+      "    \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n",
+      "      \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"what was my name?\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n",
+      "    \u001b[1;39m}\u001b[0m\u001b[1;39m\n",
+      "  \u001b[1;39m}\u001b[0m\u001b[1;39m,\n",
+      "  \u001b[1;39m{\n",
+      "    \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"ai\"\u001b[0m\u001b[1;39m,\n",
+      "    \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n",
+      "      \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Your name is Eugene. Is there something specific you would like assistance with, Eugene?\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"ai\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n",
+      "    \u001b[1;39m}\u001b[0m\u001b[1;39m\n",
+      "  \u001b[1;39m}\u001b[0m\u001b[1;39m\n",
+      "\u001b[1;39m]\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "!cat chat_histories/eugene/cd8e5a55-0295-41cd-a885-775e0403fd25.json | jq ."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[1;39m[\n",
+      "  \u001b[1;39m{\n",
+      "    \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n",
+      "    \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n",
+      "      \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"what was my name?\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n",
+      "    \u001b[1;39m}\u001b[0m\u001b[1;39m\n",
+      "  \u001b[1;39m}\u001b[0m\u001b[1;39m,\n",
+      "  \u001b[1;39m{\n",
+      "    \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"ai\"\u001b[0m\u001b[1;39m,\n",
+      "    \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n",
+      "      \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"I apologize, but I don't have access to personal information about users. As an AI assistant, I prioritize user privacy and data protection.\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"ai\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n",
+      "    \u001b[1;39m}\u001b[0m\u001b[1;39m\n",
+      "  \u001b[1;39m}\u001b[0m\u001b[1;39m,\n",
+      "  \u001b[1;39m{\n",
+      "    \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n",
+      "    \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n",
+      "      \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Can you count till 10?\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"human\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n",
+      "    \u001b[1;39m}\u001b[0m\u001b[1;39m\n",
+      "  \u001b[1;39m}\u001b[0m\u001b[1;39m,\n",
+      "  \u001b[1;39m{\n",
+      "    \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"AIMessageChunk\"\u001b[0m\u001b[1;39m,\n",
+      "    \u001b[0m\u001b[34;1m\"data\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n",
+      "      \u001b[0m\u001b[34;1m\"content\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"Of course! Here you go:\\n\\n1, 2, 3, 4, 5, 6, 7, 8, 9, 10.\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"additional_kwargs\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{}\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"type\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"AIMessageChunk\"\u001b[0m\u001b[1;39m,\n",
+      "      \u001b[0m\u001b[34;1m\"example\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;39mfalse\u001b[0m\u001b[1;39m\n",
+      "    \u001b[1;39m}\u001b[0m\u001b[1;39m\n",
+      "  \u001b[1;39m}\u001b[0m\u001b[1;39m\n",
+      "\u001b[1;39m]\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "!cat chat_histories/nuno/cd8e5a55-0295-41cd-a885-775e0403fd25.json | jq ."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/chat_with_persistence_and_user/server.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python
+"""Example of a chat server with persistence handled on the backend.
+
+For simplicity, we're using file storage here -- to avoid the need to set up
+a database. This is obviously not a good idea for a production environment,
+but will help us to demonstrate the RunnableWithMessageHistory interface.
+
+We'll use cookies to identify the user. This will help illustrate how to
+fetch configuration from the request.
+"""
+import re
+from pathlib import Path
+from typing import Any, Callable, Dict, Union
+
+from fastapi import FastAPI, HTTPException, Request
+from langchain.chat_models import ChatOpenAI
+from langchain.memory import FileChatMessageHistory
+from langchain.schema.runnable.utils import ConfigurableFieldSpec
+from langchain_core import __version__
+from langchain_core.chat_history import BaseChatMessageHistory
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain_core.runnables.history import RunnableWithMessageHistory
+from typing_extensions import TypedDict
+
+from langserve import add_routes
+
+# Define the minimum required version as (0, 1, 0)
+# Earlier versions did not allow specifying custom config fields in
+# RunnableWithMessageHistory.
+MIN_VERSION_LANGCHAIN_CORE = (0, 1, 0)
+
+# Split the version string by "." and convert to integers
+LANGCHAIN_CORE_VERSION = tuple(map(int, __version__.split(".")))
+
+if LANGCHAIN_CORE_VERSION < MIN_VERSION_LANGCHAIN_CORE:
+    raise RuntimeError(
+        f"Minimum required version of langchain-core is {MIN_VERSION_LANGCHAIN_CORE}, "
+        f"but found {LANGCHAIN_CORE_VERSION}"
+    )
+
+
+def _is_valid_identifier(value: str) -> bool:
+    """Check if the value is a valid identifier."""
+    # Use a regular expression to match the allowed characters
+    valid_characters = re.compile(r"^[a-zA-Z0-9-_]+$")
+    return bool(valid_characters.match(value))
+
+
+def create_session_factory(
+    base_dir: Union[str, Path],
+) -> Callable[[str], BaseChatMessageHistory]:
+    """Create a factory that can retrieve chat histories.
+
+    The chat histories are keyed by user ID and conversation ID.
+
+    Args:
+        base_dir: Base directory to use for storing the chat histories.
+
+    Returns:
+        A factory that can retrieve chat histories keyed by user ID and conversation ID.
+    """
+    base_dir_ = Path(base_dir) if isinstance(base_dir, str) else base_dir
+    if not base_dir_.exists():
+        base_dir_.mkdir(parents=True)
+
+    def get_chat_history(user_id: str, conversation_id: str) -> FileChatMessageHistory:
+        """Get a chat history from a user id and conversation id."""
+        if not _is_valid_identifier(user_id):
+            raise ValueError(
+                f"User ID {user_id} is not in a valid format. "
+                "User ID must only contain alphanumeric characters, "
+                "hyphens, and underscores."
+                "Please include a valid cookie in the request headers called 'user-id'."
+            )
+        if not _is_valid_identifier(conversation_id):
+            raise ValueError(
+                f"Conversation ID {conversation_id} is not in a valid format. "
+                "Conversation ID must only contain alphanumeric characters, "
+                "hyphens, and underscores. Please provide a valid conversation id "
+                "via config. For example, "
+                "chain.invoke(.., {'configurable': {'conversation_id': '123'}})"
+            )
+
+        user_dir = base_dir_ / user_id
+        if not user_dir.exists():
+            user_dir.mkdir(parents=True)
+        file_path = user_dir / f"{conversation_id}.json"
+        return FileChatMessageHistory(str(file_path))
+
+    return get_chat_history
+
+
+app = FastAPI(
+    title="LangChain Server",
+    version="1.0",
+    description="Spin up a simple api server using Langchain's Runnable interfaces",
+)
+
+
+def _per_request_config_modifier(
+    config: Dict[str, Any], request: Request
+) -> Dict[str, Any]:
+    """Update the config"""
+    config = config.copy()
+    configurable = config.get("configurable", {})
+    # Look for a cookie named "user_id"
+    user_id = request.cookies.get("user_id", None)
+
+    if user_id is None:
+        raise HTTPException(
+            status_code=400,
+            detail="No user id found. Please set a cookie named 'user_id'.",
+        )
+
+    configurable["user_id"] = user_id
+    config["configurable"] = configurable
+    return config
+
+
+# Declare a chain
+prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", "You're an assistant by the name of Bob."),
+        MessagesPlaceholder(variable_name="history"),
+        ("human", "{human_input}"),
+    ]
+)
+
+chain = prompt | ChatOpenAI()
+
+
+class InputChat(TypedDict):
+    """Input for the chat endpoint."""
+
+    human_input: str
+    """Human input"""
+
+
+chain_with_history = RunnableWithMessageHistory(
+    chain,
+    create_session_factory("chat_histories"),
+    input_messages_key="human_input",
+    history_messages_key="history",
+    history_factory_config=[
+        ConfigurableFieldSpec(
+            id="user_id",
+            annotation=str,
+            name="User ID",
+            description="Unique identifier for the user.",
+            default="",
+            is_shared=True,
+        ),
+        ConfigurableFieldSpec(
+            id="conversation_id",
+            annotation=str,
+            name="Conversation ID",
+            description="Unique identifier for the conversation.",
+            default="",
+            is_shared=True,
+        ),
+    ],
+).with_types(input_type=InputChat)
+
+
+add_routes(
+    app,
+    chain_with_history,
+    per_req_config_modifier=_per_request_config_modifier,
+    # Disable playground and batch
+    # 1) Playground we're passing information via headers, which is not supported via
+    #    the playground right now.
+    # 2) Disable batch to avoid users being confused. Batch will work fine
+    #    as long as users invoke it with multiple configs appropriately, but
+    #    without validation users are likely going to forget to do that.
+    #    In addition, there's likely little sense in support batch for a chatbot.
+    disabled_endpoints=["playground", "batch"],
+)
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=8000)

examples/configurable_agent_executor/client.ipynb

@@ -0,0 +1,770 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Client\n",
+    "\n",
+    "Demo of a client interacting with a custom runnable executor that supports configuration.\n",
+    "\n",
+    "This server does not support invoke or batch! only stream and astream log! (see backend code.)\n",
+    "\n",
+    "The underlying backend code is just a demo in this case -- it's working around an existing bug, but uses \n",
+    "the opportunity to show how to create custom runnables."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can interact with this via API directly"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "event: metadata\n",
+      "data: {\"run_id\": \"5e6ce60a-95c4-4fe5-8c4a-ec1d347afd83\"}\n",
+      "\n",
+      "event: data\n",
+      "data: {\"actions\":[{\"tool\":\"get_eugene_thoughts\",\"tool_input\":{\"query\":\"cats\"},\"log\":\"\\nInvoking: `get_eugene_thoughts` with `{'query': 'cats'}`\\n\\n\\n\",\"type\":\"AgentActionMessageLog\",\"message_log\":[{\"content\":\"\",\"additional_kwargs\":{\"function_call\":{\"name\":\"get_eugene_thoughts\",\"arguments\":\"{\\n  \\\"query\\\": \\\"cats\\\"\\n}\"}},\"type\":\"ai\",\"example\":false}]}],\"messages\":[{\"content\":\"\",\"additional_kwargs\":{\"function_call\":{\"name\":\"get_eugene_thoughts\",\"arguments\":\"{\\n  \\\"query\\\": \\\"cats\\\"\\n}\"}},\"type\":\"ai\",\"example\":false}]}\n",
+      "\n",
+      "event: data\n",
+      "data: {\"steps\":[{\"action\":{\"tool\":\"get_eugene_thoughts\",\"tool_input\":{\"query\":\"cats\"},\"log\":\"\\nInvoking: `get_eugene_thoughts` with `{'query': 'cats'}`\\n\\n\\n\",\"type\":\"AgentActionMessageLog\",\"message_log\":[{\"content\":\"\",\"additional_kwargs\":{\"function_call\":{\"name\":\"get_eugene_thoughts\",\"arguments\":\"{\\n  \\\"query\\\": \\\"cats\\\"\\n}\"}},\"type\":\"ai\",\"example\":false}]},\"observation\":[{\"page_content\":\"cats like fish\",\"metadata\":{},\"type\":\"Document\"},{\"page_content\":\"dogs like sticks\",\"metadata\":{},\"type\":\"Document\"}]}],\"messages\":[{\"content\":\"[Document(page_content='cats like fish'), Document(page_content='dogs like sticks')]\",\"additional_kwargs\":{},\"type\":\"function\",\"name\":\"get_eugene_thoughts\"}]}\n",
+      "\n",
+      "event: data\n",
+      "data: {\"output\":\"Eugene thinks that cats like fish.\",\"messages\":[{\"content\":\"Eugene thinks that cats like fish.\",\"additional_kwargs\":{},\"type\":\"ai\",\"example\":false}]}\n",
+      "\n",
+      "event: end\n",
+      "\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "import requests\n",
+    "\n",
+    "inputs = {\"input\": {\"input\": \"what does eugene think of cats?\"}}\n",
+    "response = requests.post(\"http://localhost:8000/stream\", json=inputs)\n",
+    "\n",
+    "print(response.text)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can also interact with this via the RemoteRunnable interface (to use in other chains)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langserve import RemoteRunnable\n",
+    "\n",
+    "remote_runnable = RemoteRunnable(\"http://localhost:8000/\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Remote runnable has the same interface as local runnables"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'output': 'Hello! How can I assist you today?', 'messages': [AIMessage(content='Hello! How can I assist you today?')]}\n"
+     ]
+    }
+   ],
+   "source": [
+    "async for chunk in remote_runnable.astream({\"input\": \"hi!\"}):\n",
+    "    print(chunk)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "RunLogPatch({'op': 'replace',\n",
+      "  'path': '',\n",
+      "  'value': {'final_output': None,\n",
+      "            'id': '9f415b49-ba69-4fdf-9b9c-5ccc1805487f',\n",
+      "            'logs': {},\n",
+      "            'streamed_output': []}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableSequence',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '95a0ba72-9511-4fff-8b8c-410e7108ee60',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'RunnableSequence',\n",
+      "            'start_time': '2024-01-06T03:12:42.213+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': [],\n",
+      "            'type': 'chain'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableParallel<input,agent_scratchpad>',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': 'a0afa5a6-a408-4a2d-83f8-8e4c8193f963',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'RunnableParallel<input,agent_scratchpad>',\n",
+      "            'start_time': '2024-01-06T03:12:42.214+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:1'],\n",
+      "            'type': 'chain'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/<lambda>',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '2aed7d13-2d0b-4c37-a4b3-d3f266bbf7e6',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': '<lambda>',\n",
+      "            'start_time': '2024-01-06T03:12:42.215+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['map:key:input'],\n",
+      "            'type': 'chain'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/<lambda>:2',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '9328b5cf-fd7b-48fe-affb-2973643189e9',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': '<lambda>',\n",
+      "            'start_time': '2024-01-06T03:12:42.215+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['map:key:agent_scratchpad'],\n",
+      "            'type': 'chain'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/<lambda>/final_output',\n",
+      "  'value': {'output': 'what does eugene think about cats?'}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/<lambda>/end_time',\n",
+      "  'value': '2024-01-06T03:12:42.217+00:00'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/<lambda>:2/final_output',\n",
+      "  'value': {'output': []}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/<lambda>:2/end_time',\n",
+      "  'value': '2024-01-06T03:12:42.217+00:00'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableParallel<input,agent_scratchpad>/final_output',\n",
+      "  'value': {'agent_scratchpad': [],\n",
+      "            'input': 'what does eugene think about cats?'}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/RunnableParallel<input,agent_scratchpad>/end_time',\n",
+      "  'value': '2024-01-06T03:12:42.218+00:00'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatPromptTemplate',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': 'a08bec71-bf7f-46c5-b082-14f7bff421cd',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'ChatPromptTemplate',\n",
+      "            'start_time': '2024-01-06T03:12:42.219+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:2'],\n",
+      "            'type': 'prompt'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatPromptTemplate/final_output',\n",
+      "  'value': {'messages': [SystemMessage(content='You are a helpful assistant.'),\n",
+      "                         HumanMessage(content='what does eugene think about cats?')]}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatPromptTemplate/end_time',\n",
+      "  'value': '2024-01-06T03:12:42.219+00:00'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/LLM',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': 'bf859261-467b-44b3-84fb-80d8c9e9dff2',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'LLM',\n",
+      "            'start_time': '2024-01-06T03:12:42.221+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:3'],\n",
+      "            'type': 'llm'}})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': ''}})})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': '{\\n'}})})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': ' '}})})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': ' \"'}})})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': 'query'}})})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': '\":'}})})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': ' \"'}})})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': 'cats'}})})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': '\"\\n'}})})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': '}'}})})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/LLM/final_output',\n",
+      "  'value': LLMResult(generations=[[ChatGeneration(generation_info={'finish_reason': 'function_call'}, message=AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}}))]], llm_output=None, run=None)},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/end_time',\n",
+      "  'value': '2024-01-06T03:12:42.806+00:00'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/OpenAIFunctionsAgentOutputParser',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': 'd635ccbb-1db3-4515-b1ca-0c14752d361a',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'OpenAIFunctionsAgentOutputParser',\n",
+      "            'start_time': '2024-01-06T03:12:42.807+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:4'],\n",
+      "            'type': 'parser'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/OpenAIFunctionsAgentOutputParser/final_output',\n",
+      "  'value': AgentActionMessageLog(tool='get_eugene_thoughts', tool_input={'query': 'cats'}, log=\"\\nInvoking: `get_eugene_thoughts` with `{'query': 'cats'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})])},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/OpenAIFunctionsAgentOutputParser/end_time',\n",
+      "  'value': '2024-01-06T03:12:42.809+00:00'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableSequence/final_output',\n",
+      "  'value': {'output': AgentActionMessageLog(tool='get_eugene_thoughts', tool_input={'query': 'cats'}, log=\"\\nInvoking: `get_eugene_thoughts` with `{'query': 'cats'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})])}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/RunnableSequence/end_time',\n",
+      "  'value': '2024-01-06T03:12:42.809+00:00'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': {'actions': [AgentActionMessageLog(tool='get_eugene_thoughts', tool_input={'query': 'cats'}, log=\"\\nInvoking: `get_eugene_thoughts` with `{'query': 'cats'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})])],\n",
+      "            'messages': [AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})]}},\n",
+      " {'op': 'replace',\n",
+      "  'path': '/final_output',\n",
+      "  'value': {'actions': [AgentActionMessageLog(tool='get_eugene_thoughts', tool_input={'query': 'cats'}, log=\"\\nInvoking: `get_eugene_thoughts` with `{'query': 'cats'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})])],\n",
+      "            'messages': [AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})]}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/get_eugene_thoughts',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '5eecdbe8-3374-4f0e-8ae8-d8a7e2394ae1',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'get_eugene_thoughts',\n",
+      "            'start_time': '2024-01-06T03:12:42.810+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': [],\n",
+      "            'type': 'tool'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/get_eugene_thoughts/final_output',\n",
+      "  'value': {'output': \"[Document(page_content='cats like fish'), \"\n",
+      "                      \"Document(page_content='dogs like sticks')]\"}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/get_eugene_thoughts/end_time',\n",
+      "  'value': '2024-01-06T03:12:43.095+00:00'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': {'messages': [FunctionMessage(content=\"[Document(page_content='cats like fish'), Document(page_content='dogs like sticks')]\", name='get_eugene_thoughts')],\n",
+      "            'steps': [{'action': AgentActionMessageLog(tool='get_eugene_thoughts', tool_input={'query': 'cats'}, log=\"\\nInvoking: `get_eugene_thoughts` with `{'query': 'cats'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})]),\n",
+      "                       'observation': [Document(page_content='cats like fish'),\n",
+      "                                       Document(page_content='dogs like sticks')]}]}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/final_output/steps',\n",
+      "  'value': [{'action': AgentActionMessageLog(tool='get_eugene_thoughts', tool_input={'query': 'cats'}, log=\"\\nInvoking: `get_eugene_thoughts` with `{'query': 'cats'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})]),\n",
+      "             'observation': [Document(page_content='cats like fish'),\n",
+      "                             Document(page_content='dogs like sticks')]}]},\n",
+      " {'op': 'add',\n",
+      "  'path': '/final_output/messages/1',\n",
+      "  'value': FunctionMessage(content=\"[Document(page_content='cats like fish'), Document(page_content='dogs like sticks')]\", name='get_eugene_thoughts')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableSequence:2',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '89bf8195-df48-4fde-81b0-9e1982051caa',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'RunnableSequence',\n",
+      "            'start_time': '2024-01-06T03:12:43.097+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': [],\n",
+      "            'type': 'chain'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableParallel<input,agent_scratchpad>:2',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '567c178a-4897-4865-af27-7122432df7bd',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'RunnableParallel<input,agent_scratchpad>',\n",
+      "            'start_time': '2024-01-06T03:12:43.098+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:1'],\n",
+      "            'type': 'chain'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/<lambda>:3',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '03530072-e5b2-4d7e-b875-f77e3d47481b',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': '<lambda>',\n",
+      "            'start_time': '2024-01-06T03:12:43.099+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['map:key:input'],\n",
+      "            'type': 'chain'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/<lambda>:4',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': 'c554e615-1b70-4dc1-a4aa-aa09ccdacb09',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': '<lambda>',\n",
+      "            'start_time': '2024-01-06T03:12:43.099+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['map:key:agent_scratchpad'],\n",
+      "            'type': 'chain'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/<lambda>:3/final_output',\n",
+      "  'value': {'output': 'what does eugene think about cats?'}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/<lambda>:3/end_time',\n",
+      "  'value': '2024-01-06T03:12:43.100+00:00'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/<lambda>:4/final_output',\n",
+      "  'value': {'output': [AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}}),\n",
+      "                       FunctionMessage(content=\"[Document(page_content='cats like fish'), Document(page_content='dogs like sticks')]\", name='get_eugene_thoughts')]}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/<lambda>:4/end_time',\n",
+      "  'value': '2024-01-06T03:12:43.100+00:00'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableParallel<input,agent_scratchpad>:2/final_output',\n",
+      "  'value': {'agent_scratchpad': [AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}}),\n",
+      "                                 FunctionMessage(content=\"[Document(page_content='cats like fish'), Document(page_content='dogs like sticks')]\", name='get_eugene_thoughts')],\n",
+      "            'input': 'what does eugene think about cats?'}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/RunnableParallel<input,agent_scratchpad>:2/end_time',\n",
+      "  'value': '2024-01-06T03:12:43.101+00:00'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatPromptTemplate:2',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '724c5bf8-d475-45ec-a69a-a4f188dc405f',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'ChatPromptTemplate',\n",
+      "            'start_time': '2024-01-06T03:12:43.101+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:2'],\n",
+      "            'type': 'prompt'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatPromptTemplate:2/final_output',\n",
+      "  'value': {'messages': [SystemMessage(content='You are a helpful assistant.'),\n",
+      "                         HumanMessage(content='what does eugene think about cats?'),\n",
+      "                         AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}}),\n",
+      "                         FunctionMessage(content=\"[Document(page_content='cats like fish'), Document(page_content='dogs like sticks')]\", name='get_eugene_thoughts')]}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatPromptTemplate:2/end_time',\n",
+      "  'value': '2024-01-06T03:12:43.102+00:00'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/LLM:2',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '4542ae99-f854-41be-9669-41d8cd7dfb24',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'LLM',\n",
+      "            'start_time': '2024-01-06T03:12:43.103+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:3'],\n",
+      "            'type': 'llm'}})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': 'E'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='E')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': 'ug'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='ug')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': 'ene'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='ene')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': ' thinks'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' thinks')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': ' that'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' that')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': ' cats'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' cats')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': ' like'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' like')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': ' fish'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' fish')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': '.'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='.')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/LLM:2/final_output',\n",
+      "  'value': LLMResult(generations=[[ChatGeneration(text='Eugene thinks that cats like fish.', generation_info={'finish_reason': 'stop'}, message=AIMessage(content='Eugene thinks that cats like fish.'))]], llm_output=None, run=None)},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/end_time',\n",
+      "  'value': '2024-01-06T03:12:43.776+00:00'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/OpenAIFunctionsAgentOutputParser:2',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '281c8ae8-dbc6-433f-86c8-20b86b3c8eee',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'OpenAIFunctionsAgentOutputParser',\n",
+      "            'start_time': '2024-01-06T03:12:43.776+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:4'],\n",
+      "            'type': 'parser'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/OpenAIFunctionsAgentOutputParser:2/final_output',\n",
+      "  'value': AgentFinish(return_values={'output': 'Eugene thinks that cats like fish.'}, log='Eugene thinks that cats like fish.')},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/OpenAIFunctionsAgentOutputParser:2/end_time',\n",
+      "  'value': '2024-01-06T03:12:43.777+00:00'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableSequence:2/final_output',\n",
+      "  'value': {'output': AgentFinish(return_values={'output': 'Eugene thinks that cats like fish.'}, log='Eugene thinks that cats like fish.')}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/RunnableSequence:2/end_time',\n",
+      "  'value': '2024-01-06T03:12:43.778+00:00'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': {'messages': [AIMessage(content='Eugene thinks that cats like fish.')],\n",
+      "            'output': 'Eugene thinks that cats like fish.'}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/final_output/output',\n",
+      "  'value': 'Eugene thinks that cats like fish.'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/final_output/messages/2',\n",
+      "  'value': AIMessage(content='Eugene thinks that cats like fish.')})\n"
+     ]
+    }
+   ],
+   "source": [
+    "async for chunk in remote_runnable.astream_log({\"input\": \"what does eugene think about cats?\"}):\n",
+    "    print(chunk)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "RunLogPatch({'op': 'replace',\n",
+      "  'path': '',\n",
+      "  'value': {'final_output': None,\n",
+      "            'id': '51c65021-1b40-4a45-81b3-95fc5c5b545a',\n",
+      "            'logs': {},\n",
+      "            'streamed_output': []}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/LLM',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '7a34b2a3-d6c0-4a0b-9939-2bde70a98958',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'LLM',\n",
+      "            'start_time': '2024-01-06T03:12:43.812+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:3'],\n",
+      "            'type': 'llm'}})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': ''}})})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': '{\\n'}})})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': ' '}})})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': ' \"'}})})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': 'query'}})})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': '\":'}})})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': ' \"'}})})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': 'cats'}})})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': '\"\\n'}})})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='', additional_kwargs={'function_call': {'arguments': '}'}})})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/LLM/final_output',\n",
+      "  'value': LLMResult(generations=[[ChatGeneration(generation_info={'finish_reason': 'function_call'}, message=AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}}))]], llm_output=None, run=None)},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM/end_time',\n",
+      "  'value': '2024-01-06T03:12:44.588+00:00'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': {'actions': [AgentActionMessageLog(tool='get_eugene_thoughts', tool_input={'query': 'cats'}, log=\"\\nInvoking: `get_eugene_thoughts` with `{'query': 'cats'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})])],\n",
+      "            'messages': [AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})]}},\n",
+      " {'op': 'replace',\n",
+      "  'path': '/final_output',\n",
+      "  'value': {'actions': [AgentActionMessageLog(tool='get_eugene_thoughts', tool_input={'query': 'cats'}, log=\"\\nInvoking: `get_eugene_thoughts` with `{'query': 'cats'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})])],\n",
+      "            'messages': [AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})]}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': {'messages': [FunctionMessage(content=\"[Document(page_content='cats like fish'), Document(page_content='dogs like sticks')]\", name='get_eugene_thoughts')],\n",
+      "            'steps': [{'action': AgentActionMessageLog(tool='get_eugene_thoughts', tool_input={'query': 'cats'}, log=\"\\nInvoking: `get_eugene_thoughts` with `{'query': 'cats'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})]),\n",
+      "                       'observation': [Document(page_content='cats like fish'),\n",
+      "                                       Document(page_content='dogs like sticks')]}]}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/final_output/steps',\n",
+      "  'value': [{'action': AgentActionMessageLog(tool='get_eugene_thoughts', tool_input={'query': 'cats'}, log=\"\\nInvoking: `get_eugene_thoughts` with `{'query': 'cats'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'name': 'get_eugene_thoughts', 'arguments': '{\\n  \"query\": \"cats\"\\n}'}})]),\n",
+      "             'observation': [Document(page_content='cats like fish'),\n",
+      "                             Document(page_content='dogs like sticks')]}]},\n",
+      " {'op': 'add',\n",
+      "  'path': '/final_output/messages/1',\n",
+      "  'value': FunctionMessage(content=\"[Document(page_content='cats like fish'), Document(page_content='dogs like sticks')]\", name='get_eugene_thoughts')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/LLM:2',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '8ff3b248-fb7e-4305-bd57-54e21a85bacb',\n",
+      "            'metadata': {'__langserve_endpoint': 'stream_log',\n",
+      "                         '__langserve_version': '0.0.37',\n",
+      "                         '__useragent': 'python-httpx/0.25.2'},\n",
+      "            'name': 'LLM',\n",
+      "            'start_time': '2024-01-06T03:12:44.756+00:00',\n",
+      "            'streamed_output': [],\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:3'],\n",
+      "            'type': 'llm'}})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': 'E'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='E')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': 'ug'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='ug')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': 'ene'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='ene')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': ' thinks'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' thinks')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': ' that'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' that')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': ' cats'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' cats')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': ' like'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' like')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': ' fish'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' fish')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': '.'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='.')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/LLM:2/streamed_output_str/-', 'value': ''},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/LLM:2/final_output',\n",
+      "  'value': LLMResult(generations=[[ChatGeneration(text='Eugene thinks that cats like fish.', generation_info={'finish_reason': 'stop'}, message=AIMessage(content='Eugene thinks that cats like fish.'))]], llm_output=None, run=None)},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/LLM:2/end_time',\n",
+      "  'value': '2024-01-06T03:12:45.171+00:00'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': {'messages': [AIMessage(content='Eugene thinks that cats like fish.')],\n",
+      "            'output': 'Eugene thinks that cats like fish.'}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/final_output/output',\n",
+      "  'value': 'Eugene thinks that cats like fish.'},\n",
+      " {'op': 'add',\n",
+      "  'path': '/final_output/messages/2',\n",
+      "  'value': AIMessage(content='Eugene thinks that cats like fish.')})\n"
+     ]
+    }
+   ],
+   "source": [
+    "async for chunk in remote_runnable.astream_log({\"input\": \"what does eugene think about cats?\"}, include_names=[\"LLM\"]):\n",
+    "    print(chunk)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/configurable_agent_executor/server.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python
+"""An example that shows how to create a custom agent executor like Runnable.
+
+At the time of writing, there is a bug in the current AgentExecutor that
+prevents it from correctly propagating configuration of the underlying
+runnable. While that bug should be fixed, this is an example shows
+how to create a more complex custom runnable.
+
+Please see documentation for custom agent streaming here:
+
+https://python.langchain.com/docs/modules/agents/how_to/streaming#stream-tokens
+
+**ATTENTION**
+To support streaming individual tokens you will need to manually set the streaming=True
+on the LLM and use the stream_log endpoint rather than stream endpoint.
+"""
+from typing import Any, AsyncIterator, Dict, List, Optional, cast
+
+from fastapi import FastAPI
+from langchain.agents import AgentExecutor, tool
+from langchain.agents.format_scratchpad import format_to_openai_functions
+from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser
+from langchain.chat_models import ChatOpenAI
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain.pydantic_v1 import BaseModel
+from langchain.tools.render import format_tool_to_openai_function
+from langchain.vectorstores import FAISS
+from langchain_core.runnables import (
+    ConfigurableField,
+    ConfigurableFieldSpec,
+    Runnable,
+    RunnableConfig,
+)
+from langchain_core.runnables.utils import Input, Output
+
+from langserve import add_routes
+
+vectorstore = FAISS.from_texts(
+    ["cats like fish", "dogs like sticks"], embedding=OpenAIEmbeddings()
+)
+retriever = vectorstore.as_retriever()
+
+
+@tool
+def get_eugene_thoughts(query: str) -> list:
+    """Returns Eugene's thoughts on a topic."""
+    return retriever.get_relevant_documents(query)
+
+
+tools = [get_eugene_thoughts]
+
+prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", "You are a helpful assistant."),
+        ("user", "{input}"),
+        MessagesPlaceholder(variable_name="agent_scratchpad"),
+    ]
+)
+
+# We need to set streaming=True on the LLM to support streaming individual tokens.
+# when using the stream_log endpoint.
+# .stream for agents streams action observation pairs not individual tokens.
+llm = ChatOpenAI(temperature=0, streaming=True).configurable_fields(
+    temperature=ConfigurableField(
+        id="llm_temperature",
+        name="LLM Temperature",
+        description="The temperature of the LLM",
+    )
+)
+
+llm_with_tools = llm.bind(
+    functions=[format_tool_to_openai_function(t) for t in tools]
+).with_config({"run_name": "LLM"})
+
+
+class CustomAgentExecutor(Runnable):
+    """A custom runnable that will be used by the agent executor."""
+
+    def __init__(self, **kwargs):
+        """Initialize the runnable."""
+        super().__init__(**kwargs)
+        self.agent = (
+            {
+                "input": lambda x: x["input"],
+                "agent_scratchpad": lambda x: format_to_openai_functions(
+                    x["intermediate_steps"]
+                ),
+            }
+            | prompt
+            | llm_with_tools
+            | OpenAIFunctionsAgentOutputParser()
+        )
+
+    def invoke(self, input: Input, config: Optional[RunnableConfig] = None) -> Output:
+        """Will not be used."""
+        raise NotImplementedError()
+
+    @property
+    def config_specs(self) -> List[ConfigurableFieldSpec]:
+        return self.agent.config_specs
+
+    async def astream(
+        self,
+        input: Input,
+        config: Optional[RunnableConfig] = None,
+        **kwargs: Optional[Any],
+    ) -> AsyncIterator[Output]:
+        """Stream the agent's output."""
+        configurable = cast(Dict[str, Any], config.pop("configurable", {}))
+
+        if configurable:
+            configured_agent = self.agent.with_config(
+                {
+                    "configurable": configurable,
+                }
+            )
+        else:
+            configured_agent = self.agent
+
+        executor = AgentExecutor(
+            agent=configured_agent,
+            tools=tools,
+        ).with_config({"run_name": "executor"})
+
+        async for output in executor.astream(input, config=config, **kwargs):
+            yield output
+
+
+app = FastAPI()
+
+
+# We need to add these input/output schemas because the current AgentExecutor
+# is lacking in schemas.
+class Input(BaseModel):
+    input: str
+
+
+class Output(BaseModel):
+    output: Any
+
+
+runnable = CustomAgentExecutor()
+
+# Add routes to the app for using the custom agent executor.
+add_routes(
+    app,
+    runnable.with_types(input_type=Input, output_type=Output),
+    disabled_endpoints=["invoke", "batch"],  # not implemented
+)
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=8000)

examples/configurable_chain/client.ipynb

@@ -0,0 +1,289 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Client\n",
+    "\n",
+    "Demo of client interacting with the simple chain server, which deploys a chain that tells jokes about a particular topic."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can interact with this via API directly"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import requests\n",
+    "\n",
+    "inputs = {\"input\": {\"topic\": \"sports\"}}\n",
+    "response = requests.post(\"http://localhost:8000/configurable_temp/invoke\", json=inputs)\n",
+    "\n",
+    "response.json()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can also interact with this via the RemoteRunnable interface (to use in other chains)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langserve import RemoteRunnable\n",
+    "\n",
+    "remote_runnable = RemoteRunnable(\"http://localhost:8000/configurable_temp\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Remote runnable has the same interface as local runnables"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "response = await remote_runnable.ainvoke({\"topic\": \"sports\"})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The client can also execute langchain code synchronously, and pass in configs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain.schema.runnable.config import RunnableConfig\n",
+    "\n",
+    "remote_runnable.batch([{\"topic\": \"sports\"}, {\"topic\": \"cars\"}])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The server supports streaming (using HTTP server-side events), which can help interact with long responses in real time"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "async for chunk in remote_runnable.astream({\"topic\": \"bears, but a bit verbose\"}):\n",
+    "    print(chunk, end=\"\", flush=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Configurability\n",
+    "\n",
+    "The server chains have been exposed as configurable chains!\n",
+    "\n",
+    "```python \n",
+    "\n",
+    "model = ChatOpenAI(temperature=0.5).configurable_alternatives(\n",
+    "    ConfigurableField(\n",
+    "        id=\"llm\",\n",
+    "        name=\"LLM\",\n",
+    "        description=(\n",
+    "            \"Decide whether to use a high or a low temperature parameter for the LLM.\"\n",
+    "        ),\n",
+    "    ),\n",
+    "    high_temp=ChatOpenAI(temperature=0.9),\n",
+    "    low_temp=ChatOpenAI(temperature=0.1),\n",
+    "    default_key=\"medium_temp\",\n",
+    ")\n",
+    "prompt = PromptTemplate.from_template(\n",
+    "    \"tell me a joke about {topic}.\"\n",
+    ").configurable_fields(  # Example of a configurable field\n",
+    "    template=ConfigurableField(\n",
+    "        id=\"prompt\",\n",
+    "        name=\"Prompt\",\n",
+    "        description=(\"The prompt to use. Must contain {topic}.\"),\n",
+    "    )\n",
+    ")\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We can now use the configurability of the runnable in the API!"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "await remote_runnable.ainvoke(\n",
+    "    {\"topic\": \"sports\"},\n",
+    "    config={\n",
+    "        \"configurable\": {\"prompt\": \"how to say {topic} in french\", \"llm\": \"low_temp\"}\n",
+    "    },\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Configurability Based on Request Properties\n",
+    "\n",
+    "If you want to change your chain invocation based on your request's properties,\n",
+    "you can do so with `add_routes`'s `per_req_config_modifier` method as follows:\n",
+    "\n",
+    "```python \n",
+    "\n",
+    "# Add another example route where you can configure the model based\n",
+    "# on properties of the request. This is useful for passing in API\n",
+    "# keys from request headers (WITH CAUTION) or using other properties\n",
+    "# of the request to configure the model.\n",
+    "def fetch_api_key_from_header(config: Dict[str, Any], req: Request) -> Dict[str, Any]:\n",
+    "    if \"x-api-key\" in req.headers:\n",
+    "        config[\"configurable\"][\"openai_api_key\"] = req.headers[\"x-api-key\"]\n",
+    "    return config\n",
+    "\n",
+    "dynamic_auth_model = ChatOpenAI(openai_api_key='placeholder').configurable_fields(\n",
+    "    openai_api_key=ConfigurableField(\n",
+    "        id=\"openai_api_key\",\n",
+    "        name=\"OpenAI API Key\",\n",
+    "        description=(\n",
+    "            \"API Key for OpenAI interactions\"\n",
+    "        ),\n",
+    "    ),\n",
+    ")\n",
+    "\n",
+    "dynamic_auth_chain = dynamic_auth_model | StrOutputParser()\n",
+    "\n",
+    "add_routes(\n",
+    "    app, \n",
+    "    dynamic_auth_chain, \n",
+    "    path=\"/auth_from_header\",\n",
+    "    config_keys=[\"configurable\"], \n",
+    "    per_req_config_modifier=fetch_api_key_from_header\n",
+    ")\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now, we can see that our request to the model will only work if we have a specific request\n",
+    "header set:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# The model will fail with an auth error\n",
+    "unauthenticated_response = requests.post(\n",
+    "    \"http://localhost:8000/auth_from_header/invoke\", json={\"input\": \"hello\"}\n",
+    ")\n",
+    "unauthenticated_response.json()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now, ensure that you have run the following locally on your shell\n",
+    "```bash\n",
+    "export TEST_API_KEY=<INSERT MY KEY HERE>\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# The model will succeed as long as the above shell script is run previously\n",
+    "import os\n",
+    "\n",
+    "test_key = os.environ[\"TEST_API_KEY\"]\n",
+    "authenticated_response = requests.post(\n",
+    "    \"http://localhost:8000/auth_from_header/invoke\",\n",
+    "    json={\"input\": \"hello\"},\n",
+    "    headers={\"x-api-key\": test_key},\n",
+    ")\n",
+    "authenticated_response.json()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/configurable_chain/server.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+"""Example of configurable runnables.
+
+This example shows how to use two options for configuration of runnables:
+
+1) Configurable Fields: Use this to specify values for a given initialization parameter
+2) Configurable Alternatives: Use this to specify complete alternative runnables
+"""
+from typing import Any, Dict
+
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.middleware.cors import CORSMiddleware
+from langchain.chat_models import ChatOpenAI
+from langchain.prompts import PromptTemplate
+from langchain.schema.output_parser import StrOutputParser
+from langchain.schema.runnable import ConfigurableField
+
+from langserve import add_routes
+
+app = FastAPI(
+    title="LangChain Server",
+    version="1.0",
+    description="Spin up a simple api server using Langchain's Runnable interfaces",
+)
+
+# Set all CORS enabled origins
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+    expose_headers=["*"],
+)
+
+###############################################################################
+#                EXAMPLE 1: Configure fields based on RunnableConfig          #
+###############################################################################
+model = ChatOpenAI(temperature=0.5).configurable_alternatives(
+    ConfigurableField(
+        id="llm",
+        name="LLM",
+        description=(
+            "Decide whether to use a high or a low temperature parameter for the LLM."
+        ),
+    ),
+    high_temp=ChatOpenAI(temperature=0.9),
+    low_temp=ChatOpenAI(temperature=0.1),
+    default_key="medium_temp",
+)
+prompt = PromptTemplate.from_template(
+    "tell me a joke about {topic}."
+).configurable_fields(  # Example of a configurable field
+    template=ConfigurableField(
+        id="prompt",
+        name="Prompt",
+        description="The prompt to use. Must contain {topic}.",
+    )
+)
+chain = prompt | model | StrOutputParser()
+
+add_routes(app, chain, path="/configurable_temp")
+
+
+###############################################################################
+#                EXAMPLE 2: Configure prompt based on RunnableConfig          #
+###############################################################################
+configurable_prompt = PromptTemplate.from_template(
+    "tell me a joke about {topic}."
+).configurable_alternatives(
+    ConfigurableField(
+        id="prompt",
+        name="Prompt",
+        description="The prompt to use. Must contain {topic}.",
+    ),
+    default_key="joke",
+    fact=PromptTemplate.from_template(
+        "tell me a fact about {topic} in {language} language."
+    ),
+)
+prompt_chain = configurable_prompt | model | StrOutputParser()
+
+add_routes(app, prompt_chain, path="/configurable_prompt")
+
+
+###############################################################################
+#             EXAMPLE 3: Configure fields based on Request metadata           #
+###############################################################################
+
+
+# Add another example route where you can configure the model based
+# on properties of the request. This is useful for passing in API
+# keys from request headers (WITH CAUTION) or using other properties
+# of the request to configure the model.
+def fetch_api_key_from_header(config: Dict[str, Any], req: Request) -> Dict[str, Any]:
+    if "x-api-key" in req.headers:
+        config["configurable"]["openai_api_key"] = req.headers["x-api-key"]
+    else:
+        raise HTTPException(401, "No API key provided")
+
+    return config
+
+
+dynamic_auth_model = ChatOpenAI(openai_api_key="placeholder").configurable_fields(
+    openai_api_key=ConfigurableField(
+        id="openai_api_key",
+        name="OpenAI API Key",
+        description=("API Key for OpenAI interactions"),
+    ),
+)
+
+dynamic_auth_chain = dynamic_auth_model | StrOutputParser()
+
+add_routes(
+    app,
+    dynamic_auth_chain,
+    path="/auth_from_header",
+    per_req_config_modifier=fetch_api_key_from_header,
+)
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=8000)

examples/configurable_retrieval/client.ipynb

@@ -0,0 +1,168 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Client\n",
+    "\n",
+    "Demo of a client interacting with a configurable retriever (see server code)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can interact with this via API directly"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'output': [{'page_content': 'cats like fish',\n",
+       "   'metadata': {},\n",
+       "   'type': 'Document'},\n",
+       "  {'page_content': 'dogs like sticks', 'metadata': {}, 'type': 'Document'}],\n",
+       " 'callback_events': [],\n",
+       " 'metadata': {'run_id': 'f375cdf6-2848-4976-9565-f69e175c24ce'}}"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import requests\n",
+    "\n",
+    "inputs = {\"input\": \"cat\"}\n",
+    "response = requests.post(\"http://localhost:8000/invoke\", json=inputs)\n",
+    "\n",
+    "response.json()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can also interact with this via the RemoteRunnable interface (to use in other chains)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langserve import RemoteRunnable\n",
+    "\n",
+    "remote_runnable = RemoteRunnable(\"http://localhost:8000/\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Remote runnable has the same interface as local runnables"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(page_content='cats like fish'),\n",
+       " Document(page_content='dogs like sticks')]"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "await remote_runnable.ainvoke(\"cat\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(page_content='cats like fish'),\n",
+       " Document(page_content='dogs like sticks')]"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "await remote_runnable.ainvoke(\"cat\", {\"configurable\": {\"collection_name\": \"Index 1\"}})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(page_content='x_n+1=a * xn * (1-xn)')]"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "await remote_runnable.ainvoke(\"cat\", {\"configurable\": {\"collection_name\": \"Index 2\"}})"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/configurable_retrieval/server.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python
+"""A more complex example that shows how to configure index name at run time."""
+from typing import Any, Iterable, List, Optional, Type
+
+from fastapi import FastAPI
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.schema import Document
+from langchain.schema.embeddings import Embeddings
+from langchain.schema.retriever import BaseRetriever
+from langchain.schema.runnable import (
+    ConfigurableFieldSingleOption,
+    RunnableConfig,
+    RunnableSerializable,
+)
+from langchain.schema.vectorstore import VST
+from langchain.vectorstores import FAISS, VectorStore
+
+from langserve import add_routes
+from langserve.pydantic_v1 import BaseModel, Field
+
+vectorstore1 = FAISS.from_texts(
+    ["cats like fish", "dogs like sticks"], embedding=OpenAIEmbeddings()
+)
+
+vectorstore2 = FAISS.from_texts(["x_n+1=a * xn * (1-xn)"], embedding=OpenAIEmbeddings())
+
+
+app = FastAPI(
+    title="LangChain Server",
+    version="1.0",
+    description="Spin up a simple api server using Langchain's Runnable interfaces",
+)
+
+
+class UnderlyingVectorStore(VectorStore):
+    """This is a fake vectorstore for demo purposes."""
+
+    def __init__(self, collection_name: str) -> None:
+        """Fake vectorstore that has a collection name."""
+        self.collection_name = collection_name
+
+    def as_retriever(self) -> BaseRetriever:
+        if self.collection_name == "index1":
+            return vectorstore1.as_retriever()
+        elif self.collection_name == "index2":
+            return vectorstore2.as_retriever()
+        else:
+            raise NotImplementedError(
+                f"No retriever for collection {self.collection_name}"
+            )
+
+    def add_texts(
+        self,
+        texts: Iterable[str],
+        metadatas: Optional[List[dict]] = None,
+        **kwargs: Any,
+    ) -> List[str]:
+        raise NotImplementedError()
+
+    @classmethod
+    def from_texts(
+        cls: Type[VST],
+        texts: List[str],
+        embedding: Embeddings,
+        metadatas: Optional[List[dict]] = None,
+        **kwargs: Any,
+    ) -> VST:
+        raise NotImplementedError()
+
+    def similarity_search(
+        self, embedding: List[float], k: int = 4, **kwargs: Any
+    ) -> List[Document]:
+        raise NotImplementedError()
+
+
+class ConfigurableRetriever(RunnableSerializable[str, List[Document]]):
+    """Create a custom retriever that can be configured by the user.
+
+    This is an example of how to create a custom runnable that can be configured
+    to use a different collection name at run time.
+
+    Configuration involves instantiating a VectorStore with a collection name.
+    at run time, so the underlying vectorstore should be *cheap* to instantiate.
+
+    For example, it should not be making any network requests at instantiation time.
+
+    Make sure that the vectorstore you use meets this criteria.
+    """
+
+    collection_name: str
+
+    def invoke(
+        self, input: str, config: Optional[RunnableConfig] = None
+    ) -> List[Document]:
+        """Invoke the retriever."""
+        vectorstore = UnderlyingVectorStore(self.collection_name)
+        retriever = vectorstore.as_retriever()
+        return retriever.invoke(input, config=config)
+
+
+configurable_collection_name = ConfigurableRetriever(
+    collection_name="index1"
+).configurable_fields(
+    collection_name=ConfigurableFieldSingleOption(
+        id="collection_name",
+        name="Collection Name",
+        description="The name of the collection to use for the retriever.",
+        options={
+            "Index 1": "index1",
+            "Index 2": "index2",
+        },
+        default="Index 1",
+    )
+)
+
+
+class Request(BaseModel):
+    __root__: str = Field(default="cat", description="Search query")
+
+
+add_routes(app, configurable_collection_name.with_types(input_type=Request))
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=8000)

examples/conversational_retrieval_chain/client.ipynb

@@ -18,16 +18,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
+   "execution_count": 11,
+   "metadata": {
+    "tags": []
+   },
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "{'output': {'answer': 'Cats like fish.'}}"
+       "{'output': {'content': 'Based on the given context, the information we have about Harrison is that he worked at Kensho.',\n",
+       "  'additional_kwargs': {},\n",
+       "  'type': 'ai',\n",
+       "  'example': False},\n",
+       " 'callback_events': [],\n",
+       " 'metadata': {'run_id': '3455df2b-93f8-4e67-b1a3-27f90670cf7b'}}"
       ]
      },
-     "execution_count": 2,
+     "execution_count": 11,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -35,7 +42,7 @@
    "source": [
     "import requests\n",
     "\n",
-    "inputs = {\"input\": {\"question\": \"what do cats like?\", \"chat_history\": \"\"}}\n",
+    "inputs = {\"input\": {\"question\": \"what do you know about harrison\", \"chat_history\": []}}\n",
     "response = requests.post(\"http://localhost:8000/invoke\", json=inputs)\n",
     "\n",
     "response.json()"
@@ -50,7 +57,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 12,
    "metadata": {
     "tags": []
    },
@@ -70,7 +77,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 13,
    "metadata": {
     "tags": []
    },
@@ -78,21 +85,21 @@
     {
      "data": {
       "text/plain": [
-       "{'answer': 'Cats like fish.'}"
+       "AIMessage(content='Based on the given context, the only information we have about Harrison is that he worked at Kensho.')"
       ]
      },
-     "execution_count": 7,
+     "execution_count": 13,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "await remote_runnable.ainvoke({\"question\": \"what do cats like?\", \"chat_history\": \"\"})"
+    "await remote_runnable.ainvoke({\"question\": \"what do you know about harrison\", \"chat_history\": []})"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 14,
    "metadata": {
     "tags": []
    },
@@ -100,26 +107,501 @@
     {
      "data": {
       "text/plain": [
-       "{'answer': 'Cats like fish.'}"
+       "AIMessage(content='Harrison worked at Kensho.')"
       ]
      },
-     "execution_count": 10,
+     "execution_count": 14,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
     "await remote_runnable.ainvoke(\n",
-    "    {\"question\": \"what do cats like?\", \"chat_history\": [(\"hi\", \"hi\")]}\n",
+    "    {\"question\": \"what do you know about harrison\", \"chat_history\": [(\"hi\", \"hi\")]}\n",
     ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 16,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "content=''\n",
+      "content='H'\n",
+      "content='arrison'\n",
+      "content=' worked'\n",
+      "content=' at'\n",
+      "content=' Kens'\n",
+      "content='ho'\n",
+      "content='.'\n",
+      "content=''\n"
+     ]
+    }
+   ],
+   "source": [
+    "async for chunk in remote_runnable.astream(\n",
+    "    {\"question\": \"what do you know about harrison\", \"chat_history\": [(\"hi\", \"hi\")]}\n",
+    "):\n",
+    "    print(chunk)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
    "metadata": {},
-   "outputs": [],
-   "source": []
+   "source": [
+    "stream log shows all intermediate steps as well!"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "RunLogPatch({'op': 'replace',\n",
+      "  'path': '',\n",
+      "  'value': {'final_output': None,\n",
+      "            'id': '2ff5a98d-49f0-40ae-92fe-489c3047d1c3',\n",
+      "            'logs': {},\n",
+      "            'streamed_output': []}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableParallel',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': 'ffdda3c9-a0ba-49a6-af18-c01748c31801',\n",
+      "            'metadata': {},\n",
+      "            'name': 'RunnableParallel',\n",
+      "            'start_time': '2023-11-16T15:59:23.348',\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:1'],\n",
+      "            'type': 'chain'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableSequence',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': 'a9da3f2c-f3ae-44c1-a2f4-1035faa0d1c2',\n",
+      "            'metadata': {},\n",
+      "            'name': 'RunnableSequence',\n",
+      "            'start_time': '2023-11-16T15:59:23.349',\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['map:key:standalone_question'],\n",
+      "            'type': 'chain'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableParallel:2',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '4d32b08f-a989-4c01-8068-479bed506d99',\n",
+      "            'metadata': {},\n",
+      "            'name': 'RunnableParallel',\n",
+      "            'start_time': '2023-11-16T15:59:23.349',\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:1'],\n",
+      "            'type': 'chain'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/<lambda>',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': 'c7b9bd78-cbb4-41f9-b31a-50a5f8940a8d',\n",
+      "            'metadata': {},\n",
+      "            'name': '<lambda>',\n",
+      "            'start_time': '2023-11-16T15:59:23.350',\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['map:key:chat_history'],\n",
+      "            'type': 'chain'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/<lambda>/final_output',\n",
+      "  'value': {'output': '\\nHuman: hi\\nAssistant: hi'}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/<lambda>/end_time',\n",
+      "  'value': '2023-11-16T15:59:23.350'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableParallel:2/final_output',\n",
+      "  'value': {'chat_history': '\\nHuman: hi\\nAssistant: hi'}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/RunnableParallel:2/end_time',\n",
+      "  'value': '2023-11-16T15:59:23.351'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/PromptTemplate',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': 'ef9f8729-2d4a-4887-86a8-a284d64f5882',\n",
+      "            'metadata': {},\n",
+      "            'name': 'PromptTemplate',\n",
+      "            'start_time': '2023-11-16T15:59:23.351',\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:2'],\n",
+      "            'type': 'prompt'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/PromptTemplate/final_output',\n",
+      "  'value': StringPromptValue(text='Given the following conversation and a follow up question, rephrase the \\nfollow up question to be a standalone question, in its original language.\\n\\nChat History:\\n\\nHuman: hi\\nAssistant: hi\\nFollow Up Input: what do you know about harrison\\nStandalone question:')},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/PromptTemplate/end_time',\n",
+      "  'value': '2023-11-16T15:59:23.351'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '7cda5923-ed2a-42ea-aee7-a2391371ff2f',\n",
+      "            'metadata': {},\n",
+      "            'name': 'ChatOpenAI',\n",
+      "            'start_time': '2023-11-16T15:59:23.352',\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:3'],\n",
+      "            'type': 'llm'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/StrOutputParser',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '6a0e199d-51fa-4306-8a7d-054444c6855f',\n",
+      "            'metadata': {},\n",
+      "            'name': 'StrOutputParser',\n",
+      "            'start_time': '2023-11-16T15:59:24.613',\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:4'],\n",
+      "            'type': 'parser'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableParallel:3',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '74a13de1-c2d1-43c0-813a-7ce0246118b3',\n",
+      "            'metadata': {},\n",
+      "            'name': 'RunnableParallel',\n",
+      "            'start_time': '2023-11-16T15:59:24.616',\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:2'],\n",
+      "            'type': 'chain'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableSequence:2',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '625e2183-7ee5-4f1f-8f47-56541afc96ee',\n",
+      "            'metadata': {},\n",
+      "            'name': 'RunnableSequence',\n",
+      "            'start_time': '2023-11-16T15:59:24.619',\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['map:key:context'],\n",
+      "            'type': 'chain'}})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/ChatOpenAI/streamed_output_str/-', 'value': ''})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/streamed_output_str/-',\n",
+      "  'value': 'What'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/streamed_output_str/-',\n",
+      "  'value': ' information'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/streamed_output_str/-',\n",
+      "  'value': ' do'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/streamed_output_str/-',\n",
+      "  'value': ' you'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/streamed_output_str/-',\n",
+      "  'value': ' have'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/streamed_output_str/-',\n",
+      "  'value': ' about'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/streamed_output_str/-',\n",
+      "  'value': ' Harrison'})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/ChatOpenAI/streamed_output_str/-', 'value': '?'})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/ChatOpenAI/streamed_output_str/-', 'value': ''})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/final_output',\n",
+      "  'value': LLMResult(generations=[[ChatGenerationChunk(text='What information do you have about Harrison?', generation_info={'finish_reason': 'stop'}, message=AIMessageChunk(content='What information do you have about Harrison?'))]], llm_output=None, run=None)},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI/end_time',\n",
+      "  'value': '2023-11-16T15:59:24.832'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/StrOutputParser/final_output',\n",
+      "  'value': {'output': 'What information do you have about Harrison?'}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/StrOutputParser/end_time',\n",
+      "  'value': '2023-11-16T15:59:24.833'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableSequence/final_output',\n",
+      "  'value': {'output': 'What information do you have about Harrison?'}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/RunnableSequence/end_time',\n",
+      "  'value': '2023-11-16T15:59:24.834'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableParallel/final_output',\n",
+      "  'value': {'standalone_question': 'What information do you have about '\n",
+      "                                   'Harrison?'}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/RunnableParallel/end_time',\n",
+      "  'value': '2023-11-16T15:59:24.835'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableLambda',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '71b2e8dc-753f-4bf2-83de-b87b26828370',\n",
+      "            'metadata': {},\n",
+      "            'name': 'RunnableLambda',\n",
+      "            'start_time': '2023-11-16T15:59:24.837',\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:1'],\n",
+      "            'type': 'chain'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableLambda/final_output',\n",
+      "  'value': {'output': 'What information do you have about Harrison?'}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/RunnableLambda/end_time',\n",
+      "  'value': '2023-11-16T15:59:24.837'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/Retriever',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': 'd3d6254d-d073-478f-b00b-bc27eafa24fd',\n",
+      "            'metadata': {},\n",
+      "            'name': 'Retriever',\n",
+      "            'start_time': '2023-11-16T15:59:24.839',\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:2', 'FAISS', 'OpenAIEmbeddings'],\n",
+      "            'type': 'retriever'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/<lambda>:2',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '77f7132a-d98f-4121-89cf-10e462c26496',\n",
+      "            'metadata': {},\n",
+      "            'name': '<lambda>',\n",
+      "            'start_time': '2023-11-16T15:59:24.839',\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['map:key:question'],\n",
+      "            'type': 'chain'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/<lambda>:2/final_output',\n",
+      "  'value': {'output': 'What information do you have about Harrison?'}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/<lambda>:2/end_time',\n",
+      "  'value': '2023-11-16T15:59:24.840'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/Retriever/final_output',\n",
+      "  'value': {'documents': [Document(page_content='harrison worked at kensho')]}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/Retriever/end_time',\n",
+      "  'value': '2023-11-16T15:59:25.074'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/_combine_documents',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': '82aecf3e-ca9d-4b48-a281-03f8e834ea62',\n",
+      "            'metadata': {},\n",
+      "            'name': '_combine_documents',\n",
+      "            'start_time': '2023-11-16T15:59:25.075',\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:3'],\n",
+      "            'type': 'chain'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/_combine_documents/final_output',\n",
+      "  'value': {'output': 'harrison worked at kensho'}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/_combine_documents/end_time',\n",
+      "  'value': '2023-11-16T15:59:25.075'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableSequence:2/final_output',\n",
+      "  'value': {'output': 'harrison worked at kensho'}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/RunnableSequence:2/end_time',\n",
+      "  'value': '2023-11-16T15:59:25.075'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/RunnableParallel:3/final_output',\n",
+      "  'value': {'context': 'harrison worked at kensho',\n",
+      "            'question': 'What information do you have about Harrison?'}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/RunnableParallel:3/end_time',\n",
+      "  'value': '2023-11-16T15:59:25.076'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatPromptTemplate',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': 'b37f36ce-3a27-4402-81c4-6893f03ec179',\n",
+      "            'metadata': {},\n",
+      "            'name': 'ChatPromptTemplate',\n",
+      "            'start_time': '2023-11-16T15:59:25.076',\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:3'],\n",
+      "            'type': 'prompt'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatPromptTemplate/final_output',\n",
+      "  'value': {'messages': [HumanMessage(content='Answer the question based only on the following context:\\nharrison worked at kensho\\n\\nQuestion: What information do you have about Harrison?\\n')]}},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatPromptTemplate/end_time',\n",
+      "  'value': '2023-11-16T15:59:25.077'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2',\n",
+      "  'value': {'end_time': None,\n",
+      "            'final_output': None,\n",
+      "            'id': 'b1179088-eb7c-49c5-af1e-bd09c48408bf',\n",
+      "            'metadata': {},\n",
+      "            'name': 'ChatOpenAI',\n",
+      "            'start_time': '2023-11-16T15:59:25.078',\n",
+      "            'streamed_output_str': [],\n",
+      "            'tags': ['seq:step:4'],\n",
+      "            'type': 'llm'}})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/ChatOpenAI:2/streamed_output_str/-', 'value': ''})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='Based')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': 'Based'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' on')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' on'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' the')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' the'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' given')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' given'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' context')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' context'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=',')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ','})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' the')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' the'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' only')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' only'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' information')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' information'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' we')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' we'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' have')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' have'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' about')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' about'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' Harrison')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' Harrison'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' is')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' is'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' that')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' that'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' he')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' he'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' worked')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' worked'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' at')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' at'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content=' Kens')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': ' Kens'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='ho')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': 'ho'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='.')})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/streamed_output_str/-',\n",
+      "  'value': '.'})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/streamed_output/-',\n",
+      "  'value': AIMessageChunk(content='')})\n",
+      "RunLogPatch({'op': 'add', 'path': '/logs/ChatOpenAI:2/streamed_output_str/-', 'value': ''})\n",
+      "RunLogPatch({'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/final_output',\n",
+      "  'value': LLMResult(generations=[[ChatGenerationChunk(text='Based on the given context, the only information we have about Harrison is that he worked at Kensho.', generation_info={'finish_reason': 'stop'}, message=AIMessageChunk(content='Based on the given context, the only information we have about Harrison is that he worked at Kensho.'))]], llm_output=None, run=None)},\n",
+      " {'op': 'add',\n",
+      "  'path': '/logs/ChatOpenAI:2/end_time',\n",
+      "  'value': '2023-11-16T15:59:26.547'})\n",
+      "RunLogPatch({'op': 'replace',\n",
+      "  'path': '/final_output',\n",
+      "  'value': {'output': AIMessageChunk(content='Based on the given context, the only information we have about Harrison is that he worked at Kensho.')}})\n"
+     ]
+    }
+   ],
+   "source": [
+    "async for chunk in remote_runnable.astream_log(\n",
+    "    {\"question\": \"what do you know about harrison\", \"chat_history\": [(\"hi\", \"hi\")]}\n",
+    "):\n",
+    "    print(chunk)"
+   ]
   }
  ],
  "metadata": {
@@ -138,7 +620,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.1"
+   "version": "3.9.6"
   }
  },
  "nbformat": 4,

examples/conversational_retrieval_chain/server.py

@@ -1,21 +1,101 @@
 #!/usr/bin/env python
-"""Example LangChain server exposes a conversational retrieval chain."""
+"""Example LangChain server exposes a conversational retrieval chain.
+
+Follow the reference here:
+
+https://python.langchain.com/docs/expression_language/cookbook/retrieval#conversational-retrieval-chain
+
+To run this example, you will need to install the following packages:
+pip install langchain openai faiss-cpu tiktoken
+"""  # noqa: F401
+
+from operator import itemgetter
+from typing import List, Tuple
+
 from fastapi import FastAPI
-from langchain.chains import ConversationalRetrievalChain
 from langchain.chat_models import ChatOpenAI
 from langchain.embeddings import OpenAIEmbeddings
+from langchain.prompts import ChatPromptTemplate
+from langchain.prompts.prompt import PromptTemplate
+from langchain.schema import format_document
+from langchain.schema.output_parser import StrOutputParser
+from langchain.schema.runnable import RunnableMap, RunnablePassthrough
 from langchain.vectorstores import FAISS
 
 from langserve import add_routes
+from langserve.pydantic_v1 import BaseModel, Field
+
+_TEMPLATE = """Given the following conversation and a follow up question, rephrase the 
+follow up question to be a standalone question, in its original language.
+
+Chat History:
+{chat_history}
+Follow Up Input: {question}
+Standalone question:"""
+CONDENSE_QUESTION_PROMPT = PromptTemplate.from_template(_TEMPLATE)
+
+ANSWER_TEMPLATE = """Answer the question based only on the following context:
+{context}
+
+Question: {question}
+"""
+ANSWER_PROMPT = ChatPromptTemplate.from_template(ANSWER_TEMPLATE)
+
+DEFAULT_DOCUMENT_PROMPT = PromptTemplate.from_template(template="{page_content}")
+
+
+def _combine_documents(
+    docs, document_prompt=DEFAULT_DOCUMENT_PROMPT, document_separator="\n\n"
+):
+    """Combine documents into a single string."""
+    doc_strings = [format_document(doc, document_prompt) for doc in docs]
+    return document_separator.join(doc_strings)
+
+
+def _format_chat_history(chat_history: List[Tuple]) -> str:
+    """Format chat history into a string."""
+    buffer = ""
+    for dialogue_turn in chat_history:
+        human = "Human: " + dialogue_turn[0]
+        ai = "Assistant: " + dialogue_turn[1]
+        buffer += "\n" + "\n".join([human, ai])
+    return buffer
+
 
 vectorstore = FAISS.from_texts(
-    ["cats like fish", "dogs like sticks"], embedding=OpenAIEmbeddings()
+    ["harrison worked at kensho"], embedding=OpenAIEmbeddings()
 )
 retriever = vectorstore.as_retriever()
 
-model = ChatOpenAI()
+_inputs = RunnableMap(
+    standalone_question=RunnablePassthrough.assign(
+        chat_history=lambda x: _format_chat_history(x["chat_history"])
+    )
+    | CONDENSE_QUESTION_PROMPT
+    | ChatOpenAI(temperature=0)
+    | StrOutputParser(),
+)
+_context = {
+    "context": itemgetter("standalone_question") | retriever | _combine_documents,
+    "question": lambda x: x["standalone_question"],
+}
 
-chain = ConversationalRetrievalChain.from_llm(model, retriever)
+
+# User input
+class ChatHistory(BaseModel):
+    """Chat history with the bot."""
+
+    chat_history: List[Tuple[str, str]] = Field(
+        ...,
+        extra={"widget": {"type": "chat", "input": "question"}},
+    )
+    question: str
+
+
+conversational_qa_chain = (
+    _inputs | _context | ANSWER_PROMPT | ChatOpenAI() | StrOutputParser()
+)
+chain = conversational_qa_chain.with_types(input_type=ChatHistory)
 
 app = FastAPI(
     title="LangChain Server",
@@ -26,7 +106,7 @@ app = FastAPI(
 # /invoke
 # /batch
 # /stream
-add_routes(app, chain)
+add_routes(app, chain, enable_feedback_endpoint=True)
 
 if __name__ == "__main__":
     import uvicorn

examples/file_processing/client.ipynb

@@ -0,0 +1,156 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# File processing\n",
+    "\n",
+    "This client will be uploading a PDF file to the langserve server which will read the PDF and extract content from the first page."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let's load the file in base64 encoding:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import base64\n",
+    "\n",
+    "with open(\"sample.pdf\", \"rb\") as f:\n",
+    "    data = f.read()\n",
+    "\n",
+    "encoded_data = base64.b64encode(data).decode(\"utf-8\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Using raw requests"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'output': 'If you’re reading this you might be using LangServe 🦜🏓!\\n\\nThis is a sample PDF!\\n\\n\\x0c',\n",
+       " 'callback_events': []}"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import requests\n",
+    "\n",
+    "requests.post(\n",
+    "    \"http://localhost:8000/pdf/invoke/\", json={\"input\": {\"file\": encoded_data}}\n",
+    ").json()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Using the SDK"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langserve import RemoteRunnable\n",
+    "\n",
+    "runnable = RemoteRunnable(\"http://localhost:8000/pdf/\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'If you’re reading this you might be using LangServe 🦜🏓!\\n\\nThis is a sample PDF!\\n\\n\\x0c'"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "runnable.invoke({\"file\": encoded_data})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['If you’re reading this you might be using LangServe 🦜🏓!\\n\\nThis is a sample PDF!\\n\\n\\x0c',\n",
+       " 'If you’re ']"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "runnable.batch([{\"file\": encoded_data}, {\"file\": encoded_data, \"num_chars\": 10}])"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/file_processing/server.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+"""Example that shows how to upload files and process files in the server.
+
+This example uses a very simple architecture for dealing with file uploads
+and processing.
+
+The main issue with this approach is that processing is done in
+the same process rather than offloaded to a process pool. A smaller
+issue is that the base64 encoding incurs an additional encoding/decoding
+overhead.
+
+This example also specifies a "base64file" widget, which will create a widget
+allowing one to upload a binary file using the langserve playground UI.
+"""
+import base64
+
+from fastapi import FastAPI
+from langchain.document_loaders.blob_loaders import Blob
+from langchain.document_loaders.parsers.pdf import PDFMinerParser
+from langchain.pydantic_v1 import Field
+from langchain.schema.runnable import RunnableLambda
+
+from langserve import CustomUserType, add_routes
+
+app = FastAPI(
+    title="LangChain Server",
+    version="1.0",
+    description="Spin up a simple api server using Langchain's Runnable interfaces",
+)
+
+
+# ATTENTION: Inherit from CustomUserType instead of BaseModel otherwise
+#            the server will decode it into a dict instead of a pydantic model.
+class FileProcessingRequest(CustomUserType):
+    """Request including a base64 encoded file."""
+
+    # The extra field is used to specify a widget for the playground UI.
+    file: str = Field(..., extra={"widget": {"type": "base64file"}})
+    num_chars: int = 100
+
+
+def _process_file(request: FileProcessingRequest) -> str:
+    """Extract the text from the first page of the PDF."""
+    content = base64.b64decode(request.file.encode("utf-8"))
+    blob = Blob(data=content)
+    documents = list(PDFMinerParser().lazy_parse(blob))
+    content = documents[0].page_content
+    return content[: request.num_chars]
+
+
+add_routes(
+    app,
+    RunnableLambda(_process_file).with_types(input_type=FileProcessingRequest),
+    config_keys=["configurable"],
+    path="/pdf",
+)
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=8000)

examples/llm/client.ipynb

@@ -229,7 +229,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain.schema.runnable import PutLocalVar, GetLocalVar"
+    "from langchain.schema.runnable import RunnablePassthrough"
    ]
   },
   {
@@ -266,13 +266,8 @@
     ")\n",
     "\n",
     "\n",
-    "chain = (\n",
-    "    comedian_chain\n",
-    "    | PutLocalVar(\"joke\")\n",
-    "    | {\"joke\": GetLocalVar(\"joke\")}\n",
-    "    | joke_classifier_chain\n",
-    "    | PutLocalVar(\"classification\")\n",
-    "    | {\"joke\": GetLocalVar(\"joke\"), \"classification\": GetLocalVar(\"classification\")}\n",
+    "chain = {\"joke\": comedian_chain} | RunnablePassthrough.assign(\n",
+    "    classification=joke_classifier_chain\n",
     ")"
    ]
   },

examples/local_llm/client.ipynb

@@ -0,0 +1,301 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Local LLM\n",
+    "\n",
+    "Here, we'll use a server that's serving a local LLM.\n",
+    "\n",
+    "**Attention** This is OK for prototyping / dev usage, but should not be used for production cases when there might be concurrent requests from different users. As of the time of writing, Ollama is designed for single user and cannot handle concurrent requests see this issue: https://github.com/ollama/ollama/issues/358"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain.prompts.chat import ChatPromptTemplate"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langserve import RemoteRunnable\n",
+    "\n",
+    "model = RemoteRunnable(\"http://localhost:8000/ollama/\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let's test out the standard interface of a chat model."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "prompt = \"Tell me a 3 sentence story about a cat.\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "AIMessage(content=\"\\nSure! Here is a three sentence story about a cat:\\n\\nMittens the cat purred contentedly on the windowsill, basking in the warm sunlight. Suddenly, a bird perched nearby and Mittens' ears perked up, ready to pounce. With lightning quick reflexes, Mittens leapt into the air, but the bird had flown away, leaving Mittens to settle for just lounging in the sun once again.\")"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model.invoke(prompt)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "AIMessage(content=\"\\nSure! Here is a three sentence story about a cat:\\n\\nMittens the cat purred contentedly on the windowsill, basking in the warm sunlight. Suddenly, a bird flew by, catching Mittens' attention and causing her to leap into action. With lightning quick reflexes, Mittens pounced on the bird, but it flew away just in time, leaving Mittens frustrated but still purring happily.\")"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "await model.ainvoke(prompt)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Batched API works, but b/c ollama does not support parallelism, it's no faster than using .invoke twice."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 7.65 ms, sys: 6.57 ms, total: 14.2 ms\n",
+      "Wall time: 5.51 s\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "[AIMessage(content='\\nSure! Here is a three sentence story about a cat:\\n\\nMr. Whiskers was a sleek black cat with bright green eyes. He spent his days lounging in the sunbeams that streamed through the living room window, chasing the occasional fly, and purring contentedly. Despite his lazy demeanor, Mr. Whiskers was always on the lookout for a warm lap to curl up in.'),\n",
+       " AIMessage(content='\\nSure! Here is a three sentence story about a cat:\\n\\nMittens the cat purred contentedly on the windowsill, basking in the warm sunlight that streamed through the glass. Suddenly, a tiny bird perched on the ledge outside, tweeting nervously as it eyed the cat with suspicion. Without hesitation, Mittens pounced, snatching the bird in mid-air and devouring it in one quick motion.')]"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "model.batch([prompt, prompt])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 9.72 ms, sys: 7.59 ms, total: 17.3 ms\n",
+      "Wall time: 5.56 s\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "for _ in range(2):\n",
+    "    model.invoke(prompt)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[AIMessage(content=\"\\nSure, here's a three sentence story about a cat:\\n\\nMittens the cat purred contentedly on the windowsill, basking in the warm sunlight that streamed through the glass. Her bright green eyes sparkled as she watched a bird flit and flutter outside, wishing she could join it in its flight. Just then, her owner entered the room with a bowl of creamy milk, causing Mittens to jump down from the windowsill and rub against their legs in excitement.\"),\n",
+       " AIMessage(content='\\nSure! Here is a three sentence story about a cat:\\n\\nMittens the cat purred contentedly on my lap, her soft fur a soothing balm for my frazzled nerves. As I stroked her back, she gazed up at me with big, round eyes, purring even louder. It was hard to resist the charm of this little ball of fluff, and I found myself smiling and scratching behind her ears.')]"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "await model.abatch([prompt, prompt])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Streaming is available by default"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "|S|ure|,| here| is| a| |3| sentence| story| about| a| cat|:|\n",
+      "|\n",
+      "|M|itt|ens| the| cat| pur|red| content|edly| on| the| windows|ill|,| her| tail| tw|itch|ing| as| she| watched| the| birds| outside|.| Sud|den|ly|,| a| squ|ir|rel| sc|am|per|ed| by| and| Mitt|ens| was| on| high| alert|,| her| ears| per|ked| up| and| ready| to| p|ounce|.| With| light|ning| quick| ref|lex|es|,| Mitt|ens| jump|ed| from| the| windows|ill| and| ch|ased| after| the| squ|ir|rel|,| her| tail| streaming| behind| her|.||"
+     ]
+    }
+   ],
+   "source": [
+    "for chunk in model.stream(prompt):\n",
+    "    print(chunk.content, end=\"|\", flush=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "|The| cat| pur|red| content|edly| on| my| lap|,| its| soft| fur| a| so|othing| bal|m| for| my| fra|zz|led| n|erves|.| As| I| st|rok|ed| its| back|,| it| gaz|ed| up| at| me| with| soul|ful| eyes|,| p|urr|ing| loud|ly| in| appro|val|.| In| that| moment|,| all| was| right| with| the| world|.||"
+     ]
+    }
+   ],
+   "source": [
+    "async for chunk in model.astream(prompt):\n",
+    "    print(chunk.content, end=\"|\", flush=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "And so is the event stream API"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'event': 'on_chat_model_start', 'run_id': '6c2bdfc1-d482-4861-886c-c737a50771c3', 'name': '/ollama', 'tags': [], 'metadata': {}, 'data': {'input': 'Tell me a 3 sentence story about a cat.'}}\n",
+      "{'event': 'on_chat_model_stream', 'run_id': '6c2bdfc1-d482-4861-886c-c737a50771c3', 'tags': [], 'metadata': {}, 'name': '/ollama', 'data': {'chunk': AIMessageChunk(content='\\n')}}\n",
+      "{'event': 'on_chat_model_stream', 'run_id': '6c2bdfc1-d482-4861-886c-c737a50771c3', 'tags': [], 'metadata': {}, 'name': '/ollama', 'data': {'chunk': AIMessageChunk(content='S')}}\n",
+      "{'event': 'on_chat_model_stream', 'run_id': '6c2bdfc1-d482-4861-886c-c737a50771c3', 'tags': [], 'metadata': {}, 'name': '/ollama', 'data': {'chunk': AIMessageChunk(content='ure')}}\n",
+      "{'event': 'on_chat_model_stream', 'run_id': '6c2bdfc1-d482-4861-886c-c737a50771c3', 'tags': [], 'metadata': {}, 'name': '/ollama', 'data': {'chunk': AIMessageChunk(content=',')}}\n",
+      "{'event': 'on_chat_model_stream', 'run_id': '6c2bdfc1-d482-4861-886c-c737a50771c3', 'tags': [], 'metadata': {}, 'name': '/ollama', 'data': {'chunk': AIMessageChunk(content=' here')}}\n",
+      "{'event': 'on_chat_model_stream', 'run_id': '6c2bdfc1-d482-4861-886c-c737a50771c3', 'tags': [], 'metadata': {}, 'name': '/ollama', 'data': {'chunk': AIMessageChunk(content=' is')}}\n",
+      "{'event': 'on_chat_model_stream', 'run_id': '6c2bdfc1-d482-4861-886c-c737a50771c3', 'tags': [], 'metadata': {}, 'name': '/ollama', 'data': {'chunk': AIMessageChunk(content=' a')}}\n",
+      "{'event': 'on_chat_model_stream', 'run_id': '6c2bdfc1-d482-4861-886c-c737a50771c3', 'tags': [], 'metadata': {}, 'name': '/ollama', 'data': {'chunk': AIMessageChunk(content=' ')}}\n",
+      "{'event': 'on_chat_model_stream', 'run_id': '6c2bdfc1-d482-4861-886c-c737a50771c3', 'tags': [], 'metadata': {}, 'name': '/ollama', 'data': {'chunk': AIMessageChunk(content='3')}}\n",
+      "{'event': 'on_chat_model_stream', 'run_id': '6c2bdfc1-d482-4861-886c-c737a50771c3', 'tags': [], 'metadata': {}, 'name': '/ollama', 'data': {'chunk': AIMessageChunk(content=' sentence')}}\n",
+      "{'event': 'on_chat_model_stream', 'run_id': '6c2bdfc1-d482-4861-886c-c737a50771c3', 'tags': [], 'metadata': {}, 'name': '/ollama', 'data': {'chunk': AIMessageChunk(content=' story')}}\n",
+      "...\n"
+     ]
+    }
+   ],
+   "source": [
+    "i = 0\n",
+    "async for event in model.astream_events(prompt, version='v1'):\n",
+    "    print(event)\n",
+    "    if i > 10:\n",
+    "        print('...')\n",
+    "        break\n",
+    "    i += 1"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/local_llm/server.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+"""Example LangChain Server that runs a local llm.
+
+**Attention** This is OK for prototyping / dev usage, but should not be used
+for production cases when there might be concurrent requests from different
+users. As of the time of writing, Ollama is designed for single user and cannot
+handle concurrent requests see this issue: 
+https://github.com/ollama/ollama/issues/358
+
+When deploying local models, make sure you understand whether the model is able
+to handle concurrent requests or not. If concurrent requests are not handled
+properly, the server will either crash or will just not be able to handle more
+than one user at a time.
+"""
+from fastapi import FastAPI
+from langchain_community.chat_models import ChatOllama
+
+from langserve import add_routes
+
+app = FastAPI(
+    title="LangChain Server",
+    version="1.0",
+    description="Spin up a simple api server using Langchain's Runnable interfaces",
+)
+
+
+llm = ChatOllama(model="llama2")
+
+add_routes(
+    app,
+    llm,
+    path="/ollama",
+)
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=8000)

examples/multiple_servers/client_server.py

@@ -0,0 +1,35 @@
+"""Client server that interacts with the main server via a remote runnable.
+
+This server sets up a simple proxy to the main server. It uses the RemoteRunnable
+to interact with the main server. The main server is expected to be running at
+http://localhost:8123.
+
+A client server will likely end up doing something more clever rather than
+just being a proxy.
+"""
+from fastapi import FastAPI
+
+from langserve import RemoteRunnable, add_routes
+
+app = FastAPI()
+
+MAIN_SERVER_URL = (
+    "http://localhost:8123/chat_model/"  # <-- URL of the RUNNABLE on the main server
+)
+# Type inference is not automatic for remote runnables at the moment,
+# so you must specify which types are used for the playground to work.
+remote_runnable = RemoteRunnable(MAIN_SERVER_URL).with_types(input_type=str)
+
+
+# Let's add an example chain
+add_routes(
+    app,
+    remote_runnable,
+    path="/proxied",
+)
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app)

examples/multiple_servers/main_server.py

@@ -0,0 +1,20 @@
+"""Main server that exposes one or more chains as HTTP endpoints."""
+from fastapi import FastAPI
+from langchain_openai import ChatOpenAI
+
+from langserve import add_routes
+
+app = FastAPI()
+
+# Let's add an example chain
+add_routes(
+    app,
+    ChatOpenAI(),
+    path="/chat_model",
+)
+
+if __name__ == "__main__":
+    import uvicorn
+
+    # Running on port 8123
+    uvicorn.run(app, port=8123)

examples/passthrough_dict/client.ipynb

@@ -0,0 +1,148 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Passthrough information\n",
+    "\n",
+    "An example that shows how to pass through additional info with the request, and get it back with the response."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain.prompts.chat import ChatPromptTemplate"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langserve import RemoteRunnable\n",
+    "\n",
+    "chain = RemoteRunnable(\"http://localhost:8000/v1/\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let's create a prompt composed of a system message and a human message."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'output': AIMessage(content='`apple` translates to `mela` in Italian.'),\n",
+       " 'info': {'info': {'user_id': 42, 'user_info': {'address': 42}}}}"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "chain.invoke({'thing': 'apple', 'language': 'italian', 'info': {\"user_id\": 42, \"user_info\": {\"address\": 42}}})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'info': {'info': {'user_id': 42, 'user_info': {'address': 42}}}}\n",
+      "{'output': AIMessageChunk(content='')}\n",
+      "{'output': AIMessageChunk(content='m')}\n",
+      "{'output': AIMessageChunk(content='ela')}\n",
+      "{'output': AIMessageChunk(content='')}\n"
+     ]
+    }
+   ],
+   "source": [
+    "for chunk in chain.stream({'thing': 'apple', 'language': 'italian', 'info': {\"user_id\": 42, \"user_info\": {\"address\": 42}}}):\n",
+    "    print(chunk)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langserve import RemoteRunnable\n",
+    "\n",
+    "chain = RemoteRunnable(\"http://localhost:8000/v2/\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'output': AIMessage(content='`apple` translates to `mela` in Italian.'),\n",
+       " 'info': {'user_id': 42, 'user_info': {'address': 42}}}"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "chain.invoke({'thing': 'apple', 'language': 'italian', 'info': {\"user_id\": 42, \"user_info\": {\"address\": 42}}})"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/passthrough_dict/server.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+"""Example LangChain server passes through some of the inputs in the response."""
+
+from typing import Any, Callable, Dict, List, Optional, TypedDict
+
+from fastapi import FastAPI
+from langchain.chat_models import ChatOpenAI
+from langchain.prompts import ChatPromptTemplate
+from langchain.schema.runnable import RunnableMap, RunnablePassthrough
+
+from langserve import add_routes
+
+app = FastAPI(
+    title="LangChain Server",
+    version="1.0",
+    description="Spin up a simple api server using Langchain's Runnable interfaces",
+)
+
+
+def _create_projection(
+    *, include_keys: Optional[List] = None, exclude_keys: Optional[List[str]] = None
+) -> Callable[[dict], dict]:
+    """Create a projection function."""
+
+    def _project_dict(
+        d: dict,
+    ) -> dict:
+        """Project dictionary."""
+        keys = d.keys()
+        if include_keys is not None:
+            keys = set(keys) & set(include_keys)
+        if exclude_keys is not None:
+            keys = set(keys) - set(exclude_keys)
+        return {k: d[k] for k in keys}
+
+    return _project_dict
+
+
+prompt = ChatPromptTemplate.from_messages(
+    [("human", "translate `{thing}` to {language}")]
+)
+model = ChatOpenAI()
+
+underlying_chain = prompt | model
+
+wrapped_chain = RunnableMap(
+    {
+        "output": _create_projection(exclude_keys=["info"]) | underlying_chain,
+        "info": _create_projection(include_keys=["info"]),
+    }
+)
+
+
+class Input(TypedDict):
+    thing: str
+    language: str
+    info: Dict[str, Any]
+
+
+class Output(TypedDict):
+    output: underlying_chain.output_schema
+    info: Dict[str, Any]
+
+
+add_routes(
+    app, wrapped_chain.with_types(input_type=Input, output_type=Output), path="/v1"
+)
+
+
+# Version 2
+# Uses RunnablePassthrough.assign
+wrapped_chain_2 = RunnablePassthrough.assign(output=underlying_chain) | {
+    "output": lambda x: x["output"],
+    "info": lambda x: x["info"],
+}
+
+add_routes(
+    app,
+    wrapped_chain_2.with_types(input_type=Input, output_type=Output),
+    path="/v2",
+)
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=8000)

examples/router/server.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+"""Example LangChain Server that uses a Fast API Router.
+
+When applications grow, it becomes useful to use FastAPI's Router to organize
+the routes.
+
+See more documentation at:
+
+https://fastapi.tiangolo.com/tutorial/bigger-applications/
+"""
+from fastapi import APIRouter, FastAPI
+from langchain.chat_models import ChatAnthropic, ChatOpenAI
+
+from langserve import add_routes
+
+app = FastAPI()
+
+router = APIRouter(prefix="/models")
+
+# Invocations to this router will appear in trace logs as /models/openai
+add_routes(
+    router,
+    ChatOpenAI(),
+    path="/openai",
+)
+# Invocations to this router will appear in trace logs as /models/anthropic
+add_routes(
+    router,
+    ChatAnthropic(),
+    path="/anthropic",
+)
+
+app.include_router(router)
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=8000)

examples/widgets/chat/message_list/server.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+"""Example of a simple chatbot that just passes current conversation
+state back and forth between server and client.
+"""
+from typing import List, Union
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from langchain.chat_models import ChatAnthropic
+from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+
+from langserve import add_routes
+from langserve.pydantic_v1 import BaseModel, Field
+
+app = FastAPI(
+    title="LangChain Server",
+    version="1.0",
+    description="Spin up a simple api server using Langchain's Runnable interfaces",
+)
+
+
+# Set all CORS enabled origins
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+    expose_headers=["*"],
+)
+
+
+# Declare a chain
+prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", "You are a helpful assisstant named Cob."),
+        MessagesPlaceholder(variable_name="messages"),
+    ]
+)
+
+chain = prompt | ChatAnthropic(model="claude-2") | StrOutputParser()
+
+
+class InputChat(BaseModel):
+    """Input for the chat endpoint."""
+
+    messages: List[Union[HumanMessage, AIMessage, SystemMessage]] = Field(
+        ...,
+        description="The chat messages representing the current conversation.",
+        extra={"widget": {"type": "chat", "input": "messages"}},
+    )
+
+
+add_routes(
+    app,
+    chain.with_types(input_type=InputChat),
+)
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=8000)

examples/widgets/chat/tuples/server.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python
+"""Endpoint shows off available playground widgets."""
+import base64
+from json import dumps
+from typing import Any, Dict, List, Tuple
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from langchain.chat_models.openai import ChatOpenAI
+from langchain.document_loaders.blob_loaders import Blob
+from langchain.document_loaders.parsers.pdf import PDFMinerParser
+from langchain.pydantic_v1 import BaseModel, Field
+from langchain.schema.messages import (
+    AIMessage,
+    BaseMessage,
+    FunctionMessage,
+    HumanMessage,
+)
+from langchain.schema.runnable import RunnableLambda
+from langchain_core.runnables import RunnableParallel
+
+from langserve import CustomUserType
+from langserve.server import add_routes
+
+app = FastAPI(
+    title="LangChain Server",
+    version="1.0",
+    description="Spin up a simple api server using Langchain's Runnable interfaces",
+)
+
+
+# Set all CORS enabled origins
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+    expose_headers=["*"],
+)
+
+# Example 1: Chat Widget
+# This shows how to create a chat widget.
+
+
+class ChatHistory(CustomUserType):
+    chat_history: List[Tuple[str, str]] = Field(
+        ...,
+        examples=[[("a", "aa")]],
+        extra={"widget": {"type": "chat", "input": "question", "output": "answer"}},
+    )
+    question: str
+
+
+def _format_to_messages(input: ChatHistory) -> List[BaseMessage]:
+    """Format the input to a list of messages."""
+    history = input.chat_history
+    user_input = input.question
+
+    messages = []
+
+    for human, ai in history:
+        messages.append(HumanMessage(content=human))
+        messages.append(AIMessage(content=ai))
+    messages.append(HumanMessage(content=user_input))
+    return messages
+
+
+model = ChatOpenAI()
+chat_model = RunnableParallel({"answer": (RunnableLambda(_format_to_messages) | model)})
+add_routes(
+    app,
+    chat_model.with_types(input_type=ChatHistory),
+    config_keys=["configurable"],
+    path="/chat",
+)
+
+
+# Example 2: Chat Widget with History
+# This one isn't hooked up toa model. It just shows that FunctionMessages can be used
+# surfaced as well in the playground.
+
+
+class ChatHistoryMessage(BaseModel):
+    chat_history: List[BaseMessage] = Field(
+        ...,
+        extra={"widget": {"type": "chat", "input": "location"}},
+    )
+    location: str
+
+
+def chat_message_bot(input: Dict[str, Any]) -> List[BaseMessage]:
+    """Bot that repeats the question twice."""
+    return [
+        AIMessage(
+            content="",
+            additional_kwargs={
+                "function_call": {
+                    "name": "get_weather",
+                    "arguments": dumps({"location": input["location"]}),
+                }
+            },
+        ),
+        FunctionMessage(name="get_weather", content='{"value": 32}'),
+        AIMessage(content=f"Weather in {input['location']}: 32"),
+    ]
+
+
+add_routes(
+    app,
+    RunnableLambda(chat_message_bot).with_types(input_type=ChatHistoryMessage),
+    config_keys=["configurable"],
+    path="/chat_message",
+)
+
+# Example 3: File Processing Widget
+
+
+class FileProcessingRequest(BaseModel):
+    file: bytes = Field(..., extra={"widget": {"type": "base64file"}})
+    num_chars: int = 100
+
+
+def process_file(input: Dict[str, Any]) -> str:
+    """Extract the text from the first page of the PDF."""
+    content = base64.decodebytes(input["file"])
+    blob = Blob(data=content)
+    documents = list(PDFMinerParser().lazy_parse(blob))
+    content = documents[0].page_content
+    return content[: input["num_chars"]]
+
+
+add_routes(
+    app,
+    RunnableLambda(process_file).with_types(input_type=FileProcessingRequest),
+    config_keys=["configurable"],
+    path="/pdf",
+)
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="localhost", port=8000)

langserve/__init__.py

@@ -1,7 +1,19 @@
-"""Main entrypoint into package."""
+"""Main entrypoint into package.
 
+This is the ONLY public interface into the package. All other modules are
+to be considered private and subject to change without notice.
+"""
+
+from langserve.api_handler import APIHandler
 from langserve.client import RemoteRunnable
+from langserve.schema import CustomUserType
 from langserve.server import add_routes
 from langserve.version import __version__
 
-__all__ = ["RemoteRunnable", "add_routes", "__version__"]
+__all__ = [
+    "RemoteRunnable",
+    "APIHandler",
+    "add_routes",
+    "__version__",
+    "CustomUserType",
+]

langserve/callbacks.py

@@ -0,0 +1,475 @@
+from __future__ import annotations
+
+import uuid
+from typing import Any, Dict, List, Optional, Sequence
+from uuid import UUID
+
+from langchain.callbacks.base import AsyncCallbackHandler
+from langchain.callbacks.manager import (
+    BaseRunManager,
+    ahandle_event,
+    handle_event,
+)
+from langchain.schema import AgentAction, AgentFinish, BaseMessage, Document, LLMResult
+from typing_extensions import TypedDict
+
+
+class CallbackEventDict(TypedDict, total=False):
+    """A dictionary representation of a callback event."""
+
+    type: str
+    parent_run_id: Optional[UUID]
+    run_id: UUID
+
+
+class AsyncEventAggregatorCallback(AsyncCallbackHandler):
+    """A callback handler that aggregates all the events that have been called.
+
+    This callback handler aggregates all the events that have been called placing
+    them in a single mutable list.
+
+    This callback handler is not threading safe, and is meant to be used in an async
+    context only.
+    """
+
+    def __init__(self) -> None:
+        """Get a list of all the callback events that have been called."""
+        super().__init__()
+        # Callback events is a mutable state that is used only in an async context,
+        # so it should be safe to mutate without the usage of a lock.
+        self.callback_events: List[CallbackEventDict] = []
+
+    def log_callback(self, event: CallbackEventDict) -> None:
+        """Log the callback event."""
+        self.callback_events.append(event)
+
+    async def on_chat_model_start(
+        self,
+        serialized: Dict[str, Any],
+        messages: List[List[BaseMessage]],
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[List[str]] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> Any:
+        """Attempt to serialize the callback event."""
+        self.log_callback(
+            {
+                "type": "on_chat_model_start",
+                "serialized": serialized,
+                "messages": messages,
+                "run_id": run_id,
+                "parent_run_id": parent_run_id,
+                "tags": tags,
+                "metadata": metadata,
+                "kwargs": kwargs,
+            }
+        )
+
+    async def on_chain_start(
+        self,
+        serialized: Dict[str, Any],
+        inputs: Dict[str, Any],
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[List[str]] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> None:
+        """Attempt to serialize the callback event."""
+        self.log_callback(
+            {
+                "type": "on_chain_start",
+                "serialized": serialized,
+                "inputs": inputs,
+                "run_id": run_id,
+                "parent_run_id": parent_run_id,
+                "tags": tags,
+                "metadata": metadata,
+                "kwargs": kwargs,
+            }
+        )
+
+    async def on_chain_end(
+        self,
+        outputs: Any,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[List[str]] = None,
+        **kwargs: Any,
+    ) -> None:
+        self.log_callback(
+            {
+                "type": "on_chain_end",
+                "outputs": outputs,
+                "run_id": run_id,
+                "parent_run_id": parent_run_id,
+                "tags": tags,
+                "kwargs": kwargs,
+            }
+        )
+
+    async def on_chain_error(
+        self,
+        error: BaseException,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[List[str]] = None,
+        **kwargs: Any,
+    ) -> None:
+        self.log_callback(
+            {
+                "type": "on_chain_error",
+                "error": error,
+                "run_id": run_id,
+                "parent_run_id": parent_run_id,
+                "tags": tags,
+                "kwargs": kwargs,
+            }
+        )
+
+    async def on_retriever_start(
+        self,
+        serialized: Dict[str, Any],
+        query: str,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[List[str]] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> None:
+        self.log_callback(
+            {
+                "type": "on_retriever_start",
+                "serialized": serialized,
+                "query": query,
+                "run_id": run_id,
+                "parent_run_id": parent_run_id,
+                "tags": tags,
+                "metadata": metadata,
+                "kwargs": kwargs,
+            }
+        )
+
+    async def on_retriever_end(
+        self,
+        documents: Sequence[Document],
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[List[str]] = None,
+        **kwargs: Any,
+    ) -> None:
+        self.log_callback(
+            {
+                "type": "on_retriever_end",
+                "documents": documents,
+                "run_id": run_id,
+                "parent_run_id": parent_run_id,
+                "tags": tags,
+                "kwargs": kwargs,
+            }
+        )
+
+    async def on_retriever_error(
+        self,
+        error: BaseException,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[List[str]] = None,
+        **kwargs: Any,
+    ) -> None:
+        self.log_callback(
+            {
+                "type": "on_retriever_error",
+                "error": error,
+                "run_id": run_id,
+                "parent_run_id": parent_run_id,
+                "tags": tags,
+                "kwargs": kwargs,
+            }
+        )
+
+    async def on_tool_start(
+        self,
+        serialized: Dict[str, Any],
+        input_str: str,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[List[str]] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> None:
+        self.log_callback(
+            {
+                "type": "on_tool_start",
+                "serialized": serialized,
+                "input_str": input_str,
+                "run_id": run_id,
+                "parent_run_id": parent_run_id,
+                "tags": tags,
+                "metadata": metadata,
+                "kwargs": kwargs,
+            }
+        )
+
+    async def on_tool_end(
+        self,
+        output: str,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[List[str]] = None,
+        **kwargs: Any,
+    ) -> None:
+        self.log_callback(
+            {
+                "type": "on_tool_end",
+                "output": output,
+                "run_id": run_id,
+                "parent_run_id": parent_run_id,
+                "tags": tags,
+                "kwargs": kwargs,
+            }
+        )
+
+    async def on_tool_error(
+        self,
+        error: BaseException,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[List[str]] = None,
+        **kwargs: Any,
+    ) -> None:
+        self.log_callback(
+            {
+                "type": "on_tool_error",
+                "error": error,
+                "run_id": run_id,
+                "parent_run_id": parent_run_id,
+                "tags": tags,
+                "kwargs": kwargs,
+            }
+        )
+
+    async def on_agent_action(
+        self,
+        action: AgentAction,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[List[str]] = None,
+        **kwargs: Any,
+    ) -> None:
+        self.log_callback(
+            {
+                "type": "on_agent_action",
+                "action": action,
+                "run_id": run_id,
+                "parent_run_id": parent_run_id,
+                "tags": tags,
+                "kwargs": kwargs,
+            }
+        )
+
+    async def on_agent_finish(
+        self,
+        finish: AgentFinish,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[List[str]] = None,
+        **kwargs: Any,
+    ) -> None:
+        self.log_callback(
+            {
+                "type": "on_agent_finish",
+                "finish": finish,
+                "run_id": run_id,
+                "parent_run_id": parent_run_id,
+                "tags": tags,
+                "kwargs": kwargs,
+            }
+        )
+
+    async def on_llm_start(
+        self,
+        serialized: Dict[str, Any],
+        prompts: List[str],
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[List[str]] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> None:
+        self.log_callback(
+            {
+                "type": "on_llm_start",
+                "serialized": serialized,
+                "prompts": prompts,
+                "run_id": run_id,
+                "parent_run_id": parent_run_id,
+                "tags": tags,
+                "metadata": metadata,
+                "kwargs": kwargs,
+            }
+        )
+
+    async def on_llm_end(
+        self,
+        response: LLMResult,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[List[str]] = None,
+        **kwargs: Any,
+    ) -> None:
+        self.log_callback(
+            {
+                "type": "on_llm_end",
+                "response": response,
+                "run_id": run_id,
+                "parent_run_id": parent_run_id,
+                "tags": tags,
+                "kwargs": kwargs,
+            }
+        )
+
+    async def on_llm_error(
+        self,
+        error: BaseException,
+        *,
+        run_id: UUID,
+        parent_run_id: Optional[UUID] = None,
+        tags: Optional[List[str]] = None,
+        **kwargs: Any,
+    ) -> None:
+        self.log_callback(
+            {
+                "type": "on_llm_error",
+                "error": error,
+                "run_id": run_id,
+                "parent_run_id": parent_run_id,
+                "tags": tags,
+                "kwargs": kwargs,
+            }
+        )
+
+
+def replace_uuids(
+    callback_events: Sequence[CallbackEventDict],
+) -> List[CallbackEventDict]:
+    """Replace uids in the event callbacks with new uids.
+
+    This function mutates the event callback events in place.
+
+    Args:
+        callback_events: A list of event callbacks.
+    """
+    # Create a dictionary to store mappings from old UID to new UID
+    uid_mapping: dict = {}
+
+    updated_events = []
+
+    # Iterate through the list of event callbacks
+    for event in callback_events:
+        updated_event = event.copy()
+        # Replace UIDs in the 'run_id' field
+        if "run_id" in updated_event and updated_event["run_id"] is not None:
+            if updated_event["run_id"] not in uid_mapping:
+                # Generate a new UUID
+                new_uid = uuid.uuid4()
+                uid_mapping[updated_event["run_id"]] = new_uid
+            # Replace the old UID with the new one
+            updated_event["run_id"] = uid_mapping[updated_event["run_id"]]
+
+        # Replace UIDs in the 'parent_run_id' field if it's not None
+        if (
+            "parent_run_id" in updated_event
+            and updated_event["parent_run_id"] is not None
+        ):
+            if updated_event["parent_run_id"] not in uid_mapping:
+                # Generate a new UUID
+                new_uid = uuid.uuid4()
+                uid_mapping[updated_event["parent_run_id"]] = new_uid
+            # Replace the old UID with the new one
+            updated_event["parent_run_id"] = uid_mapping[updated_event["parent_run_id"]]
+        updated_events.append(updated_event)
+    return updated_events
+
+
+# Mapping from event name to ignore condition name
+NAME_TO_IGNORE_CONDITION = {
+    "on_retry": "ignore_retry",
+    "on_text": None,
+    "on_agent_action": "ignore_agent",
+    "on_agent_finish": "ignore_agent",
+    "on_llm_start": "ignore_llm",
+    "on_llm_end": "ignore_llm",
+    "on_llm_error": "ignore_llm",
+    "on_chain_start": "ignore_chain",
+    "on_chain_end": "ignore_chain",
+    "on_chain_error": "ignore_chain",
+    "on_chat_model_start": "ignore_chat_model",
+    "on_tool_start": "ignore_agent",
+    "on_tool_end": "ignore_agent",
+    "on_tool_error": "ignore_agent",
+    "on_retriever_start": "ignore_retriever",
+    "on_retriever_end": "ignore_retriever",
+    "on_retriever_error": "ignore_retriever",
+}
+
+
+async def ahandle_callbacks(
+    callback_manager: BaseRunManager,
+    callback_events: Sequence[CallbackEventDict],
+) -> None:
+    """Invoke all the callback handlers with the given callback events."""
+    callback_events = replace_uuids(callback_events)
+
+    # 1. Do I need inheritable handlers
+    for event in callback_events:
+        if event["parent_run_id"] is None:  # How do we make sure it's None!?
+            event["parent_run_id"] = callback_manager.run_id
+
+        event_data = {key: value for key, value in event.items() if key != "type"}
+
+        await ahandle_event(
+            # Unpacking like this may not work
+            callback_manager.handlers,
+            event["type"],
+            ignore_condition_name=NAME_TO_IGNORE_CONDITION.get(event["type"], None),
+            **event_data,
+        )
+
+
+def handle_callbacks(
+    callback_manager: BaseRunManager,
+    callback_events: Sequence[CallbackEventDict],
+) -> None:
+    """Invoke all the callback handlers with the given callback events."""
+    callback_events = replace_uuids(callback_events)
+
+    for event in callback_events:
+        if event["parent_run_id"] is None:  # How do we make sure it's None!?
+            event["parent_run_id"] = callback_manager.run_id
+
+        event_data = {key: value for key, value in event.items() if key != "type"}
+
+        handle_event(
+            # Unpacking like this may not work
+            callback_manager.handlers,
+            event["type"],
+            ignore_condition_name=NAME_TO_IGNORE_CONDITION.get(event["type"], None),
+            **event_data,
+        )

langserve/chat_playground/.eslintrc.cjs

@@ -0,0 +1,18 @@
+module.exports = {
+  root: true,
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/recommended',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parser: '@typescript-eslint/parser',
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': [
+      'warn',
+      { allowConstantExport: true },
+    ],
+  },
+}

langserve/chat_playground/.gitignore

@@ -0,0 +1,28 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
+
+.yarn
+
+!dist

langserve/chat_playground/README.md

@@ -0,0 +1,27 @@
+# React + TypeScript + Vite
+
+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
+
+Currently, two official plugins are available:
+
+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react/README.md) uses [Babel](https://babeljs.io/) for Fast Refresh
+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
+
+## Expanding the ESLint configuration
+
+If you are developing a production application, we recommend updating the configuration to enable type aware lint rules:
+
+- Configure the top-level `parserOptions` property like this:
+
+```js
+   parserOptions: {
+    ecmaVersion: 'latest',
+    sourceType: 'module',
+    project: ['./tsconfig.json', './tsconfig.node.json'],
+    tsconfigRootDir: __dirname,
+   },
+```
+
+- Replace `plugin:@typescript-eslint/recommended` to `plugin:@typescript-eslint/recommended-type-checked` or `plugin:@typescript-eslint/strict-type-checked`
+- Optionally add `plugin:@typescript-eslint/stylistic-type-checked`
+- Install [eslint-plugin-react](https://github.com/jsx-eslint/eslint-plugin-react) and add `plugin:react/recommended` & `plugin:react/jsx-runtime` to the `extends` list

langserve/chat_playground/dist/index.html

@@ -0,0 +1,42 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" href="/____LANGSERVE_BASE_URL/favicon.ico" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Chat Playground</title>
+    <script type="module" crossorigin src="/____LANGSERVE_BASE_URL/assets/index-86d4d9c0.js"></script>
+    <link rel="stylesheet" href="/____LANGSERVE_BASE_URL/assets/index-434ff580.css">
+  </head>
+  <body>
+    <div id="root"></div>
+    <script>
+      try {
+        window.CONFIG_SCHEMA = ____LANGSERVE_CONFIG_SCHEMA;
+      } catch (e) {
+        // pass
+      }
+      try {
+        window.INPUT_SCHEMA = ____LANGSERVE_INPUT_SCHEMA;
+      } catch (e) {
+        // pass
+      }
+      try {
+        window.OUTPUT_SCHEMA = ____LANGSERVE_OUTPUT_SCHEMA;
+      } catch (e) {
+        // pass
+      }
+      try {
+        window.FEEDBACK_ENABLED = ____LANGSERVE_FEEDBACK_ENABLED;
+      } catch (e) {
+        // pass
+      }
+      try {
+        window.PUBLIC_TRACE_LINK_ENABLED = ____LANGSERVE_PUBLIC_TRACE_LINK_ENABLED;
+      } catch (e) {
+        // pass
+      }
+    </script>
+    
+  </body>
+</html>

langserve/chat_playground/index.html

@@ -0,0 +1,40 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" href="/favicon.ico" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Chat Playground</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script>
+      try {
+        window.CONFIG_SCHEMA = ____LANGSERVE_CONFIG_SCHEMA;
+      } catch (e) {
+        // pass
+      }
+      try {
+        window.INPUT_SCHEMA = ____LANGSERVE_INPUT_SCHEMA;
+      } catch (e) {
+        // pass
+      }
+      try {
+        window.OUTPUT_SCHEMA = ____LANGSERVE_OUTPUT_SCHEMA;
+      } catch (e) {
+        // pass
+      }
+      try {
+        window.FEEDBACK_ENABLED = ____LANGSERVE_FEEDBACK_ENABLED;
+      } catch (e) {
+        // pass
+      }
+      try {
+        window.PUBLIC_TRACE_LINK_ENABLED = ____LANGSERVE_PUBLIC_TRACE_LINK_ENABLED;
+      } catch (e) {
+        // pass
+      }
+    </script>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>

langserve/chat_playground/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "langserve-chat-playground",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "packageManager": "yarn@1.22.19",
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc && vite build",
+    "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@emotion/react": "^11.11.1",
+    "@emotion/styled": "^11.11.0",
+    "@jsonforms/core": "^3.2.1",
+    "@microsoft/fetch-event-source": "^2.0.1",
+    "@mui/icons-material": "^5.14.11",
+    "@mui/material": "^5.14.11",
+    "@mui/x-date-pickers": "^6.16.0",
+    "@radix-ui/react-toggle-group": "^1.0.4",
+    "clsx": "^2.0.0",
+    "dayjs": "^1.11.10",
+    "fast-json-patch": "^3.1.1",
+    "lodash": "^4.17.21",
+    "lz-string": "^1.5.0",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "react-toastify": "^9.1.3",
+    "swr": "^2.2.4",
+    "tailwind-merge": "^1.14.0",
+    "use-debounce": "^9.0.4",
+    "vaul": "^0.7.3"
+  },
+  "devDependencies": {
+    "@types/lodash": "^4.14.200",
+    "@types/react": "^18.2.15",
+    "@types/react-dom": "^18.2.7",
+    "@typescript-eslint/eslint-plugin": "^6.0.0",
+    "@typescript-eslint/parser": "^6.0.0",
+    "@vitejs/plugin-react": "^4.0.3",
+    "autoprefixer": "^10.4.16",
+    "eslint": "^8.45.0",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "eslint-plugin-react-refresh": "^0.4.3",
+    "postcss": "^8.4.31",
+    "tailwindcss": "^3.3.3",
+    "typescript": "^5.0.2",
+    "vite": "^4.4.5",
+    "vite-plugin-svgr": "^4.1.0"
+  }
+}

langserve/chat_playground/postcss.config.js

@@ -0,0 +1,6 @@
+export default {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {},
+  },
+};

langserve/chat_playground/src/App.css

@@ -0,0 +1,116 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+@layer base {
+  * {
+    color: #043D5C;
+    font-weight: 300;
+    @font-face {
+      font-family: 'Manrope';
+      src: url('/dist/Manrope-VariableFont_wght.ttf') format('truetype');
+    }
+    border-color: #043D5C;
+  }
+  
+  input,
+  textarea,
+  select {
+    background: transparent;
+  }
+
+  /* clash between MUI and Tailwind */
+  input:focus,
+  textarea:focus,
+  select:focus {
+    box-shadow: none;
+    outline: none;
+  }
+
+  :root {
+    --popover: 0 0% 100%;
+    --background: #F8F7FF;
+
+    --divider-500: 210 40% 96.1%; /* slate-100 */
+    --divider-700: 214.3 31.8% 91.4%; /* slate-200 */
+
+    --ls-blue: 211.5 91.8% 61.8%;
+    --ls-black: 222.2 47.4% 11.2%; /* slate-900 */
+    --ls-gray-100: 215.4 16.3% 46.9%; /* slate-500 */
+    --ls-gray-200: 212.7 26.8% 83.9%; /* slate-300 */
+    --ls-gray-300: 214.3 31.8% 91.4%; /* slate-200 */
+    --ls-gray-400: 210 40% 96.1%; /* slate-100 */
+
+    --button-green: #162E2E;
+    --button-green-disabled: rgba(4, 61, 92, 0.20);
+    --button-inline: #006BA41A;
+  }
+
+  @media (prefers-color-scheme: dark) {
+    :root {
+      --popover: 240 11.6% 8.4%;
+
+      --divider-500: 217.2 32.6% 17.5%; /* slate-800 */
+      --divider-700: 215.3 25% 26.7%; /* slate-700 */
+
+      --ls-blue: 211.5 91.8% 61.8%;
+      --ls-black: 0 0% 100%; /* white */
+      --ls-gray-100: 215 20.2% 65.1%; /* slate-400 */
+      --ls-gray-200: 215.4 16.3% 46.9%; /* slate-500 */
+      --ls-gray-300: 215.3 25% 26.7%; /* slate-700 */
+      --ls-gray-400: 217.2 32.6% 17.5%; /* slate-800 */
+    }
+  }
+}
+
+.control {
+  @apply flex flex-col border border-divider-700 rounded-lg p-3 gap-1 relative bg-background transition-all outline-ls-blue/20;
+  @apply focus-within:border-ls-blue focus-within:outline focus-within:outline-4 focus-within:outline-ls-blue/20;
+}
+
+.control > label,
+.control h6 {
+  @apply text-xs uppercase font-semibold text-ls-gray-100;
+}
+
+.control div .MuiGrid-item {
+  @apply pt-0;
+}
+
+.control > select {
+  @apply -ml-1;
+}
+
+.control > .input-description,
+.control > .validation {
+  @apply absolute right-3 top-3 text-xs;
+}
+
+.group-layout {
+  @apply flex flex-col gap-4 bg-background p-4 border border-divider-700 rounded-lg;
+}
+
+.no-scrollbar {
+  scrollbar-width: none;
+}
+
+.no-scrollbar::-webkit-scrollbar {
+  display: none;
+}
+
+.vertical-layout {
+  @apply flex flex-col gap-4;
+}
+
+.share-button:hover {
+  background: linear-gradient(270deg, #BCB2FD 0.29%, #D65622 92%);
+}
+
+.share-button:hover > * {
+  color: white;
+}
+
+a {
+  color: blue;
+  text-decoration: underline;
+}

langserve/chat_playground/src/App.tsx

@@ -0,0 +1,72 @@
+import "./App.css";
+
+import { ChatWindow } from "./components/ChatWindow";
+import { AppCallbackContext, useAppStreamCallbacks } from "./useStreamCallback";
+import { useInputSchema, useOutputSchema } from "./useSchemas";
+import { useStreamLog } from "./useStreamLog";
+
+export function App() {
+  const { context, callbacks } = useAppStreamCallbacks();
+  const { startStream, stopStream } = useStreamLog(callbacks);
+  const inputSchema = useInputSchema({});
+  const outputSchema = useOutputSchema({});
+  const inputProps = inputSchema?.data?.schema?.properties;
+  const outputDataSchema = outputSchema?.data?.schema;
+  const isLoading = inputProps === undefined || outputDataSchema === undefined;
+  const inputKeys = Object.keys(inputProps ?? {});
+  const inputSchemaSupported = (
+    inputKeys.length === 1 &&
+    inputProps?.[inputKeys[0]].type === "array"
+  ) || (
+    inputKeys.length === 2 && (
+      (
+        inputProps?.[inputKeys[0]].type === "array" ||
+        inputProps?.[inputKeys[1]].type === "string"
+      ) || (
+        inputProps?.[inputKeys[0]].type === "string" ||
+        inputProps?.[inputKeys[1]].type === "array" 
+      )
+    )
+  );
+  const outputSchemaSupported = (
+    outputDataSchema?.anyOf?.find((option) => option.properties?.type?.enum?.includes("ai")) ||
+    outputDataSchema?.type === "string"
+  );
+  const isSupported = isLoading || (inputSchemaSupported && outputSchemaSupported);
+  return (
+    <div className="flex items-center flex-col text-ls-black bg-background">
+      <AppCallbackContext.Provider value={context}>
+        {isSupported
+          ? <ChatWindow
+              startStream={startStream}
+              stopStream={stopStream}
+              messagesInputKey={inputProps?.[inputKeys[0]].type === "array" ? inputKeys[0] : inputKeys[1]}
+              inputKey={inputProps?.[inputKeys[0]].type === "string" ? inputKeys[0] : inputKeys[1]}
+            ></ChatWindow>
+          : <div className="h-[100vh] w-[100vw] flex justify-center items-center text-xl p-16">
+              <span>
+                The chat playground is only supported for chains that take one of the following as input:
+                <ul className="mt-8 list-disc ml-6">
+                  <li>
+                    a dict with a single key containing a list of messages
+                  </li>
+                  <li>
+                    a dict with two keys: one a string input, one an list of messages
+                  </li>
+                </ul>
+                <br />
+                and which return either an <code>AIMessage</code> or a string.
+                <br />
+                <br />
+                You can test this chain in the default LangServe playground instead.
+                <br />
+                <br />
+                To use the default playground, set <code>playground_type="default"</code> when adding the route in your backend.
+              </span>
+            </div>}
+      </AppCallbackContext.Provider>
+    </div>
+  );
+}
+
+export default App;

langserve/chat_playground/src/assets/ArrowUp.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-arrow-up"><line x1="12" y1="19" x2="12" y2="5"></line><polyline points="5 12 12 5 19 12"></polyline></svg>

langserve/chat_playground/src/assets/ChatIcon.svg

@@ -0,0 +1,6 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+  <path fill-rule="evenodd" clip-rule="evenodd"
+    d="M7.7588 2H16.2414C17.0464 1.99999 17.7107 1.99998 18.2519 2.04419C18.814 2.09012 19.3307 2.18868 19.8161 2.43597C20.5687 2.81947 21.1806 3.43139 21.5641 4.18404C21.8114 4.66937 21.91 5.18608 21.9559 5.74817C22.0001 6.28936 22.0001 6.95372 22.0001 7.75868V13.2413C22.0001 14.0463 22.0001 14.7106 21.9559 15.2518C21.91 15.8139 21.8114 16.3306 21.5641 16.816C21.1806 17.5686 20.5687 18.1805 19.8161 18.564C19.3307 18.8113 18.814 18.9099 18.2519 18.9558C17.7107 19 17.0464 19 16.2414 19H13.6838C13.0197 19 12.8263 19.0047 12.6504 19.0408C12.4738 19.0771 12.303 19.137 12.1425 19.219C11.9826 19.3007 11.8286 19.4178 11.31 19.8327L8.89688 21.7632C8.7132 21.9102 8.52597 22.06 8.36137 22.1689C8.20394 22.273 7.8987 22.4593 7.50172 22.4597C7.0449 22.4602 6.61276 22.2525 6.32778 21.8955C6.08012 21.5852 6.03492 21.2305 6.01785 21.0425C6 20.846 6.00005 20.6062 6.00009 20.371L6.0001 18.9918C5.60829 18.9789 5.27229 18.9461 4.96482 18.8637C3.58445 18.4938 2.50626 17.4156 2.13639 16.0353C1.9993 15.5236 1.99962 14.933 2.00005 14.1376C2.00007 14.0924 2.0001 14.0465 2.0001 14L2.0001 7.7587C2.00008 6.95373 2.00007 6.28937 2.04429 5.74817C2.09022 5.18608 2.18878 4.66937 2.43607 4.18404C2.81956 3.43139 3.43149 2.81947 4.18413 2.43597C4.66947 2.18868 5.18617 2.09012 5.74827 2.04419C6.28947 1.99998 6.95383 1.99999 7.7588 2ZM5.91113 4.03755C5.47272 4.07337 5.24852 4.1383 5.09212 4.21799C4.71579 4.40973 4.40983 4.7157 4.21808 5.09202C4.13839 5.24842 4.07347 5.47262 4.03765 5.91104C4.00087 6.36113 4.0001 6.94342 4.0001 7.8V14C4.0001 14.9944 4.00869 15.2954 4.06824 15.5176C4.25318 16.2078 4.79227 16.7469 5.48246 16.9319C5.70474 16.9914 6.00574 17 7.0001 17C7.55238 17 8.0001 17.4477 8.0001 18V19.9194L10.0606 18.271C10.0834 18.2528 10.1058 18.2348 10.1279 18.2171C10.55 17.8791 10.8691 17.6237 11.2326 17.4379C11.5536 17.274 11.8952 17.1541 12.2483 17.0817C12.6482 16.9996 13.0569 16.9998 13.5976 17C13.626 17 13.6547 17 13.6838 17H16.2001C17.0567 17 17.639 16.9992 18.0891 16.9624C18.5275 16.9266 18.7517 16.8617 18.9081 16.782C19.2844 16.5903 19.5904 16.2843 19.7821 15.908C19.8618 15.7516 19.9267 15.5274 19.9625 15.089C19.9993 14.6389 20.0001 14.0566 20.0001 13.2V7.8C20.0001 6.94342 19.9993 6.36113 19.9625 5.91104C19.9267 5.47262 19.8618 5.24842 19.7821 5.09202C19.5904 4.7157 19.2844 4.40973 18.9081 4.21799C18.7517 4.1383 18.5275 4.07337 18.0891 4.03755C17.639 4.00078 17.0567 4 16.2001 4H7.8001C6.94352 4 6.36122 4.00078 5.91113 4.03755Z"
+    fill="currentColor" />
+</svg>
+  

langserve/chat_playground/src/assets/CheckCircleIcon.svg

@@ -0,0 +1,5 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+  <path fill-rule="evenodd" clip-rule="evenodd"
+    d="M12 5.45455C8.38505 5.45455 5.45455 8.38505 5.45455 12C5.45455 15.615 8.38505 18.5455 12 18.5455C15.615 18.5455 18.5455 15.615 18.5455 12C18.5455 8.38505 15.615 5.45455 12 5.45455ZM4 12C4 7.58172 7.58172 4 12 4C16.4183 4 20 7.58172 20 12C20 16.4183 16.4183 20 12 20C7.58172 20 4 16.4183 4 12ZM15.787 9.30392C16.071 9.58794 16.071 10.0484 15.787 10.3324L11.4233 14.6961C11.1393 14.9801 10.6788 14.9801 10.3948 14.6961L8.21301 12.5143C7.929 12.2303 7.929 11.7697 8.21301 11.4857C8.49703 11.2017 8.95751 11.2017 9.24153 11.4857L10.9091 13.1533L14.7585 9.30392C15.0425 9.01991 15.503 9.01991 15.787 9.30392Z"
+    fill="currentColor" />
+</svg>

langserve/chat_playground/src/assets/CheckCircleIcon2.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-check-circle"><path d="M22 11.08V12a10 10 0 1 1-5.93-9.14"></path><polyline points="22 4 12 14.01 9 11.01"></polyline></svg>

langserve/chat_playground/src/assets/ChevronRight.svg

@@ -0,0 +1,6 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+  <path fill-rule="evenodd" clip-rule="evenodd"
+    d="M8.29289 5.29289C8.68342 4.90237 9.31658 4.90237 9.70711 5.29289L15.7071 11.2929C16.0976 11.6834 16.0976 12.3166 15.7071 12.7071L9.70711 18.7071C9.31658 19.0976 8.68342 19.0976 8.29289 18.7071C7.90237 18.3166 7.90237 17.6834 8.29289 17.2929L13.5858 12L8.29289 6.70711C7.90237 6.31658 7.90237 5.68342 8.29289 5.29289Z"
+    fill="currentColor" />
+</svg>
+  

langserve/chat_playground/src/assets/CircleSpinIcon.svg

@@ -0,0 +1,15 @@
+<svg
+  aria-hidden="true"
+  viewBox="0 0 100 101"
+  fill="none"
+  xmlns="http://www.w3.org/2000/svg"
+>
+  <path
+    d="M100 50.5908C100 78.2051 77.6142 100.591 50 100.591C22.3858 100.591 0 78.2051 0 50.5908C0 22.9766 22.3858 0.59082 50 0.59082C77.6142 0.59082 100 22.9766 100 50.5908ZM9.08144 50.5908C9.08144 73.1895 27.4013 91.5094 50 91.5094C72.5987 91.5094 90.9186 73.1895 90.9186 50.5908C90.9186 27.9921 72.5987 9.67226 50 9.67226C27.4013 9.67226 9.08144 27.9921 9.08144 50.5908Z"
+    fill="currentColor"
+  />
+  <path
+    d="M93.9676 39.0409C96.393 38.4038 97.8624 35.9116 97.0079 33.5539C95.2932 28.8227 92.871 24.3692 89.8167 20.348C85.8452 15.1192 80.8826 10.7238 75.2124 7.41289C69.5422 4.10194 63.2754 1.94025 56.7698 1.05124C51.7666 0.367541 46.6976 0.446843 41.7345 1.27873C39.2613 1.69328 37.813 4.19778 38.4501 6.62326C39.0873 9.04874 41.5694 10.4717 44.0505 10.1071C47.8511 9.54855 51.7191 9.52689 55.5402 10.0491C60.8642 10.7766 65.9928 12.5457 70.6331 15.2552C75.2735 17.9648 79.3347 21.5619 82.5849 25.841C84.9175 28.9121 86.7997 32.2913 88.1811 35.8758C89.083 38.2158 91.5421 39.6781 93.9676 39.0409Z"
+    fill="currentFill"
+  />
+</svg>

langserve/chat_playground/src/assets/CodeIcon.svg

@@ -0,0 +1,6 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+  <path fill-rule="evenodd" clip-rule="evenodd"
+    d="M8.70711 5.29289C9.09763 5.68342 9.09763 6.31658 8.70711 6.70711L3.41421 12L8.70711 17.2929C9.09763 17.6834 9.09763 18.3166 8.70711 18.7071C8.31658 19.0976 7.68342 19.0976 7.29289 18.7071L1.29289 12.7071C0.902369 12.3166 0.902369 11.6834 1.29289 11.2929L7.29289 5.29289C7.68342 4.90237 8.31658 4.90237 8.70711 5.29289ZM15.2929 5.29289C15.6834 4.90237 16.3166 4.90237 16.7071 5.29289L22.7071 11.2929C23.0976 11.6834 23.0976 12.3166 22.7071 12.7071L16.7071 18.7071C16.3166 19.0976 15.6834 19.0976 15.2929 18.7071C14.9024 18.3166 14.9024 17.6834 15.2929 17.2929L20.5858 12L15.2929 6.70711C14.9024 6.31658 14.9024 5.68342 15.2929 5.29289Z"
+    fill="currentColor" />
+</svg>
+  

langserve/chat_playground/src/assets/CopyIcon.svg

@@ -0,0 +1,6 @@
+<svg width="25" height="25" viewBox="0 0 25 25" fill="none" xmlns="http://www.w3.org/2000/svg">
+  <path
+    d="M4.5 19.7783C4.5 21.5132 5.35498 22.3848 7.07324 22.3848H14.876C16.5942 22.3848 17.4492 21.5049 17.4492 19.7783V18.2427H18.9019C20.6118 18.2427 21.4751 17.3628 21.4751 15.6362V8.896C21.4751 7.875 21.2676 7.22754 20.645 6.58838L16.4531 2.33008C15.8638 1.72412 15.1665 1.5 14.2783 1.5H11.0991C9.38916 1.5 8.52588 2.37988 8.52588 4.10645V5.64209H7.07324C5.36328 5.64209 4.5 6.51367 4.5 8.24854V19.7783ZM16.6606 11.0874L12.0869 6.43066C11.4561 5.7832 10.9331 5.64209 10.0034 5.64209H9.8623V4.13135C9.8623 3.30957 10.3022 2.83643 11.1655 2.83643H14.8345V7.09473C14.8345 8.05762 15.2993 8.51416 16.2539 8.51416H20.1387V15.6113C20.1387 16.4414 19.6904 16.9062 18.8271 16.9062H17.4492V13.2954C17.4492 12.2329 17.3247 11.7681 16.6606 11.0874ZM16.0381 6.89551V3.49219L19.79 7.31055H16.4448C16.1543 7.31055 16.0381 7.18604 16.0381 6.89551ZM5.83643 19.7534V8.26514C5.83643 7.45166 6.27637 6.97852 7.13965 6.97852H9.8623V11.793C9.8623 12.8389 10.3936 13.3618 11.4229 13.3618H16.1128V19.7534C16.1128 20.5835 15.6646 21.0483 14.8096 21.0483H7.13135C6.27637 21.0483 5.83643 20.5835 5.83643 19.7534ZM11.5806 12.1084C11.2485 12.1084 11.1157 11.9756 11.1157 11.6436V7.28564L15.8555 12.1084H11.5806Z"
+    fill="currentColor" />
+</svg>
+  

langserve/chat_playground/src/assets/EmptyState.svg

@@ -0,0 +1,21 @@
+<svg width="76" height="64" viewBox="0 0 76 64" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g id="Group 145">
+<path id="Vector" d="M46.897 46.0711C56.7342 36.282 58.3487 22.0173 50.5032 14.2101C42.6577 6.40285 28.3231 8.00951 18.486 17.7986C8.64888 27.5878 7.03435 41.8524 14.8798 49.6597C22.7253 57.4669 37.0599 55.8602 46.897 46.0711Z" stroke="url(#paint0_linear_10720_23418)" stroke-width="1.5" stroke-miterlimit="10"/>
+<path id="Vector_2" d="M52.3165 46.0721C62.1536 36.2829 63.7681 22.0183 55.9226 14.211C48.0771 6.40383 33.7426 8.01049 23.9054 17.7996C14.0683 27.5888 12.4538 41.8534 20.2993 49.6606C28.1448 57.4679 42.4794 55.8612 52.3165 46.0721Z" stroke="url(#paint1_linear_10720_23418)" stroke-width="1.5" stroke-miterlimit="10"/>
+<path id="Vector_3" d="M57.7354 46.0721C67.5725 36.2829 69.1871 22.0183 61.3416 14.211C53.4961 6.40383 39.1615 8.01049 29.3244 17.7996C19.4873 27.5888 17.8727 41.8534 25.7182 49.6606C33.5637 57.4679 47.8983 55.8612 57.7354 46.0721Z" stroke="url(#paint2_linear_10720_23418)" stroke-width="1.5" stroke-miterlimit="10"/>
+</g>
+<defs>
+<linearGradient id="paint0_linear_10720_23418" x1="7.45971" y1="29.1331" x2="55.0891" y2="34.5103" gradientUnits="userSpaceOnUse">
+<stop stop-color="#A75A3D"/>
+<stop offset="1" stop-color="#BCB2FD"/>
+</linearGradient>
+<linearGradient id="paint1_linear_10720_23418" x1="12.9979" y1="29.1441" x2="60.6273" y2="34.5213" gradientUnits="userSpaceOnUse">
+<stop stop-color="#BB5831"/>
+<stop offset="1" stop-color="#BCB2FD"/>
+</linearGradient>
+<linearGradient id="paint2_linear_10720_23418" x1="18.5356" y1="29.1542" x2="66.165" y2="34.5313" gradientUnits="userSpaceOnUse">
+<stop stop-color="#CF5726"/>
+<stop offset="1" stop-color="#BCB2FD"/>
+</linearGradient>
+</defs>
+</svg>

langserve/chat_playground/src/assets/LangServeLogo.svg

@@ -0,0 +1,12 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="140" height="23" viewBox="0 0 140 23" fill="none">
+  <path d="M51.7334 4.44824V17.796H60.8269V15.7806H53.7978V4.44824H51.7334Z" fill="#043D5C"/>
+  <path d="M65.6734 7.61523C63.6917 7.61523 62.2098 8.55468 61.6081 10.1923C61.5698 10.2971 61.454 10.6132 61.454 10.6132L63.1527 11.7243L63.3834 11.1159C63.777 10.0788 64.5049 9.59538 65.6734 9.59538C66.8418 9.59538 67.5105 10.1686 67.4983 11.2973C67.4983 11.3431 67.4949 11.4813 67.4949 11.4813C67.4949 11.4813 65.9485 11.7349 65.3112 11.8714C62.592 12.4525 61.4531 13.502 61.4531 15.2189C61.4531 16.1337 61.9555 17.1242 62.8724 17.6797C63.4226 18.0126 64.141 18.1385 64.9342 18.1385C65.4557 18.1385 65.9624 18.0601 66.4317 17.9157C67.4983 17.5574 67.7961 16.853 67.7961 16.853V17.774H69.5628V11.1899C69.5628 8.95177 68.1087 7.61523 65.6734 7.61523ZM67.5053 14.6413C67.5053 15.3333 66.76 16.308 65.0238 16.308C64.5336 16.308 64.1862 16.1768 63.9546 15.9813C63.6447 15.7198 63.5428 15.3439 63.5855 15.012C63.6037 14.8676 63.6899 14.5568 64.0095 14.2873C64.336 14.0118 64.9133 13.8145 65.8048 13.6182C66.538 13.4571 67.5062 13.2792 67.5062 13.2792V14.6422L67.5053 14.6413Z" fill="#043D5C"/>
+  <path d="M76.0944 7.61514C75.8489 7.61514 75.6095 7.63275 75.377 7.66532C73.7949 7.90569 73.3317 8.71835 73.3317 8.71835L73.3335 7.90569H71.3535V17.7756H73.4179V12.3009C73.4179 10.4405 74.7597 9.59353 76.0065 9.59353C77.3543 9.59353 78.0091 10.3261 78.0091 11.8343V17.7756H80.0735V11.5473C80.0735 9.12072 78.5489 7.61426 76.0953 7.61426L76.0944 7.61514Z" fill="#043D5C"/>
+  <path d="M88.6883 7.89874V8.91567C88.6883 8.91567 88.1824 7.61523 85.8812 7.61523C83.0218 7.61523 81.2456 9.61035 81.2456 12.8231C81.2456 14.636 81.8185 16.0632 82.8294 16.9648C83.6156 17.6657 84.6657 18.0249 85.916 18.0495C86.7858 18.0663 87.3492 17.8268 87.7009 17.6005C88.3766 17.1656 88.6273 16.7526 88.6273 16.7526C88.6273 16.7526 88.5986 17.0758 88.5464 17.5133C88.5089 17.8303 88.4384 18.0531 88.4384 18.0531C88.1241 19.1845 87.2046 19.8386 85.8638 19.8386C84.5229 19.8386 83.7105 19.3922 83.5495 18.5127L81.5425 19.1184C81.889 20.8089 83.458 21.8179 85.7392 21.8179C87.29 21.8179 88.5054 21.3918 89.3526 20.55C90.2068 19.7013 90.6404 18.4783 90.6404 16.9146V7.89786H88.6883V7.89874ZM88.5586 12.9129C88.5586 14.8887 87.6043 16.0685 86.0057 16.0685C84.293 16.0685 83.3109 14.8852 83.3109 12.8231C83.3109 10.7611 84.293 9.59538 86.0057 9.59538C87.566 9.59538 88.5438 10.7699 88.5586 12.6611V12.9129Z" fill="#043D5C"/>
+  <path d="M102.741 15.7585C103.02 15.247 103.161 14.6439 103.161 13.965C103.161 13.2862 103.042 12.765 102.806 12.3432C102.57 11.9197 102.265 11.5737 101.901 11.314C101.534 11.0525 101.147 10.8465 100.752 10.7003C100.355 10.5541 99.9898 10.4362 99.6686 10.3508L97.3116 9.68602C97.0138 9.60589 96.716 9.50464 96.4252 9.38402C96.1318 9.26252 95.8819 9.09435 95.6843 8.88392C95.4823 8.66997 95.3795 8.39174 95.3795 8.05453C95.3795 7.70058 95.4988 7.38626 95.733 7.12036C95.9655 6.8571 96.2755 6.65284 96.6551 6.51285C97.0321 6.37285 97.4474 6.30506 97.8897 6.31122C98.3442 6.32355 98.7787 6.41776 99.1801 6.59209C99.5832 6.76642 99.9298 7.02351 100.21 7.35632C100.483 7.68033 100.672 8.07478 100.773 8.52997L103.051 8.12937C102.857 7.34048 102.526 6.65636 102.066 6.09639C101.598 5.52585 101.008 5.08386 100.311 4.78363C99.6137 4.48251 98.8083 4.32667 97.9167 4.32051C97.0391 4.31434 96.2267 4.4561 95.5084 4.74224C94.7918 5.02751 94.2145 5.46246 93.7923 6.03476C93.3708 6.60617 93.1566 7.32287 93.1566 8.16458C93.1566 8.7404 93.2515 9.22906 93.4396 9.61822C93.6277 10.0083 93.8732 10.3323 94.1684 10.5832C94.4653 10.835 94.784 11.034 95.1166 11.1766C95.4518 11.3201 95.7679 11.4364 96.0569 11.5209L99.4544 12.5369C99.6999 12.6118 99.9193 12.7016 100.107 12.8037C100.295 12.9067 100.452 13.023 100.573 13.1489C100.695 13.2756 100.789 13.4218 100.852 13.5838C100.915 13.7458 100.946 13.9237 100.946 14.1121C100.946 14.5338 100.812 14.8957 100.546 15.1854C100.284 15.4724 99.9385 15.6943 99.5197 15.8431C99.1026 15.9919 98.6524 16.0676 98.1805 16.0676C97.3839 16.0676 96.6699 15.8475 96.0595 15.4134C95.4579 14.9855 95.0513 14.3745 94.851 13.5979L92.6499 13.936C92.7857 14.7742 93.0983 15.5094 93.5789 16.1204C94.0683 16.7429 94.7021 17.228 95.4614 17.5617C96.2223 17.8963 97.0913 18.0662 98.0421 18.0662C98.7108 18.0662 99.3595 17.9799 99.9716 17.8091C100.582 17.6392 101.132 17.3803 101.607 17.0396C102.08 16.7006 102.462 16.2683 102.739 15.7568L102.741 15.7585Z" fill="#043D5C"/>
+  <path d="M113.245 15.2689L111.168 14.6332C110.954 15.0814 110.651 15.4291 110.264 15.6686C109.869 15.9134 109.392 16.0375 108.848 16.0375C108.004 16.0375 107.351 15.754 106.908 15.1949C106.563 14.7582 106.35 14.1736 106.276 13.456L106.271 13.4076H113.371C113.449 12.2498 113.318 11.2302 112.981 10.3753C112.642 9.51334 112.114 8.83627 111.414 8.36258C110.714 7.88978 109.85 7.64941 108.847 7.64941C107.897 7.64941 107.05 7.86953 106.328 8.30359C105.607 8.73766 105.035 9.3575 104.63 10.1446C104.224 10.9326 104.018 11.8747 104.018 12.9462C104.018 13.9376 104.228 14.826 104.643 15.5859C105.057 16.3457 105.642 16.9488 106.382 17.3767C107.122 17.8055 107.994 18.0221 108.974 18.0221C109.955 18.0221 110.77 17.7747 111.542 17.2851C112.3 16.8053 112.873 16.1265 113.243 15.2671L113.245 15.2689ZM106.331 11.7744C106.432 11.1959 106.627 10.7178 106.909 10.3551C107.351 9.78717 108.034 9.49838 108.94 9.49838C109.742 9.49838 110.341 9.75459 110.717 10.2591C110.981 10.6148 111.156 11.1255 111.236 11.7761L111.243 11.8254H106.322L106.331 11.7735V11.7744Z" fill="#043D5C"/>
+  <path d="M117.531 10.7923C117.724 10.5316 117.978 10.3142 118.285 10.1469C118.589 9.96199 118.939 9.85193 119.323 9.82023C119.68 9.79118 120.004 9.81407 120.285 9.88891V7.91932C120.011 7.87618 119.724 7.86473 119.434 7.8841C119.13 7.90524 118.83 7.96335 118.542 8.05667C118.253 8.15 117.987 8.27943 117.749 8.44143C117.469 8.61665 117.228 8.8394 117.029 9.10618C116.941 9.22416 116.861 9.34919 116.783 9.49094L116.701 9.63798V7.92637H114.855V17.7488H116.964V12.7548C116.964 12.3736 117.01 12.0126 117.101 11.6833C117.194 11.3522 117.337 11.052 117.531 10.7923Z" fill="#043D5C"/>
+  <path d="M127.917 7.92578L125.426 15.201L122.928 7.92578H120.833L124.359 17.7482H126.494L130.019 7.92578H127.917Z" fill="#043D5C"/>
+  <path d="M138.918 10.3753C138.579 9.51334 138.051 8.83627 137.351 8.36258C136.651 7.88978 135.787 7.64941 134.784 7.64941C133.834 7.64941 132.987 7.86953 132.265 8.30359C131.544 8.73766 130.972 9.3575 130.567 10.1446C130.16 10.9326 129.955 11.8756 129.955 12.9462C129.955 13.9376 130.165 14.826 130.58 15.5859C130.994 16.3457 131.579 16.9488 132.319 17.3767C133.059 17.8055 133.931 18.0221 134.911 18.0221C135.892 18.0221 136.706 17.7747 137.479 17.2851C138.237 16.8053 138.81 16.1265 139.18 15.2671L137.103 14.6314C136.889 15.0796 136.585 15.4274 136.199 15.6669C135.804 15.9116 135.328 16.0358 134.783 16.0358C133.939 16.0358 133.286 15.7523 132.844 15.1932C132.499 14.7565 132.285 14.1718 132.211 13.4543L132.206 13.4058H139.307C139.384 12.248 139.254 11.2285 138.917 10.3736L138.918 10.3753ZM132.258 11.8254L132.266 11.7735C132.367 11.195 132.561 10.7178 132.844 10.3542C133.287 9.78628 133.969 9.49749 134.875 9.49749C135.678 9.49749 136.276 9.75371 136.652 10.2582C136.917 10.6139 137.092 11.1246 137.172 11.7752L137.178 11.8245H132.258V11.8254Z" fill="#043D5C"/>
+  <path d="M16.343 16.9848C16.3204 17.4497 15.7466 17.6381 15.3582 17.5808C15.2729 17.3722 15.1545 17.1635 14.9029 17.2797C14.6486 17.3819 14.37 17.2612 14.4292 16.9408C15.1449 16.9135 15.1911 15.9652 15.3321 15.4096C15.5837 16.0048 15.7631 16.6476 16.343 16.9848ZM43.1377 11.1333C43.1377 17.1987 38.2575 22.1337 32.2593 22.1337H11.2085C5.21031 22.1328 0.330078 17.1987 0.330078 11.1333C0.330078 5.06778 5.21031 0.132812 11.2085 0.132812H32.2601C38.2583 0.132812 43.1386 5.06778 43.1386 11.1333H43.1377ZM21.2937 16.6493C21.4653 16.4389 20.6729 15.8472 20.511 15.6289C20.1819 15.2679 20.1801 14.7493 19.9581 14.3276C19.4148 13.0553 18.7905 11.7918 17.9181 10.715C16.9951 9.53698 15.8571 8.56144 14.8567 7.45558C14.114 6.68342 13.9164 5.58461 13.2607 4.75434C12.3578 3.40547 9.50108 3.03744 9.08228 4.94275C9.08402 5.00175 9.06574 5.04048 9.01437 5.07834C8.78276 5.24827 8.57641 5.44197 8.40401 5.67794C7.98086 6.27401 7.91468 7.28653 8.44406 7.82185C8.46148 7.53923 8.47106 7.27333 8.69221 7.06994C9.10144 7.42477 9.71963 7.55067 10.1933 7.28565C11.2416 8.79828 10.9795 10.8911 11.8119 12.5209C12.0418 12.9065 12.2725 13.2992 12.5685 13.6382C12.808 14.015 13.6351 14.4596 13.683 14.8074C13.6917 15.4052 13.6221 16.0603 14.0104 16.5604C14.1924 16.9355 13.744 17.3123 13.3818 17.2656C12.9116 17.3308 12.3387 16.946 11.9268 17.1829C11.7814 17.3422 11.4967 17.1662 11.3713 17.3871C11.3278 17.5016 11.0927 17.6627 11.2329 17.7719C11.3887 17.6522 11.5333 17.528 11.7431 17.5985C11.7118 17.771 11.8467 17.7957 11.953 17.8459C11.9495 17.963 11.8816 18.0827 11.9704 18.1822C12.074 18.0765 12.1358 17.9269 12.3004 17.8828C12.8463 18.6198 13.4027 17.1371 14.5859 17.8045C14.3456 17.7922 14.1332 17.823 13.9704 18.0228C13.9303 18.0677 13.8963 18.1206 13.9669 18.1787C14.6051 17.7622 14.6016 18.3204 15.0161 18.1496C15.3347 17.9815 15.6508 17.771 16.0295 17.8309C15.6612 17.9383 15.6464 18.2377 15.4314 18.4904C15.3948 18.5291 15.3774 18.5731 15.4201 18.6374C16.1837 18.5722 16.2463 18.3152 16.8645 18.0008C17.3243 17.7164 17.7831 18.4058 18.1819 18.0132C18.2698 17.9277 18.39 17.9568 18.4988 17.9454C18.3595 17.1943 16.8288 18.0827 16.8532 17.0763C17.346 16.7374 17.2328 16.0894 17.2659 15.5655C17.8327 15.8833 18.4623 16.0682 19.0178 16.3711C19.2972 16.8281 19.7369 17.4329 20.3229 17.3933C20.3386 17.3475 20.3525 17.307 20.3691 17.2604C20.5467 17.2912 20.7748 17.41 20.8723 17.1829C21.1379 17.4637 21.528 17.4497 21.8754 17.3775C22.1322 17.1662 21.3921 16.8659 21.2929 16.6484L21.2937 16.6493ZM35.6828 14.8224C35.6828 13.5624 34.6728 12.5411 33.4268 12.5411C32.1809 12.5411 31.1709 13.5624 31.1709 14.8224C31.1709 16.0823 32.1809 17.1036 33.4268 17.1036C34.6728 17.1036 35.6828 16.0823 35.6828 14.8224ZM35.85 6.1754C34.1095 4.01123 30.551 4.02268 27.9032 6.20093C26.8549 7.0629 26.1069 8.13529 25.6925 9.25084C25.6368 9.40139 25.622 9.5634 25.6524 9.721C25.7378 10.1612 25.9084 11.1641 25.8492 11.9389C25.8048 12.5253 25.7317 12.9303 25.6768 13.1698C25.6394 13.3335 25.561 13.4832 25.4487 13.6073C25.4487 13.6073 24.9454 14.1629 24.7487 14.3804C24.5179 14.6357 24.1801 14.9492 23.9076 15.1825C23.7404 15.3251 23.433 15.5074 23.2754 15.5937C23.1239 15.6773 22.8932 15.8463 23.0743 16.0559C23.2554 16.2654 24.7077 17.639 24.7077 17.639C24.7077 17.639 24.9672 17.8388 25.1744 17.3669C25.3816 16.895 25.8719 16.3165 26.147 16.0427C26.43 15.7609 26.9576 15.2917 27.3781 15.1385C27.763 14.9985 28.7782 14.7194 30.4891 15.1385C30.6049 15.1534 30.7129 15.1684 30.8156 15.1842C30.8 15.0663 30.7913 14.9456 30.7913 14.8241C30.7913 13.3538 31.9737 12.1581 33.4277 12.1581C34.1565 12.1581 34.8173 12.4592 35.2945 12.9435C37.0228 10.8057 37.3215 8.00675 35.8491 6.17716L35.85 6.1754Z" fill="#043D5C"/>
+</svg>

langserve/chat_playground/src/assets/PadlockIcon.svg

@@ -0,0 +1,6 @@
+<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+  <path fill-rule="evenodd" clip-rule="evenodd"
+    d="M4 5.33301C4 3.12387 5.79086 1.33301 8 1.33301C10.2091 1.33301 12 3.12387 12 5.33301V6.76656C12.1884 6.80784 12.3692 6.86796 12.544 6.95699C13.0457 7.21265 13.4537 7.6206 13.7093 8.12237C13.8742 8.44592 13.9399 8.79039 13.9705 9.16512C14 9.52592 14 9.96882 14 10.5055V10.8272C14 11.3639 14 11.8068 13.9705 12.1676C13.9399 12.5423 13.8742 12.8868 13.7093 13.2103C13.4537 13.7121 13.0457 14.12 12.544 14.3757C12.2204 14.5406 11.8759 14.6063 11.5012 14.6369C11.1404 14.6664 10.6975 14.6663 10.1609 14.6663H5.83912C5.30248 14.6663 4.85958 14.6664 4.49878 14.6369C4.12405 14.6063 3.77958 14.5406 3.45603 14.3757C2.95426 14.12 2.54631 13.7121 2.29065 13.2103C2.12579 12.8868 2.06008 12.5423 2.02946 12.1676C1.99998 11.8068 1.99999 11.3639 2 10.8272V10.5055C1.99999 9.96883 1.99998 9.52592 2.02946 9.16512C2.06008 8.79039 2.12579 8.44592 2.29065 8.12237C2.54631 7.6206 2.95426 7.21265 3.45603 6.95699C3.63076 6.86796 3.81159 6.80784 4 6.76656V5.33301ZM5.33333 6.66742C5.49181 6.66634 5.66026 6.66634 5.83913 6.66634H10.1609C10.3397 6.66634 10.5082 6.66634 10.6667 6.66742V5.33301C10.6667 3.86025 9.47276 2.66634 8 2.66634C6.52724 2.66634 5.33333 3.86025 5.33333 5.33301V6.66742ZM4.60736 8.02471C4.31508 8.04859 4.16561 8.09187 4.06135 8.145C3.81046 8.27283 3.60649 8.4768 3.47866 8.72769C3.42553 8.83195 3.38225 8.98142 3.35837 9.2737C3.33385 9.57376 3.33333 9.96195 3.33333 10.533V10.7997C3.33333 11.3707 3.33385 11.7589 3.35837 12.059C3.38225 12.3513 3.42553 12.5007 3.47866 12.605C3.60649 12.8559 3.81046 13.0599 4.06135 13.1877C4.16561 13.2408 4.31508 13.2841 4.60736 13.308C4.90742 13.3325 5.29561 13.333 5.86667 13.333H10.1333C10.7044 13.333 11.0926 13.3325 11.3926 13.308C11.6849 13.2841 11.8344 13.2408 11.9387 13.1877C12.1895 13.0599 12.3935 12.8559 12.5213 12.605C12.5745 12.5007 12.6178 12.3513 12.6416 12.059C12.6661 11.7589 12.6667 11.3707 12.6667 10.7997V10.533C12.6667 9.96195 12.6661 9.57376 12.6416 9.2737C12.6178 8.98142 12.5745 8.83195 12.5213 8.72769C12.3935 8.4768 12.1895 8.27283 11.9387 8.145C11.8344 8.09187 11.6849 8.04859 11.3926 8.02471C11.0926 8.00019 10.7044 7.99967 10.1333 7.99967H5.86667C5.29561 7.99967 4.90742 8.00019 4.60736 8.02471Z"
+    fill="currentColor" />
+</svg>
+  

langserve/chat_playground/src/assets/PlusIcon.svg

@@ -0,0 +1,6 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+  <path fill-rule="evenodd" clip-rule="evenodd"
+    d="M11.9999 2.51489C12.5522 2.51489 12.9999 2.96261 12.9999 3.51489V11.0002L20.4852 11.0002C21.0375 11.0002 21.4852 11.4479 21.4852 12.0002C21.4852 12.5525 21.0375 13.0002 20.4852 13.0002H12.9999V20.4855C12.9999 21.0377 12.5522 21.4855 11.9999 21.4855C11.4476 21.4855 10.9999 21.0377 10.9999 20.4855V13.0002H3.51465C2.96236 13.0002 2.51465 12.5525 2.51465 12.0002C2.51465 11.4479 2.96236 11.0002 3.51465 11.0002L10.9999 11.0002V3.51489C10.9999 2.96261 11.4476 2.51489 11.9999 2.51489Z"
+    fill="currentColor" />
+</svg>
+  

langserve/chat_playground/src/assets/RefreshCW.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-refresh-cw"><polyline points="23 4 23 10 17 10"></polyline><polyline points="1 20 1 14 7 14"></polyline><path d="M3.51 9a9 9 0 0 1 14.85-3.36L23 10M1 14l4.64 4.36A9 9 0 0 0 20.49 15"></path></svg>

langserve/chat_playground/src/assets/SendIcon.svg

@@ -0,0 +1,6 @@
+<svg width="20" height="21" viewBox="0 0 20 21" fill="none" xmlns="http://www.w3.org/2000/svg">
+  <path fill-rule="evenodd" clip-rule="evenodd"
+    d="M3.36651 2.85015C3.37578 2.85432 3.38505 2.85849 3.39431 2.86266L17.353 9.14401C17.5431 9.22954 17.7338 9.31532 17.8826 9.39905C18.0208 9.47682 18.2876 9.63803 18.4396 9.94548C18.6122 10.2947 18.6122 10.7043 18.4396 11.0535C18.2876 11.361 18.0208 11.5222 17.8826 11.5999C17.7338 11.6837 17.5431 11.7694 17.353 11.855L3.37128 18.1467C3.17613 18.2346 2.98174 18.3221 2.81784 18.3789C2.6676 18.4309 2.36452 18.5263 2.02916 18.4327C1.65046 18.327 1.34355 18.0493 1.20065 17.6831C1.07411 17.3587 1.13883 17.0476 1.17565 16.8929C1.21583 16.7242 1.28354 16.522 1.35152 16.3191L3.28934 10.5306L1.35514 4.70306C1.35194 4.69342 1.34873 4.68377 1.34553 4.67412C1.27829 4.47166 1.21126 4.26982 1.17161 4.10129C1.13521 3.94656 1.07155 3.63604 1.19844 3.31251C1.34183 2.9469 1.64871 2.66994 2.02706 2.56467C2.36186 2.47151 2.66425 2.56656 2.81444 2.61859C2.97804 2.67526 3.17198 2.76257 3.36651 2.85015ZM3.05652 4.5383L4.75852 9.66616H8.75109C9.21133 9.66616 9.58442 10.0393 9.58442 10.4995C9.58442 10.9597 9.21133 11.3328 8.75109 11.3328H4.77834L3.06259 16.458L16.3037 10.4995L3.05652 4.5383Z"
+    fill="#fff" />
+</svg>
+  

langserve/chat_playground/src/assets/ShareIcon.svg

@@ -0,0 +1,6 @@
+<svg width="20" height="21" viewBox="0 0 20 21" fill="none" xmlns="http://www.w3.org/2000/svg">
+  <path fill-rule="evenodd" clip-rule="evenodd"
+    d="M9.41009 2.41009C9.73553 2.08466 10.2632 2.08466 10.5886 2.41009L13.9219 5.74343C14.2474 6.06886 14.2474 6.5965 13.9219 6.92194C13.5965 7.24738 13.0689 7.24738 12.7434 6.92194L10.8327 5.01119V12.9993C10.8327 13.4596 10.4596 13.8327 9.99935 13.8327C9.53911 13.8327 9.16602 13.4596 9.16602 12.9993V5.01119L7.25527 6.92194C6.92984 7.24738 6.4022 7.24738 6.07676 6.92194C5.75132 6.5965 5.75132 6.06886 6.07676 5.74343L9.41009 2.41009ZM2.49935 9.66602C2.95959 9.66602 3.33268 10.0391 3.33268 10.4993V13.9993C3.33268 14.7132 3.33333 15.1984 3.36398 15.5735C3.39383 15.9388 3.44793 16.1257 3.51434 16.256C3.67413 16.5696 3.9291 16.8246 4.2427 16.9844C4.37303 17.0508 4.55987 17.1049 4.92521 17.1347C5.30029 17.1654 5.78553 17.166 6.49935 17.166H13.4993C14.2132 17.166 14.6984 17.1654 15.0735 17.1347C15.4388 17.1049 15.6257 17.0508 15.756 16.9844C16.0696 16.8246 16.3246 16.5696 16.4844 16.256C16.5508 16.1257 16.6049 15.9388 16.6347 15.5735C16.6654 15.1984 16.666 14.7132 16.666 13.9993V10.4993C16.666 10.0391 17.0391 9.66602 17.4993 9.66602C17.9596 9.66602 18.3327 10.0391 18.3327 10.4993V14.0338C18.3327 14.7046 18.3327 15.2582 18.2959 15.7092C18.2576 16.1776 18.1754 16.6082 17.9694 17.0127C17.6498 17.6399 17.1399 18.1498 16.5126 18.4694C16.1082 18.6754 15.6776 18.7576 15.2092 18.7959C14.7582 18.8327 14.2046 18.8327 13.5338 18.8327H6.46491C5.79411 18.8327 5.24049 18.8327 4.78949 18.7959C4.32108 18.7576 3.89049 18.6754 3.48605 18.4694C2.85884 18.1498 2.34891 17.6399 2.02933 17.0127C1.82325 16.6082 1.74112 16.1776 1.70284 15.7092C1.666 15.2582 1.66601 14.7046 1.66602 14.0338L1.66602 10.4993C1.66602 10.0391 2.03911 9.66602 2.49935 9.66602Z"
+    fill="currentColor" />
+</svg>
+  

langserve/chat_playground/src/assets/ThumbsDownIcon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-thumbs-down"><path d="M10 15v4a3 3 0 0 0 3 3l4-9V2H5.72a2 2 0 0 0-2 1.7l-1.38 9a2 2 0 0 0 2 2.3zm7-13h2.67A2.31 2.31 0 0 1 22 4v7a2.31 2.31 0 0 1-2.33 2H17"></path></svg>

langserve/chat_playground/src/assets/ThumbsUpIcon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-thumbs-up"><path d="M14 9V5a3 3 0 0 0-3-3l-4 9v11h11.28a2 2 0 0 0 2-1.7l1.38-9a2 2 0 0 0-2-2.3zM7 22H4a2 2 0 0 1-2-2v-7a2 2 0 0 1 2-2h3"></path></svg>

langserve/chat_playground/src/assets/TrashIcon.svg

@@ -0,0 +1,6 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+  <path fill-rule="evenodd" clip-rule="evenodd"
+    d="M8 3C8 2.44772 8.44772 2 9 2H15C15.5523 2 16 2.44772 16 3C16 3.55228 15.5523 4 15 4H9C8.44772 4 8 3.55228 8 3ZM4.99224 5H3C2.44772 5 2 5.44772 2 6C2 6.55228 2.44772 7 3 7H4.06445L4.70614 16.6254C4.75649 17.3809 4.79816 18.006 4.87287 18.5149C4.95066 19.0447 5.07405 19.5288 5.33109 19.98C5.73123 20.6824 6.33479 21.247 7.06223 21.5996C7.52952 21.826 8.0208 21.917 8.55459 21.9593C9.06728 22 9.69383 22 10.4509 22H13.5491C14.3062 22 14.9327 22 15.4454 21.9593C15.9792 21.917 16.4705 21.826 16.9378 21.5996C17.6652 21.247 18.2688 20.6824 18.6689 19.98C18.926 19.5288 19.0493 19.0447 19.1271 18.5149C19.2018 18.006 19.2435 17.3808 19.2939 16.6253L19.9356 7H21C21.5523 7 22 6.55228 22 6C22 5.44772 21.5523 5 21 5H19.0078C19.0019 4.99995 18.9961 4.99995 18.9903 5H5.00974C5.00392 4.99995 4.99809 4.99995 4.99224 5ZM17.9311 7H6.06889L6.69907 16.4528C6.75274 17.2578 6.78984 17.8034 6.85166 18.2243C6.9117 18.6333 6.98505 18.8429 7.06888 18.99C7.26895 19.3412 7.57072 19.6235 7.93444 19.7998C8.08684 19.8736 8.30086 19.9329 8.71286 19.9656C9.13703 19.9993 9.68385 20 10.4907 20H13.5093C14.3161 20 14.863 19.9993 15.2871 19.9656C15.6991 19.9329 15.9132 19.8736 16.0656 19.7998C16.4293 19.6235 16.7311 19.3412 16.9311 18.99C17.015 18.8429 17.0883 18.6333 17.1483 18.2243C17.2102 17.8034 17.2473 17.2578 17.3009 16.4528L17.9311 7Z"
+    fill="currentColor" />
+</svg>
+  

langserve/chat_playground/src/assets/XCircle.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-x-circle"><circle cx="12" cy="12" r="10"></circle><line x1="15" y1="9" x2="9" y2="15"></line><line x1="9" y1="9" x2="15" y2="15"></line></svg>

langserve/chat_playground/src/components/AutosizeTextarea.tsx

@@ -0,0 +1,55 @@
+import { Ref } from "react";
+import { cn } from "../utils/cn";
+
+const COMMON_CLS = cn(
+  "text-lg col-[1] row-[1] m-0 resize-none overflow-hidden whitespace-pre-wrap break-words border-none bg-transparent p-0"
+);
+
+export function AutosizeTextarea(props: {
+  id?: string;
+  inputRef?: Ref<HTMLTextAreaElement>;
+  value?: string | null | undefined;
+  placeholder?: string;
+  className?: string;
+  onChange?: (e: string) => void;
+  onFocus?: () => void;
+  onBlur?: () => void;
+  onKeyDown?: (e: React.KeyboardEvent<HTMLTextAreaElement>) => void;
+  autoFocus?: boolean;
+  readOnly?: boolean;
+  cursorPointer?: boolean;
+  disabled?: boolean;
+  fullHeight?: boolean;
+}) {
+  return (
+    <div className={cn("grid w-full", props.className) + (props.fullHeight ? "" : " max-h-80 overflow-auto")}>
+      <textarea
+        ref={props.inputRef}
+        id={props.id}
+        className={cn(
+          COMMON_CLS,
+          "text-transparent caret-black"
+        )}
+        disabled={props.disabled}
+        value={props.value ?? ""}
+        rows={1}
+        onChange={(e) => {
+          const target = e.target as HTMLTextAreaElement;
+          props.onChange?.(target.value);
+        }}
+        onFocus={props.onFocus}
+        onBlur={props.onBlur}
+        placeholder={props.placeholder}
+        readOnly={props.readOnly}
+        autoFocus={props.autoFocus && !props.readOnly}
+        onKeyDown={props.onKeyDown}
+      />
+      <div
+        aria-hidden
+        className={cn(COMMON_CLS, "pointer-events-none select-none")}
+      >
+        {props.value}{" "}
+      </div>
+    </div>
+  );
+}

langserve/chat_playground/src/components/ChatMessage.tsx

@@ -0,0 +1,118 @@
+import { useState } from "react";
+import { CorrectnessFeedback } from "./feedback/CorrectnessFeedback";
+import { resolveApiUrl } from "../utils/url";
+import { AutosizeTextarea } from "./AutosizeTextarea";
+import TrashIcon from "../assets/TrashIcon.svg?react";
+import RefreshCW from "../assets/RefreshCW.svg?react";
+
+export type ChatMessageType = "human" | "ai" | "function" | "tool" | "system";
+
+export type ChatMessageBody = {
+  type: ChatMessageType;
+  content: string;
+  runId?: string;
+}
+
+export function ChatMessage(props: {
+  message: ChatMessageBody;
+  isLoading?: boolean;
+  onError?: (e: any) => void;
+  onTypeChange?: (newValue: string) => void;
+  onChange?: (newValue: string) => void;
+  onRemove?: (e: any) => void;
+  onRegenerate?: (e?: any) => void;
+  isFinalMessage?: boolean;
+  feedbackEnabled?: boolean;
+  publicTraceLinksEnabled?: boolean;
+}) {
+  const { message, feedbackEnabled, publicTraceLinksEnabled, onError, isLoading } = props;
+  const { content, type, runId } = message;
+
+  const [publicTraceLink, setPublicTraceLink] = useState<string | null>(null);
+  const [messageActionIsLoading, setMessageActionIsLoading] = useState(false);
+  const openPublicTrace = async () => {
+    if (messageActionIsLoading) {
+      return;
+    }
+    if (publicTraceLink) {
+      window.open(publicTraceLink, '_blank');
+      return;
+    }
+    setMessageActionIsLoading(true);
+    const payload = { run_id: runId };
+    const response = await fetch(resolveApiUrl("/public_trace_link"), {
+      method: "PUT",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify(payload),
+    });
+
+    if (!response.ok) {
+      if (response.status === 404) {
+        onError?.(new Error(`Feedback endpoint not found. Please enable it in your LangServe endpoint.`));
+      } else {
+        try {
+          const errorResponse = await response.json();
+          onError?.(new Error(`${errorResponse.detail}`));
+        } catch (e) {
+          onError?.(new Error(`Request failed with status: ${response.status}`));
+        }
+      }
+      setMessageActionIsLoading(false);
+      throw new Error(`Failed request ${response.status}`)
+    }
+    const parsedResponse = await response.json();
+    setMessageActionIsLoading(false);
+    setPublicTraceLink(parsedResponse.public_url);
+    window.open(parsedResponse.public_url, '_blank');
+  };
+
+  return (
+    <div className="mb-8 group">
+      <div className="flex justify-between">
+        <select
+          className="font-medium text-transform uppercase mb-2 appearance-none"
+          defaultValue={type}
+          onChange={(e) => props.onTypeChange?.(e.target.value)}
+        >
+          <option value="human">HUMAN</option>
+          <option value="ai">AI</option>
+          <option value="system">SYSTEM</option>
+        </select>
+        <span className="flex">
+          {props.isFinalMessage &&
+            type === "human" && 
+            <RefreshCW className="opacity-0 group-hover:opacity-50 transition-opacity duration-200 cursor-pointer h-4 w-4 mr-2" onMouseUp={props.onRegenerate}></RefreshCW>}
+          <TrashIcon
+            className="opacity-0 group-hover:opacity-50 transition-opacity duration-200 cursor-pointer h-4 w-4"
+            onMouseUp={props.onRemove}
+          ></TrashIcon>
+        </span>
+      </div>
+      <AutosizeTextarea value={content} fullHeight={true} onChange={props.onChange} onKeyDown={(e) => {
+        if (
+          e.key === 'Enter' &&
+          !e.shiftKey &&
+          props.isFinalMessage &&
+          type === "human"
+        ) {
+          e.preventDefault();
+          props.onRegenerate?.();
+        }
+      }}></AutosizeTextarea>
+      {type === "ai" && !isLoading && runId != null && (
+        <div className="mt-2 flex items-center">
+          {feedbackEnabled && <span className="mr-2"><CorrectnessFeedback runId={runId} onError={props.onError}></CorrectnessFeedback></span>}
+          {publicTraceLinksEnabled && <>
+            <button
+              className="bg-button-inline p-2 rounded-lg text-xs font-medium hover:opacity-80"
+              disabled={messageActionIsLoading || isLoading}
+              onMouseUp={openPublicTrace}
+            >
+              🛠️ View LangSmith trace
+            </button>
+          </>}
+        </div>
+      )}
+    </div> 
+  )
+};

langserve/chat_playground/src/components/ChatWindow.tsx

@@ -0,0 +1,217 @@
+import { useState, useRef } from "react";
+import { ToastContainer, toast } from 'react-toastify';
+import 'react-toastify/dist/ReactToastify.css';
+
+import { AutosizeTextarea } from "./AutosizeTextarea";
+import {
+  ChatMessage,
+  type ChatMessageType,
+  type ChatMessageBody,
+} from "./ChatMessage";
+import { ShareDialog } from "./ShareDialog";
+import { useStreamCallback } from "../useStreamCallback";
+
+import ArrowUp from "../assets/ArrowUp.svg?react";
+import CircleSpinIcon from "../assets/CircleSpinIcon.svg?react";
+import EmptyState from "../assets/EmptyState.svg?react";
+import LangServeLogo from "../assets/LangServeLogo.svg?react";
+import { useFeedback, usePublicTraceLink } from "../useSchemas";
+
+export type AIMessage = {
+  content: string;
+  type: "AIMessage" | "AIMessageChunk";
+  name?: string;
+  additional_kwargs?: { [key: string]: unknown };
+}
+
+export function isAIMessage(x: unknown): x is AIMessage {
+  return x != null &&
+    typeof (x as AIMessage).content === "string" &&
+    ["AIMessageChunk", "AIMessage"].includes((x as AIMessage).type);
+}
+
+export function ChatWindow(props: {
+  startStream: (input: unknown, config: unknown) => Promise<void>;
+  stopStream: (() => void) | undefined;
+  messagesInputKey: string;
+  inputKey?: string;
+}) {
+  const { startStream, messagesInputKey, inputKey } = props;
+
+  const [currentInputValue, setCurrentInputValue] = useState("");
+  const [isLoading, setIsLoading] = useState(false);
+  const [messages, setMessages] = useState<ChatMessageBody[]>([]);
+  const messageInputRef = useRef<HTMLTextAreaElement>(null);
+
+  const feedbackEnabled = useFeedback()
+  const publicTraceLinksEnabled = usePublicTraceLink();
+
+  const submitMessage = () => {
+    const submittedValue = currentInputValue;
+    if (submittedValue.length === 0 || isLoading) {
+      return;
+    }
+    setIsLoading(true);
+    const newMessages = [
+      ...messages,
+      { type: "human", content: submittedValue } as const
+    ];
+    setMessages(newMessages);
+    setCurrentInputValue("");
+    // TODO: Add config schema support
+    if (inputKey === undefined) {
+      startStream({ [messagesInputKey]: newMessages }, {});
+    } else {
+      startStream({
+        [messagesInputKey]: newMessages.slice(0, -1),
+        [inputKey]: newMessages[newMessages.length - 1].content
+      }, {});
+    }
+  };
+
+  const regenerateMessages = () => {
+    if (isLoading) {
+      return;
+    }
+    setIsLoading(true);
+    // TODO: Add config schema support
+    if (inputKey === undefined) {
+      startStream({ [messagesInputKey]: messages }, {});
+    } else {
+      startStream({
+        [messagesInputKey]: messages.slice(0, -1),
+        [inputKey]: messages[messages.length - 1].content
+      }, {});
+    }
+  };
+  
+  useStreamCallback("onStart", () => {
+    setMessages((prevMessages) => [
+      ...prevMessages,
+      { type: "ai", content: "" },
+    ]);
+  });
+  useStreamCallback("onChunk", (_chunk, aggregatedState) => {
+    const finalOutput = aggregatedState?.final_output;
+    if (typeof finalOutput === "string") {
+      setMessages((prevMessages) => [
+        ...prevMessages.slice(0, -1),
+        { type: "ai", content: finalOutput, runId: aggregatedState?.id }
+      ]); 
+    } else if (isAIMessage(finalOutput)) {
+      setMessages((prevMessages) => [
+        ...prevMessages.slice(0, -1),
+        { type: "ai", content: finalOutput.content, runId: aggregatedState?.id }
+      ]);
+    }
+  });
+  useStreamCallback("onSuccess", () => {
+    setIsLoading(false);
+  });
+  useStreamCallback("onError", (e) => {
+    setIsLoading(false);
+    toast(e.message + "\nCheck your backend logs for errors.", { hideProgressBar: true });
+    setCurrentInputValue(messages[messages.length - 2]?.content);
+    setMessages((prevMessages) => [
+      ...prevMessages.slice(0, -2),
+    ]);
+  });
+
+  return (
+    <div className="flex flex-col h-screen w-screen">
+      <nav className="flex items-center justify-between p-8">
+        <div className="flex items-center">
+          <LangServeLogo />
+          <span className="ml-1">Playground</span>
+        </div>
+        <div className="flex items-center space-x-4">
+          <ShareDialog config={{}}>
+            <button
+              type="button"
+              className="px-3 py-1 border rounded-full px-8 py-2 share-button"
+            >
+              <span>Share</span>
+            </button>
+          </ShareDialog>
+        </div>
+      </nav>
+      <div className="flex-grow flex flex-col items-center justify-center mt-8">
+        {messages.length > 0 ? (
+          <div className="flex flex-col-reverse basis-0 overflow-auto flex-re grow max-w-[640px] w-[640px]">
+            {messages.map((message, i) => {
+              return (
+                <ChatMessage
+                  message={message}
+                  key={i}
+                  isLoading={isLoading}
+                  onError={(e: any) => toast(e.message, { hideProgressBar: true })}
+                  feedbackEnabled={feedbackEnabled.data}
+                  publicTraceLinksEnabled={publicTraceLinksEnabled.data}
+                  isFinalMessage={i === messages.length - 1}
+                  onRemove={() => setMessages(
+                    (previousMessages) => [...previousMessages.slice(0, i), ...previousMessages.slice(i + 1)]
+                  )}
+                  onTypeChange={(newValue) => {
+                    setMessages(
+                      (previousMessages) => [
+                        ...previousMessages.slice(0, i),
+                        {...message, type: newValue as ChatMessageType},
+                        ...previousMessages.slice(i + 1)
+                      ]
+                    )
+                  }}
+                  onChange={(newValue) => {
+                    setMessages(
+                      (previousMessages) => [
+                        ...previousMessages.slice(0, i),
+                        {...message, content: newValue},
+                        ...previousMessages.slice(i + 1)
+                      ]
+                    );
+                  }}
+                  onRegenerate={() => regenerateMessages()}
+                ></ChatMessage>
+              );
+            }).reverse()}
+          </div>
+        ) : (
+          <div className="flex flex-col items-center justify-center">
+            <EmptyState />
+            <h1 className="text-lg">Start testing your application</h1>
+          </div>
+        )}
+      </div>
+      <div className="m-16 mt-4 flex justify-center">
+        <div className="flex items-center p-3 rounded-[48px] border shadow-sm max-w-[768px] grow" onClick={() => messageInputRef.current?.focus()}>
+          <AutosizeTextarea
+            inputRef={messageInputRef}
+            className="flex-grow mr-4 ml-8 border-none focus:ring-0 py-2 cursor-text"
+            placeholder="Send a message..."
+            value={currentInputValue}
+            onChange={(newValue) => {
+              setCurrentInputValue(newValue);
+            }}
+            onKeyDown={(e) => {
+              if (e.key === 'Enter' && !e.shiftKey) {
+                e.preventDefault();
+                submitMessage();
+              }
+            }}
+          />
+          <button
+            className={"flex items-center justify-center px-3 py-1 rounded-[40px] " + (isLoading ? "" : currentInputValue.length > 0 ? "bg-button-green" : "bg-button-green-disabled")}
+            onClick={(e) => {
+              e.preventDefault();
+              submitMessage();
+            }}
+          >
+            {isLoading 
+              ? <CircleSpinIcon className="animate-spin w-5 h-5 text-background fill-background" />
+              : <ArrowUp className="mx-2 my-2 h-5 w-5 stroke-white" />}
+          </button>
+        </div>
+      </div>
+      <ToastContainer />
+    </div>
+  )
+}

langserve/chat_playground/src/components/ShareDialog.tsx

@@ -0,0 +1,141 @@
+import { Drawer } from "vaul";
+import { ReactNode, useEffect, useMemo, useRef, useState } from "react";
+import CheckCircleIcon from "../assets/CheckCircleIcon.svg?react";
+import CodeIcon from "../assets/CodeIcon.svg?react";
+import CopyIcon from "../assets/CopyIcon.svg?react";
+import PadlockIcon from "../assets/PadlockIcon.svg?react";
+import ShareIcon from "../assets/ShareIcon.svg?react";
+import { compressToEncodedURIComponent } from "lz-string";
+import { getStateFromUrl } from "../utils/url";
+
+const URL_LENGTH_LIMIT = 2000;
+
+function CopyButton(props: { value: string }) {
+  const [copied, setCopied] = useState(false);
+  const cbRef = useRef<number | null>(null);
+
+  function toggleCopied() {
+    setCopied(true);
+
+    if (cbRef.current != null) window.clearTimeout(cbRef.current);
+    cbRef.current = window.setTimeout(() => setCopied(false), 1500);
+  }
+
+  useEffect(() => {
+    return () => {
+      if (cbRef.current != null) {
+        window.clearTimeout(cbRef.current);
+      }
+    };
+  }, []);
+
+  return (
+    <button
+      className="px-3 py-1"
+      onClick={() => {
+        navigator.clipboard.writeText(props.value).then(toggleCopied);
+      }}
+    >
+      {copied ? <CheckCircleIcon /> : <CopyIcon />}
+    </button>
+  );
+}
+
+export function ShareDialog(props: { config: unknown; children: ReactNode }) {
+  const hash = useMemo(() => {
+    return compressToEncodedURIComponent(JSON.stringify(props.config));
+  }, [props.config]);
+
+  const state = getStateFromUrl(window.location.href);
+
+  // get base URL
+  const targetUrl = `${state.basePath}/c/${hash}`;
+
+  // .../c/[hash]/playground
+  const playgroundUrl = `${targetUrl}/playground`;
+
+  // cURL, JS: .../c/[hash]/invoke
+  // Python: .../c/[hash]
+  const invokeUrl = `${targetUrl}/invoke`;
+
+  const pythonSnippet = `
+from langserve import RemoteRunnable
+
+chain = RemoteRunnable("${targetUrl}")
+chain.invoke({ ... })
+`;
+
+  const typescriptSnippet = `
+import { RemoteRunnable } from "langchain/runnables/remote";
+
+const chain = new RemoteRunnable({ url: \`${invokeUrl}\` });
+const result = await chain.invoke({ ... });
+`;
+
+  return (
+    <Drawer.Root>
+      <Drawer.Trigger asChild>{props.children}</Drawer.Trigger>
+      <Drawer.Portal>
+        <Drawer.Overlay className="fixed inset-0 bg-black/40" />
+        <Drawer.Content className="flex justify-center items-center mt-24 fixed bottom-0 left-0 right-0 !pointer-events-none after:!bg-background">
+          <div className="p-4 bg-background max-w-[calc(800px-2rem)] rounded-t-2xl border border-divider-500 border-b-background pointer-events-auto">
+            <h3 className="flex items-center text-lg font-light">
+              <ShareIcon className="flex-shrink-0 mr-2" />
+              <span>Share</span>
+            </h3>
+
+            <hr className="border-divider-500 my-4 -mx-4" />
+
+            <div className="flex flex-col gap-3">
+              {playgroundUrl.length < URL_LENGTH_LIMIT && (
+                <div className="flex flex-col gap-2 p-3 rounded-2xl">
+                  <div className="flex items-center">
+                    <div className="w-10 h-10 flex items-center justify-center text-center text-sm bg-background rounded-xl">
+                      🦜
+                    </div>
+                    <span>Chat interface</span>
+                  </div>
+                  <div className="grid grid-cols-[auto,1fr,auto] rounded-xl text-sm items-center border">
+                    <PadlockIcon className="mx-3" />
+                    <div className="overflow-auto whitespace-nowrap py-3 no-scrollbar">
+                      {playgroundUrl.split("://")[1]}
+                    </div>
+                    <CopyButton value={playgroundUrl} />
+                  </div>
+                </div>
+              )}
+
+              <div className="flex flex-col gap-2 p-3 rounded-2xl">
+                <div className="flex items-center">
+                  <div className="w-10 h-10 flex items-center justify-center text-center text-sm bg-background rounded-xl">
+                    <CodeIcon className="w-4 h-4" />
+                  </div>
+                  <span>Get the code</span>
+                </div>
+
+                {targetUrl.length < URL_LENGTH_LIMIT && (
+                  <div className="grid grid-cols-[1fr,auto] rounded-xl text-sm items-center border">
+                    <div className="overflow-auto whitespace-nowrap px-3 py-3 no-scrollbar">
+                      Python SDK
+                    </div>
+                    <CopyButton value={pythonSnippet.trim()} />
+                  </div>
+                )}
+
+                {invokeUrl.length < URL_LENGTH_LIMIT && (
+                  <div className="grid grid-cols-[1fr,auto] rounded-xl text-sm items-center border">
+                    <div className="overflow-auto whitespace-nowrap px-3 py-3 no-scrollbar">
+                      TypeScript SDK
+                    </div>
+
+                    <CopyButton value={typescriptSnippet.trim()} />
+                  </div>
+                )}
+              </div>
+            </div>
+          </div>
+        </Drawer.Content>
+      </Drawer.Portal>
+    </Drawer.Root>
+  );
+}

langserve/chat_playground/src/components/feedback/CorrectnessFeedback.tsx

@@ -0,0 +1,102 @@
+import { toast } from "react-toastify";
+import ThumbsUpIcon from "../../assets/ThumbsUpIcon.svg?react";
+import ThumbsDownIcon from "../../assets/ThumbsDownIcon.svg?react";
+import CircleSpinIcon from "../../assets/CircleSpinIcon.svg?react";
+import CheckCircleIcon2 from "../../assets/CheckCircleIcon2.svg?react";
+import XCircle from "../../assets/XCircle.svg?react";
+import { resolveApiUrl } from "../../utils/url";
+import { useState } from "react";
+import useSWRMutation from "swr/mutation";
+
+const useFeedbackMutation = (runId: string, onError?: (e: any) => void) => {
+  interface FeedbackArguments {
+    key: string;
+    score: number;
+  }
+
+  const [lastArg, setLastArg] = useState<FeedbackArguments | null>(null);
+
+  const mutation = useSWRMutation(
+    ["feedback", runId],
+    async ([, runId], { arg }: { arg: FeedbackArguments }) => {
+      const payload = { run_id: runId, key: arg.key, score: arg.score };
+      setLastArg(arg);
+
+      const request = await fetch(resolveApiUrl("/feedback"), {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(payload),
+      });
+
+      if (!request.ok) {
+        if (request.status === 404) {
+          onError?.(new Error(`Feedback endpoint not found. Please enable it in your LangServe endpoint.`));
+        } else {
+          try {
+            const errorResponse = await request.json();
+            onError?.(new Error(`${errorResponse.detail}`));
+          } catch (e) {
+            onError?.(new Error(`Request failed with status: ${request.status}`));
+          }
+        }
+        throw new Error(`Failed request ${request.status}`)
+      }
+      const json: {
+        id: string;
+        score: number;
+      } = await request.json();
+
+      toast("Feedback sent successfully!", { hideProgressBar: true });
+      return json;
+    }
+  );
+
+  return { lastArg: mutation.isMutating ? lastArg : null, mutation };
+};
+
+export function CorrectnessFeedback(props: { runId: string, onError?: (e: any) => void }) {
+  const score = useFeedbackMutation(props.runId, props.onError);
+
+  if (props.runId == null) return null;
+  return (
+    <>
+      <button
+        type="button"
+        className={"bg-background rounded p-1 hover:opacity-80"}
+        disabled={score.mutation.isMutating}
+        onClick={() => {
+          if (score.mutation.data?.score !== 1) {
+            score.mutation.trigger({ key: "correctness", score: 1 });
+          }
+        }}
+      >
+        {score.lastArg?.score === 1 ? (
+          <CircleSpinIcon className="animate-spin w-4 h-4 text-white/50 fill-white" />
+        ) : (
+          (score.mutation.data?.score !== 1
+            ? <ThumbsUpIcon className="w-4 h-4" />
+            : <CheckCircleIcon2 className="w-4 h-4 stroke-teal-500" />)
+        )}
+      </button>
+
+      <button
+        type="button"
+        className={"bg-background rounded p-1 hover:opacity-80"}
+        disabled={score.mutation.isMutating}
+        onClick={() => {
+          if (score.mutation.data?.score !== 0) {
+            score.mutation.trigger({ key: "correctness", score: 0 });
+          }
+        }}
+      >
+        {score.lastArg?.score === 0 ? (
+          <CircleSpinIcon className="animate-spin w-4 h-4 text-white/50 fill-white" />
+        ) : (
+          (score.mutation.data?.score !== 0
+            ? <ThumbsDownIcon className="w-4 h-4" />
+            : <XCircle className="w-4 h-4 stroke-red-500" />)
+        )}
+      </button>
+    </>
+  );
+}

langserve/chat_playground/src/main.tsx

@@ -0,0 +1,11 @@
+import ReactDOM from "react-dom/client";
+import App from "./App.tsx";
+
+import dayjs from "dayjs";
+import utc from "dayjs/plugin/utc";
+import relativeDate from "dayjs/plugin/relativeTime";
+
+dayjs.extend(relativeDate);
+dayjs.extend(utc);
+
+ReactDOM.createRoot(document.getElementById("root")!).render(<App />);

langserve/chat_playground/src/types.tsx

@@ -0,0 +1,9 @@
+import type { Operation } from "fast-json-patch";
+import type { RunState } from "./useStreamLog";
+
+export interface StreamCallback {
+  onSuccess?: (ctx: { input: unknown; output: unknown }) => void;
+  onChunk?: (chunk: { ops?: Operation[] }, aggregatedState: RunState | null) => void;
+  onError?: (error: any) => void;
+  onStart?: (ctx: { input: unknown }) => void;
+}

langserve/chat_playground/src/useSchemas.tsx

@@ -0,0 +1,131 @@
+import { JsonSchema } from "@jsonforms/core";
+import { compressToEncodedURIComponent } from "lz-string";
+import { resolveApiUrl } from "./utils/url";
+import { simplifySchema } from "./utils/simplifySchema";
+
+import useSWR from "swr";
+import defaults from "./utils/defaults";
+
+declare global {
+  interface Window {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    CONFIG_SCHEMA?: any;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    INPUT_SCHEMA?: any;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    OUTPUT_SCHEMA?: any;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    FEEDBACK_ENABLED?: any;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    PUBLIC_TRACE_LINK_ENABLED?: any;
+  }
+}
+
+export function useFeedback() {
+  return useSWR(["/feedback"], async () => {
+    if (!import.meta.env.DEV && window.FEEDBACK_ENABLED) {
+      return window.FEEDBACK_ENABLED === "true";
+    }
+
+    const response = await fetch(resolveApiUrl("/feedback"), {
+      method: "HEAD",
+    });
+    return response.ok;
+  });
+}
+
+export function usePublicTraceLink() {
+  return useSWR(["/public_trace_link"], async () => {
+    if (!import.meta.env.DEV && window.PUBLIC_TRACE_LINK_ENABLED) {
+      return window.PUBLIC_TRACE_LINK_ENABLED === "true";
+    }
+
+    const response = await fetch(resolveApiUrl("/public_trace_link"), {
+      method: "HEAD",
+    });
+    return response.ok;
+  });
+}
+
+export function useConfigSchema() {
+  return useSWR(["/config_schema"], async () => {
+    let schema: JsonSchema | null = null;
+    if (!import.meta.env.DEV && window.CONFIG_SCHEMA) {
+      schema = await simplifySchema(window.CONFIG_SCHEMA);
+    } else {
+      const response = await fetch(resolveApiUrl(`/config_schema`));
+      if (!response.ok) throw new Error(await response.text());
+
+      const json = await response.json();
+      schema = await simplifySchema(json);
+    }
+
+    if (schema == null) return null;
+    return {
+      schema,
+      defaults: defaults(schema),
+    };
+  });
+}
+
+export function useInputSchema(configData?: unknown) {
+  return useSWR(
+    ["/input_schema", configData],
+    async ([, configData]) => {
+      // TODO: this won't work if we're already seeing a prefixed URL
+      const prefix = configData
+        ? `/c/${compressToEncodedURIComponent(JSON.stringify(configData))}`
+        : "";
+
+      let schema: JsonSchema | null = null;
+
+      if (!prefix && !import.meta.env.DEV && window.INPUT_SCHEMA) {
+        schema = await simplifySchema(window.INPUT_SCHEMA);
+      } else {
+        const response = await fetch(resolveApiUrl(`${prefix}/input_schema`));
+        if (!response.ok) throw new Error(await response.text());
+
+        const json = await response.json();
+        schema = await simplifySchema(json);
+      }
+
+      if (schema == null) return null;
+      return {
+        schema,
+        defaults: defaults(schema),
+      };
+    },
+    { keepPreviousData: true }
+  );
+}
+
+export function useOutputSchema(configData?: unknown) {
+  return useSWR(
+    ["/output_schema", configData],
+    async ([, configData]) => {
+      // TODO: this won't work if we're already seeing a prefixed URL
+      const prefix = configData
+        ? `/c/${compressToEncodedURIComponent(JSON.stringify(configData))}`
+        : "";
+
+      let schema: JsonSchema | null = null;
+
+      if (!prefix && !import.meta.env.DEV && window.OUTPUT_SCHEMA) {
+        schema = await simplifySchema(window.OUTPUT_SCHEMA);
+      } else {
+        const response = await fetch(resolveApiUrl(`${prefix}/output_schema`));
+        if (!response.ok) throw new Error(await response.text());
+
+        const json = await response.json();
+        schema = await simplifySchema(json);
+      }
+
+      if (schema == null) return null;
+      return {
+        schema,
+        defaults: defaults(schema),
+      };
+    },
+    { keepPreviousData: true }
+  );
+}

langserve/chat_playground/src/useStreamCallback.tsx

@@ -0,0 +1,78 @@
+import {
+  MutableRefObject,
+  createContext,
+  useContext,
+  useEffect,
+  useRef,
+} from "react";
+
+import { StreamCallback } from "./types";
+
+export const AppCallbackContext = createContext<MutableRefObject<{
+  onStart: Exclude<StreamCallback["onStart"], undefined>[];
+  onChunk: Exclude<StreamCallback["onChunk"], undefined>[];
+  onSuccess: Exclude<StreamCallback["onSuccess"], undefined>[];
+  onError: Exclude<StreamCallback["onError"], undefined>[];
+}> | null>(null);
+
+export function useAppStreamCallbacks() {
+  // callbacks handling
+  const context = useRef<{
+    onStart: Exclude<StreamCallback["onStart"], undefined>[];
+    onChunk: Exclude<StreamCallback["onChunk"], undefined>[];
+    onSuccess: Exclude<StreamCallback["onSuccess"], undefined>[];
+    onError: Exclude<StreamCallback["onError"], undefined>[];
+  }>({ onStart: [], onChunk: [], onSuccess: [], onError: [] });
+
+  const callbacks: StreamCallback = {
+    onStart(...args) {
+      for (const callback of context.current.onStart) {
+        callback(...args);
+      }
+    },
+    onChunk(...args) {
+      for (const callback of context.current.onChunk) {
+        callback(...args);
+      }
+    },
+    onSuccess(...args) {
+      for (const callback of context.current.onSuccess) {
+        callback(...args);
+      }
+    },
+    onError(...args) {
+      for (const callback of context.current.onError) {
+        callback(...args);
+      }
+    },
+  };
+
+  return { context, callbacks };
+}
+
+export function useStreamCallback<
+  Type extends "onStart" | "onChunk" | "onSuccess" | "onError"
+>(type: Type, callback: Exclude<StreamCallback[Type], undefined>) {
+  type CallbackType = Exclude<StreamCallback[Type], undefined>;
+
+  const appCbRef = useContext(AppCallbackContext);
+
+  const callbackRef = useRef<CallbackType>(callback);
+  callbackRef.current = callback;
+
+  useEffect(() => {
+    // @ts-expect-error Not sure why I can't expand the tuple
+    const current = (...args) => callbackRef.current?.(...args);
+    appCbRef?.current[type].push(current);
+
+    return () => {
+      if (!appCbRef) return;
+
+      // @ts-expect-error Assingability issues due to the tuple object
+      // eslint-disable-next-line react-hooks/exhaustive-deps
+      appCbRef.current[type] = appCbRef.current[type].filter(
+        (callbacks) => callbacks !== current
+      );
+    };
+  }, [type, appCbRef]);
+}

langserve/chat_playground/src/useStreamLog.tsx

@@ -0,0 +1,106 @@
+import { useCallback, useRef, useState } from "react";
+import { applyPatch, Operation } from "fast-json-patch";
+import { fetchEventSource } from "@microsoft/fetch-event-source";
+import { resolveApiUrl } from "./utils/url";
+import { StreamCallback } from "./types";
+
+export interface LogEntry {
+  // ID of the sub-run.
+  id: string;
+  // Name of the object being run.
+  name: string;
+  // Type of the object being run, eg. prompt, chain, llm, etc.
+  type: string;
+  // List of tags for the run.
+  tags: string[];
+  // Key-value pairs of metadata for the run.
+  metadata: { [key: string]: unknown };
+  // ISO-8601 timestamp of when the run started.
+  start_time: string;
+  // List of LLM tokens streamed by this run, if applicable.
+  streamed_output_str: string[];
+  // Final output of this run.
+  // Only available after the run has finished successfully.
+  final_output?: unknown;
+  // ISO-8601 timestamp of when the run ended.
+  // Only available after the run has finished.
+  end_time?: string;
+}
+
+export interface RunState {
+  // ID of the run.
+  id: string;
+  // List of output chunks streamed by Runnable.stream()
+  streamed_output: unknown[];
+  // Final output of the run, usually the result of aggregating (`+`) streamed_output.
+  // Only available after the run has finished successfully.
+  final_output?: unknown;
+
+  // Map of run names to sub-runs. If filters were supplied, this list will
+  // contain only the runs that matched the filters.
+  logs: { [name: string]: LogEntry };
+}
+
+function reducer(state: RunState | null, action: Operation[]) {
+  return applyPatch(state, action, true, false).newDocument;
+}
+
+export function useStreamLog(callbacks: StreamCallback = {}) {
+  const [latest, setLatest] = useState<RunState | null>(null);
+  const [controller, setController] = useState<AbortController | null>(null);
+
+  const startRef = useRef(callbacks.onStart);
+  startRef.current = callbacks.onStart;
+
+  const chunkRef = useRef(callbacks.onChunk);
+  chunkRef.current = callbacks.onChunk;
+
+  const successRef = useRef(callbacks.onSuccess);
+  successRef.current = callbacks.onSuccess;
+
+  const errorRef = useRef(callbacks.onError);
+  errorRef.current = callbacks.onError;
+
+  const startStream = useCallback(async (input: unknown, config: unknown) => {
+    const controller = new AbortController();
+    setController(controller);
+    startRef.current?.({ input });
+
+    let innerLatest: RunState | null = null;
+
+    await fetchEventSource(resolveApiUrl("/stream_log").toString(), {
+      signal: controller.signal,
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ input, config }),
+      onmessage(msg) {
+        if (msg.event === "data") {
+          innerLatest = reducer(innerLatest, JSON.parse(msg.data)?.ops);
+          setLatest(innerLatest);
+          chunkRef.current?.(JSON.parse(msg.data), innerLatest);
+        }
+      },
+      openWhenHidden: true,
+      onclose() {
+        setController(null);
+        successRef.current?.({ input, output: innerLatest?.final_output });
+      },
+      onerror(error) {
+        setController(null);
+        errorRef.current?.(error);
+        throw error;
+      },
+    });
+  }, []);
+
+  const stopStream = useCallback(() => {
+    controller?.abort();
+    setController(null);
+  }, [controller]);
+
+  return {
+    startStream,
+    stopStream: controller ? stopStream : undefined,
+    latest,
+  };
+}

langserve/chat_playground/src/utils/cn.ts

@@ -0,0 +1,7 @@
+import clsx from "clsx";
+import { ClassValue } from "clsx";
+import { twMerge } from "tailwind-merge";
+
+export function cn(...inputs: ClassValue[]) {
+  return twMerge(clsx(inputs));
+}

langserve/chat_playground/src/utils/defaults.ts

@@ -0,0 +1,220 @@
+// (c) 2015 Chute Corporation. Released under the terms of the MIT License.
+// Modified to use TypeScript and handle edge cases with tuples
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/* eslint-disable no-prototype-builtins */
+
+"use strict";
+
+/**
+ * check whether item is plain object
+ * @param {*} item
+ * @return {Boolean}
+ */
+const isObject = (item: unknown): item is Record<string, unknown> => {
+  return (
+    typeof item === "object" &&
+    item !== null &&
+    item.toString() === {}.toString()
+  );
+};
+
+/**
+ * deep JSON object clone
+ *
+ * @param {Object} source
+ * @return {Object}
+ */
+const cloneJSON = (source: any): any => {
+  return JSON.parse(JSON.stringify(source));
+};
+
+/**
+ * returns a result of deep merge of two objects
+ *
+ * @param {Object} target
+ * @param {Object} source
+ * @return {Object}
+ */
+const merge = (
+  target: Record<string, unknown>,
+  source: Record<string, unknown>
+) => {
+  target = cloneJSON(target);
+
+  for (const key in source) {
+    if (source.hasOwnProperty(key)) {
+      const sourceKeyValue = source[key];
+      const targetKeyValue = target[key];
+
+      if (isObject(sourceKeyValue) && isObject(targetKeyValue)) {
+        target[key] = merge(targetKeyValue, sourceKeyValue);
+      } else {
+        target[key] = sourceKeyValue;
+      }
+    }
+  }
+  return target;
+};
+
+/**
+ * get object by reference. works only with local references that points on
+ * definitions object
+ *
+ * @param {String} path
+ * @param {Object} definitions
+ * @return {Object}
+ */
+const getLocalRef = function (
+  inputPath: string,
+  definitions: Record<string, unknown>
+) {
+  const path = inputPath.replace(/^#\/definitions\//, "").split("/");
+
+  const find = function (path: string[], root: any): any {
+    const key = path.shift();
+    if (!key) return {};
+
+    if (!root[key]) {
+      return {};
+    } else if (!path.length) {
+      return root[key];
+    } else {
+      return find(path, root[key]);
+    }
+  };
+
+  const result = find(path, definitions);
+
+  if (!isObject(result)) {
+    return result;
+  }
+  return cloneJSON(result);
+};
+
+/**
+ * merge list of objects from allOf properties
+ * if some of objects contains $ref field extracts this reference and merge it
+ *
+ * @param {Array} allOfList
+ * @param {Object} definitions
+ * @return {Object}
+ */
+const mergeAllOf = function (allOfList: any[], definitions: any) {
+  const length = allOfList.length;
+  let index = -1,
+    result = {};
+
+  while (++index < length) {
+    let item = allOfList[index];
+
+    item =
+      typeof item.$ref !== "undefined"
+        ? getLocalRef(item.$ref, definitions)
+        : item;
+
+    result = merge(result, item);
+  }
+
+  return result;
+};
+
+/**
+ * returns a object that built with default values from json schema
+ *
+ * @param {Object} schema
+ * @param {Object} definitions
+ * @return {Object}
+ */
+const defaults = (schema: any, definitions: Record<string, any>): unknown => {
+  if (typeof schema["default"] !== "undefined") {
+    return schema["default"];
+  } else if (typeof schema.allOf !== "undefined") {
+    const mergedItem = mergeAllOf(schema.allOf, definitions);
+    return defaults(mergedItem, definitions);
+  } else if (typeof schema.$ref !== "undefined") {
+    const reference = getLocalRef(schema.$ref, definitions);
+    return defaults(reference, definitions);
+  } else if (schema.type === "object") {
+    if (!schema.properties) {
+      return {};
+    }
+
+    for (const key in schema.properties) {
+      if (schema.properties.hasOwnProperty(key)) {
+        schema.properties[key] = defaults(schema.properties[key], definitions);
+
+        if (typeof schema.properties[key] === "undefined") {
+          delete schema.properties[key];
+        }
+      }
+    }
+
+    return schema.properties;
+  } else if (schema.type === "array") {
+    if (!schema.items) {
+      return [];
+    }
+
+    // minimum item count
+    const ct = schema.minItems || 0;
+
+    // tuple-typed arrays
+    if (schema.items.constructor === Array) {
+      const values = schema.items.map((item: unknown) =>
+        defaults(item, definitions)
+      );
+
+      // remove undefined items at the end (unless required by minItems)
+      for (let i = values.length - 1; i >= 0; i--) {
+        if (typeof values[i] !== "undefined") {
+          break;
+        }
+        if (i + 1 > ct) {
+          values.pop();
+        }
+      }
+
+      // if all values are undefined -> return undefined even
+      // if minItems is set
+      if (values.every((item: unknown) => typeof item === "undefined")) {
+        return undefined;
+      }
+
+      return values;
+    }
+
+    // object-typed arrays
+    const value = defaults(schema.items, definitions);
+
+    if (typeof value === "undefined") {
+      return [];
+    } else {
+      const values = [];
+      for (let i = 0; i < Math.max(0, ct); i++) {
+        values.push(cloneJSON(value));
+      }
+      return values;
+    }
+  }
+};
+
+/**
+ * main function
+ *
+ * @param {Object} schema
+ * @param {Object|undefined} definitions
+ * @return {Object}
+ */
+export default function (
+  schema: any,
+  definitions?: Record<string, unknown> | undefined
+) {
+  if (typeof definitions === "undefined") {
+    definitions = (schema.definitions as Record<string, unknown>) || {};
+  } else if (isObject(schema.definitions)) {
+    definitions = merge(definitions, schema.definitions);
+  }
+
+  return defaults(cloneJSON(schema), definitions);
+}

langserve/chat_playground/src/utils/json-refs.d.ts

@@ -0,0 +1,4 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+export const JsonRefs: {
+  resolveRefs(schema: any): Promise<{ resolved: any }>;
+};

langserve/chat_playground/src/utils/messages.ts

@@ -0,0 +1,7 @@
+export function getMessageContent(x: unknown) {
+  if (typeof x === "string") return x;
+  if (typeof x === "object" && x != null) {
+    if ("content" in x && typeof x.content === "string") return x.content;
+  }
+  return null;
+}