Merge branch 'main' into add-playground-test

.clabot

@@ -1,4 +1,4 @@
 {
-  "contributors": ["eyurtsev", "hwchase17", "nfcampos", "efriis", "jacoblee93", "dqbd", "kreneskyp", "adarsh-jha-dev", "harris", "baskaryan", "hinthornw", "bracesproul", "jakerachleff", "craigsdennis", "anhi", "169", "LarchLiu", "PaulLockett", "RCMatthias", "jwynia", "majiayu000", "mpskex", "shivachittamuru", "sinashaloudegi", "sowsan", "akira", "lucianotonet", "JGalego", "nat-n", "dirien", "donbr", "rahilvora", "WarrenTheRabbit", "StreetLamb", "ccurme"],
+  "contributors": ["eyurtsev", "hwchase17", "nfcampos", "efriis", "jacoblee93", "dqbd", "kreneskyp", "adarsh-jha-dev", "harris", "baskaryan", "hinthornw", "bracesproul", "jakerachleff", "craigsdennis", "anhi", "169", "LarchLiu", "PaulLockett", "RCMatthias", "jwynia", "majiayu000", "mpskex", "shivachittamuru", "sinashaloudegi", "sowsan", "akira", "lucianotonet", "JGalego", "nat-n", "dirien", "donbr", "rahilvora", "WarrenTheRabbit", "StreetLamb"],
   "message": "Thank you for your pull request and welcome to our community. We require contributors to sign our Contributor License Agreement, and we don't seem to have the username {{usersWithoutCLA}} on file. In order for us to review and merge your code, please complete the Individual Contributor License Agreement here https://forms.gle/AQFbtkWRoHXUgipM6 .\n\nThis process is done manually on our side, so after signing the form one of the maintainers will add you to the contributors list.\n\nFor more details about why we have a CLA and other contribution guidelines please see: https://github.com/langchain-ai/langserve/blob/main/CONTRIBUTING.md."
 }

README.md

@@ -6,13 +6,13 @@
 [![](https://dcbadge.vercel.app/api/server/6adMQxSpJS?compact=true&style=flat)](https://discord.com/channels/1038097195422978059/1170024642245832774)
 
 🚩 We will be releasing a hosted version of LangServe for one-click deployments of
-LangChain applications. [Sign up here](https://forms.gle/KC13Nzn76UeLaghK7)
+LangChain applications. [Sign up here](https://airtable.com/apppQ9p5XuujRl3wJ/shrABpHWdxry8Bacm)
 to get on the waitlist.
 
 ## Overview
 
 [LangServe](https://github.com/langchain-ai/langserve) helps developers
-deploy `LangChain` [runnables and chains](https://python.langchain.com/v0.2/docs/how_to/#langchain-expression-language-lcel)
+deploy `LangChain` [runnables and chains](https://python.langchain.com/docs/expression_language/)
 as a REST API.
 
 This library is integrated with [FastAPI](https://fastapi.tiangolo.com/) and
@@ -28,11 +28,11 @@ in [LangChain.js](https://js.langchain.com/docs/ecosystem/langserve).
 - Input and Output schemas automatically inferred from your LangChain object, and
   enforced on every API call, with rich error messages
 - API docs page with JSONSchema and Swagger (insert example link)
-- Efficient `/invoke`, `/batch` and `/stream` endpoints with support for many
+- Efficient `/invoke/`, `/batch/` and `/stream/` endpoints with support for many
   concurrent requests on a single server
-- `/stream_log` endpoint for streaming all (or some) intermediate steps from your
+- `/stream_log/` endpoint for streaming all (or some) intermediate steps from your
   chain/agent
-- **new** as of 0.0.40, supports `/stream_events` to make it easier to stream without needing to parse the output of `/stream_log`.
+- **new** as of 0.0.40, supports `astream_events` to make it easier to stream without needing to parse the output of `stream_log`.
 - Playground page at `/playground/` with streaming output and intermediate steps
 - Built-in (optional) tracing to [LangSmith](https://www.langchain.com/langsmith), just
   add your API key (see [Instructions](https://docs.smith.langchain.com/))
@@ -53,12 +53,12 @@ in [LangChain.js](https://js.langchain.com/docs/ecosystem/langserve).
 
 We will be releasing a hosted version of LangServe for one-click deployments of
 LangChain
-applications. [Sign up here](https://forms.gle/KC13Nzn76UeLaghK7)
+applications. [Sign up here](https://airtable.com/apppQ9p5XuujRl3wJ/shrABpHWdxry8Bacm)
 to get on the waitlist.
 
 ## Security
 
-- Vulnerability in Versions 0.0.13 - 0.0.15 -- playground endpoint allows accessing
+* Vulnerability in Versions 0.0.13 - 0.0.15 -- playground endpoint allows accessing
   arbitrary files on
   server. [Resolved in 0.0.16](https://github.com/langchain-ai/langserve/pull/98).
 
@@ -79,41 +79,35 @@ Use the `LangChain` CLI to bootstrap a `LangServe` project quickly.
 
 To use the langchain CLI make sure that you have a recent version of `langchain-cli`
 installed. You can install it with `pip install -U langchain-cli`.
-
 ## Setup
-
 **Note**: We use `poetry` for dependency management. Please follow poetry [doc](https://python-poetry.org/docs/) to learn more about it.
 
 ### 1. Create new app using langchain cli command
-
 ```sh
 langchain app new my-app
 ```
 
 ### 2. Define the runnable in add_routes. Go to server.py and edit
-
 ```sh
 add_routes(app. NotImplemented)
 ```
 
 ### 3. Use `poetry` to add 3rd party packages (e.g., langchain-openai, langchain-anthropic, langchain-mistral etc).
-
 ```sh
 poetry add [package-name] // e.g `poetry add langchain-openai`
 ```
 
 ### 4. Set up relevant env variables. For example,
-
 ```sh
 export OPENAI_API_KEY="sk-..."
 ```
 
 ### 5. Serve your app
-
 ```sh
 poetry run langchain serve --port=8100
 ```
 
+
 ## Examples
 
 Get your LangServe instance started quickly with
@@ -125,24 +119,24 @@ or the [examples](https://github.com/langchain-ai/langserve/tree/main/examples)
 directory.
 
 | Description                                                                                                                                                                                                                                                        | Links                                                                                                                                                                                                                               |
-| :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **LLMs** Minimal example that reserves OpenAI and Anthropic chat models. Uses async, supports batching and streaming.                                                                                                                                              | [server](https://github.com/langchain-ai/langserve/tree/main/examples/llm/server.py), [client](https://github.com/langchain-ai/langserve/blob/main/examples/llm/client.ipynb)                                                       |
-| **Retriever** Simple server that exposes a retriever as a runnable.                                                                                                                                                                                                | [server](https://github.com/langchain-ai/langserve/tree/main/examples/retrieval/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/retrieval/client.ipynb)                                           |
-| **Conversational Retriever** A Conversational Retriever exposed via LangServe                                                                           | [server](https://github.com/langchain-ai/langserve/tree/main/examples/conversational_retrieval_chain/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/conversational_retrieval_chain/client.ipynb) |
-| **Agent** without **conversation history** based on [OpenAI tools](https://python.langchain.com/docs/modules/agents/agent_types/openai_functions_agent)                                                                                                            | [server](https://github.com/langchain-ai/langserve/tree/main/examples/agent/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/agent/client.ipynb)                                                   |
-| **Agent** with **conversation history** based on [OpenAI tools](https://python.langchain.com/docs/modules/agents/agent_types/openai_functions_agent)                                                                                                               | [server](https://github.com/langchain-ai/langserve/blob/main/examples/agent_with_history/server.py), [client](https://github.com/langchain-ai/langserve/blob/main/examples/agent_with_history/client.ipynb)                         |
-| [RunnableWithMessageHistory](https://python.langchain.com/v0.2/docs/how_to/message_history/) to implement chat persisted on backend, keyed off a `session_id` supplied by client.                                                                    | [server](https://github.com/langchain-ai/langserve/tree/main/examples/chat_with_persistence/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/chat_with_persistence/client.ipynb)                   |
-| [RunnableWithMessageHistory](https://python.langchain.com/v0.2/docs/how_to/message_history/) to implement chat persisted on backend, keyed off a `conversation_id` supplied by client, and `user_id` (see Auth for implementing `user_id` properly). | [server](https://github.com/langchain-ai/langserve/tree/main/examples/chat_with_persistence_and_user/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/chat_with_persistence_and_user/client.ipynb) |
-| [Configurable Runnable](https://python.langchain.com/v0.2/docs/how_to/configure/) to create a retriever that supports run time configuration of the index name.                                                                                      | [server](https://github.com/langchain-ai/langserve/tree/main/examples/configurable_retrieval/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/configurable_retrieval/client.ipynb)                 |
-| [Configurable Runnable](https://python.langchain.com/v0.2/docs/how_to/configure/) that shows configurable fields and configurable alternatives.                                                                                                      | [server](https://github.com/langchain-ai/langserve/tree/main/examples/configurable_chain/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/configurable_chain/client.ipynb)                         |
+|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| **LLMs** Minimal example that reserves OpenAI and Anthropic chat models. Uses async, supports batching and streaming.                                                                                                                                              | [server](https://github.com/langchain-ai/langserve/tree/main/examples/llm/server.py), [client](https://github.com/langchain-ai/langserve/blob/main/examples/llm/client.ipynb)                                                       | 
+| **Retriever** Simple server that exposes a retriever as a runnable.                                                                                                                                                                                                | [server](https://github.com/langchain-ai/langserve/tree/main/examples/retrieval/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/retrieval/client.ipynb)                                           | 
+| **Conversational Retriever** A [Conversational Retriever](https://python.langchain.com/docs/expression_language/cookbook/retrieval#conversational-retrieval-chain) exposed via LangServe                                                                           | [server](https://github.com/langchain-ai/langserve/tree/main/examples/conversational_retrieval_chain/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/conversational_retrieval_chain/client.ipynb) | 
+| **Agent** without **conversation history** based on [OpenAI tools](https://python.langchain.com/docs/modules/agents/agent_types/openai_functions_agent)                                                                                             | [server](https://github.com/langchain-ai/langserve/tree/main/examples/agent/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/agent/client.ipynb)                                                   | 
+| **Agent** with **conversation history** based on [OpenAI tools](https://python.langchain.com/docs/modules/agents/agent_types/openai_functions_agent)                                                                                                               | [server](https://github.com/langchain-ai/langserve/blob/main/examples/agent_with_history/server.py), [client](https://github.com/langchain-ai/langserve/blob/main/examples/agent_with_history/client.ipynb)                         | 
+| [RunnableWithMessageHistory](https://python.langchain.com/docs/expression_language/how_to/message_history) to implement chat persisted on backend, keyed off a `session_id` supplied by client.                                                                    | [server](https://github.com/langchain-ai/langserve/tree/main/examples/chat_with_persistence/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/chat_with_persistence/client.ipynb)                   |
+| [RunnableWithMessageHistory](https://python.langchain.com/docs/expression_language/how_to/message_history) to implement chat persisted on backend, keyed off a `conversation_id` supplied by client, and `user_id` (see Auth for implementing `user_id` properly). | [server](https://github.com/langchain-ai/langserve/tree/main/examples/chat_with_persistence_and_user/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/chat_with_persistence_and_user/client.ipynb) | 
+| [Configurable Runnable](https://python.langchain.com/docs/expression_language/how_to/configure) to create a retriever that supports run time configuration of the index name.                                                                                      | [server](https://github.com/langchain-ai/langserve/tree/main/examples/configurable_retrieval/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/configurable_retrieval/client.ipynb)                 | 
+| [Configurable Runnable](https://python.langchain.com/docs/expression_language/how_to/configure) that shows configurable fields and configurable alternatives.                                                                                                      | [server](https://github.com/langchain-ai/langserve/tree/main/examples/configurable_chain/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/configurable_chain/client.ipynb)                         | 
 | **APIHandler** Shows how to use `APIHandler` instead of `add_routes`. This provides more flexibility for developers to define endpoints. Works well with all FastAPI patterns, but takes a bit more effort.                                                        | [server](https://github.com/langchain-ai/langserve/tree/main/examples/api_handler_examples/server.py)                                                                                                                               |
-| **LCEL Example** Example that uses LCEL to manipulate a dictionary input.                                                                                                                                                                                          | [server](https://github.com/langchain-ai/langserve/tree/main/examples/passthrough_dict/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/passthrough_dict/client.ipynb)                             |
-| **Auth** with `add_routes`: Simple authentication that can be applied across all endpoints associated with app. (Not useful on its own for implementing per user logic.)                                                                                           | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/global_deps/server.py)                                                                                                                                   |
-| **Auth** with `add_routes`: Simple authentication mechanism based on path dependencies. (No useful on its own for implementing per user logic.)                                                                                                                    | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/path_dependencies/server.py)                                                                                                                             |
-| **Auth** with `add_routes`: Implement per user logic and auth for endpoints that use per request config modifier. (**Note**: At the moment, does not integrate with OpenAPI docs.)                                                                                 | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/per_req_config_modifier/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/auth/per_req_config_modifier/client.ipynb)     |
-| **Auth** with `APIHandler`: Implement per user logic and auth that shows how to search only within user owned documents.                                                                                                                                           | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/api_handler/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/auth/api_handler/client.ipynb)                             |
-| **Widgets** Different widgets that can be used with playground (file upload and chat)                                                                                                                                                                              | [server](https://github.com/langchain-ai/langserve/tree/main/examples/widgets/chat/tuples/server.py)                                                                                                                                |
-| **Widgets** File upload widget used for LangServe playground.                                                                                                                                                                                                      | [server](https://github.com/langchain-ai/langserve/tree/main/examples/file_processing/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/file_processing/client.ipynb)                               |
+| **LCEL Example** Example that uses LCEL to manipulate a dictionary input.                                                                                                                                                                                          | [server](https://github.com/langchain-ai/langserve/tree/main/examples/passthrough_dict/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/passthrough_dict/client.ipynb)                          | 
+| **Auth** with `add_routes`: Simple authentication that can be applied across all endpoints associated with app. (Not useful on its own for implementing per user logic.)                                                                                           | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/global_deps/server.py)                                                                                                                                   | 
+| **Auth** with `add_routes`: Simple authentication mechanism based on path dependencies. (No useful on its own for implementing per user logic.)                                                                                                                    | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/path_dependencies/server.py)                                                                                                                             | 
+| **Auth** with `add_routes`: Implement per user logic and auth for endpoints that use per request config modifier. (**Note**: At the moment, does not integrate with OpenAPI docs.)                                                                                 | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/per_req_config_modifier/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/auth/per_req_config_modifier/client.ipynb)     | 
+| **Auth** with `APIHandler`: Implement per user logic and auth that shows how to search only within user owned documents.                                                                                                                                           | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/api_handler/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/auth/api_handler/client.ipynb)                             | 
+| **Widgets** Different widgets that can be used with playground (file upload and chat)                                                                                                                                                                              | [server](https://github.com/langchain-ai/langserve/tree/main/examples/widgets/chat/tuples/server.py)                                                                                                                                            | 
+| **Widgets** File upload widget used for LangServe playground.                                                                                                                                                                                                      | [server](https://github.com/langchain-ai/langserve/tree/main/examples/file_processing/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/file_processing/client.ipynb)                               | 
 
 ## Sample Application
 
@@ -167,17 +161,17 @@ app = FastAPI(
 
 add_routes(
     app,
-    ChatOpenAI(model="gpt-3.5-turbo-0125"),
+    ChatOpenAI(),
     path="/openai",
 )
 
 add_routes(
     app,
-    ChatAnthropic(model="claude-3-haiku-20240307"),
+    ChatAnthropic(),
     path="/anthropic",
 )
 
-model = ChatAnthropic(model="claude-3-haiku-20240307")
+model = ChatAnthropic()
 prompt = ChatPromptTemplate.from_template("tell me a joke about {topic}")
 add_routes(
     app,
@@ -212,8 +206,8 @@ app.add_middleware(
 
 If you've deployed the server above, you can view the generated OpenAPI docs using:
 
-> ⚠️ If using pydantic v2, docs will not be generated for _invoke_, _batch_, _stream_,
-> _stream_log_. See [Pydantic](#pydantic) section below for more details.
+> ⚠️ If using pydantic v2, docs will not be generated for *invoke*, *batch*, *stream*,
+*stream_log*. See [Pydantic](#pydantic) section below for more details.
 
 ```sh
 curl localhost:8000/docs
@@ -273,10 +267,10 @@ In TypeScript (requires LangChain.js version 0.0.166 or later):
 import { RemoteRunnable } from "@langchain/core/runnables/remote";
 
 const chain = new RemoteRunnable({
-  url: `http://localhost:8000/joke/`,
+    url: `http://localhost:8000/joke/`,
 });
 const result = await chain.invoke({
-  topic: "cats",
+    topic: "cats",
 });
 ```
 
@@ -325,20 +319,20 @@ adds of these endpoints to the server:
 - `POST /my_runnable/stream_log` - invoke on a single input and stream the output,
   including output of intermediate steps as it's generated
 - `POST /my_runnable/astream_events` - invoke on a single input and stream events as they are generated,
-  including from intermediate steps.
+ including from intermediate steps.
 - `GET /my_runnable/input_schema` - json schema for input to the runnable
 - `GET /my_runnable/output_schema` - json schema for output of the runnable
 - `GET /my_runnable/config_schema` - json schema for config of the runnable
 
 These endpoints match
-the [LangChain Expression Language interface](https://python.langchain.com/v0.2/docs/how_to/lcel_cheatsheet/) --
+the [LangChain Expression Language interface](https://python.langchain.com/docs/expression_language/interface) --
 please reference this documentation for more details.
 
 ## Playground
 
 You can find a playground page for your runnable at `/my_runnable/playground/`. This
 exposes a simple UI
-to [configure](https://python.langchain.com/v0.2/docs/how_to/configure/)
+to [configure](https://python.langchain.com/docs/expression_language/how_to/configure)
 and invoke your runnable with streaming output and intermediate steps.
 
 <p align="center">
@@ -427,7 +421,7 @@ for demos or testing.
 ## Legacy Chains
 
 LangServe works with both Runnables (constructed
-via [LangChain Expression Language](https://python.langchain.com/v0.2/docs/how_to/#langchain-expression-language-lcel))
+via [LangChain Expression Language](https://python.langchain.com/docs/expression_language/))
 and legacy chains (inheriting from `Chain`).
 However, some of the input schemas for legacy chains may be incomplete/incorrect,
 leading to errors.
@@ -466,6 +460,27 @@ You can deploy to GCP Cloud Run using the following command:
 gcloud run deploy [your-service-name] --source . --port 8001 --allow-unauthenticated --region us-central1 --set-env-vars=OPENAI_API_KEY=your_key
 ```
 
+### Deploy using Infrastructure as Code
+
+#### Pulumi
+
+You can deploy your LangServe server with [Pulumi](https://www.pulumi.com/) using your preferred general purpose language. Below are some quickstart 
+examples for deploying LangServe to different cloud providers.
+
+These examples are a good starting point for your own infrastructure as code (IaC) projects. You can easily modify them to suit your needs.
+
+
+| Cloud | Language   | Repository                                          | Quickstart                                                                                                                                  |
+|-------|------------|-----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------|
+| AWS   | dotnet     | https://github.com/pulumi/examples/aws-cs-langserve | [![Deploy](https://get.pulumi.com/new/button.svg)](https://app.pulumi.com/new?template=https://github.com/pulumi/examples/aws-cs-langserve) |
+| AWS   | golang     | https://github.com/pulumi/examples/aws-go-langserve | [![Deploy](https://get.pulumi.com/new/button.svg)](https://app.pulumi.com/new?template=https://github.com/pulumi/examples/aws-go-langserve) |
+| AWS   | python     | https://github.com/pulumi/examples/aws-py-langserve | [![Deploy](https://get.pulumi.com/new/button.svg)](https://app.pulumi.com/new?template=https://github.com/pulumi/examples/aws-py-langserve) |
+| AWS   | typescript | https://github.com/pulumi/examples/aws-ts-langserve | [![Deploy](https://get.pulumi.com/new/button.svg)](https://app.pulumi.com/new?template=https://github.com/pulumi/examples/aws-ts-langserve) |
+| AWS   | javascript | https://github.com/pulumi/examples/aws-js-langserve | [![Deploy](https://get.pulumi.com/new/button.svg)](https://app.pulumi.com/new?template=https://github.com/pulumi/examples/aws-js-langserve) |
+
+
+
+
 ### Community Contributed
 
 #### Deploy to Railway
@@ -496,7 +511,7 @@ and [security](https://fastapi.tiangolo.com/tutorial/security/).
 
 The below examples show how to wire up authentication logic LangServe endpoints using FastAPI primitives.
 
-You are responsible for providing the actual authentication logic, the users table etc.
+You are responsible for providing the actual authentication logic, the users table etc. 
 
 If you're not sure what you're doing, you could try using an existing solution [Auth0](https://auth0.com/).
 
@@ -505,11 +520,11 @@ If you're not sure what you're doing, you could try using an existing solution [
 If you're using `add_routes`, see
 examples [here](https://github.com/langchain-ai/langserve/tree/main/examples/auth).
 
-| Description                                                                                                                                                                        | Links                                                                                                                                                                                                                           |
-| :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **Auth** with `add_routes`: Simple authentication that can be applied across all endpoints associated with app. (Not useful on its own for implementing per user logic.)           | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/global_deps/server.py)                                                                                                                               |
-| **Auth** with `add_routes`: Simple authentication mechanism based on path dependencies. (No useful on its own for implementing per user logic.)                                    | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/path_dependencies/server.py)                                                                                                                         |
-| **Auth** with `add_routes`: Implement per user logic and auth for endpoints that use per request config modifier. (**Note**: At the moment, does not integrate with OpenAPI docs.) | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/per_req_config_modifier/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/auth/per_req_config_modifier/client.ipynb) |
+| Description                                                                                                                                                                                                                                                        | Links                                                                                                                                                                                                                                |
+|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| **Auth** with `add_routes`: Simple authentication that can be applied across all endpoints associated with app. (Not useful on its own for implementing per user logic.)                                                                                           | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/global_deps/server.py)                                                                                                                                    | 
+| **Auth** with `add_routes`: Simple authentication mechanism based on path dependencies. (No useful on its own for implementing per user logic.)                                                                                                                    | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/path_dependencies/server.py)                                                                                                                              | 
+| **Auth** with `add_routes`: Implement per user logic and auth for endpoints that use per request config modifier. (**Note**: At the moment, does not integrate with OpenAPI docs.)                                                                                 | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/per_req_config_modifier/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/auth/per_req_config_modifier/client.ipynb)     | 
 
 Alternatively, you can use FastAPI's [middleware](https://fastapi.tiangolo.com/tutorial/middleware/).
 
@@ -529,10 +544,10 @@ authorization purposes.
 
 If you feel comfortable with FastAPI and python, you can use LangServe's [APIHandler](https://github.com/langchain-ai/langserve/blob/main/examples/api_handler_examples/server.py).
 
-| Description                                                                                                                                                                                                 | Links                                                                                                                                                                                                           |
-| :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **Auth** with `APIHandler`: Implement per user logic and auth that shows how to search only within user owned documents.                                                                                    | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/api_handler/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/auth/api_handler/client.ipynb)         |
-| **APIHandler** Shows how to use `APIHandler` instead of `add_routes`. This provides more flexibility for developers to define endpoints. Works well with all FastAPI patterns, but takes a bit more effort. | [server](https://github.com/langchain-ai/langserve/tree/main/examples/api_handler_examples/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/api_handler_examples/client.ipynb) |
+| Description                                                                                                                                                                                                                                                        | Links                                                                                                                                                                                                                                |
+|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| **Auth** with `APIHandler`: Implement per user logic and auth that shows how to search only within user owned documents.                                                                                                                                           | [server](https://github.com/langchain-ai/langserve/tree/main/examples/auth/api_handler/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/auth/api_handler/client.ipynb)                             | 
+| **APIHandler** Shows how to use `APIHandler` instead of `add_routes`. This provides more flexibility for developers to define endpoints. Works well with all FastAPI patterns, but takes a bit more effort.                                                        | [server](https://github.com/langchain-ai/langserve/tree/main/examples/api_handler_examples/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/api_handler_examples/client.ipynb)                     |
 
 It's a bit more work, but gives you complete control over the endpoint definitions, so
 you can do whatever custom logic you need for auth.
@@ -601,8 +616,8 @@ add_routes(app, runnable)
 Inherit from `CustomUserType` if you want the data to de-serialize into a
 pydantic model rather than the equivalent dict representation.
 
-At the moment, this type only works _server_ side and is used
-to specify desired _decoding_ behavior. If inheriting from this type
+At the moment, this type only works *server* side and is used
+to specify desired *decoding* behavior. If inheriting from this type
 the server will keep the decoded type as a pydantic model instead
 of converting it into a dict.
 
@@ -641,10 +656,10 @@ The playground allows you to define custom widgets for your runnable from the ba
 
 Here are a few examples:
 
-| Description                                                                           | Links                                                                                                                                                                                                 |
-| :------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **Widgets** Different widgets that can be used with playground (file upload and chat) | [server](https://github.com/langchain-ai/langserve/tree/main/examples/widgets/chat/tuples/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/widgets/client.ipynb)     |
-| **Widgets** File upload widget used for LangServe playground.                         | [server](https://github.com/langchain-ai/langserve/tree/main/examples/file_processing/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/file_processing/client.ipynb) |
+| Description                                                                                                                                                                                                                                                        | Links                                                                                                                                                                                                                                |
+|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| **Widgets** Different widgets that can be used with playground (file upload and chat)                                                                                                                                                                              | [server](https://github.com/langchain-ai/langserve/tree/main/examples/widgets/chat/tuples/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/widgets/client.ipynb)                                               | 
+| **Widgets** File upload widget used for LangServe playground.                                                                                                                                                                                                      | [server](https://github.com/langchain-ai/langserve/tree/main/examples/file_processing/server.py), [client](https://github.com/langchain-ai/langserve/tree/main/examples/file_processing/client.ipynb)                               | 
 
 #### Schema
 
@@ -660,8 +675,8 @@ type NameSpacedPath = { title: string; path: JsonPath }; // Using title to mimic
 type OneOfPath = { oneOf: JsonPath[] };
 
 type Widget = {
-  type: string; // Some well known type (e.g., base64file, chat etc.)
-  [key: string]: JsonPath | NameSpacedPath | OneOfPath;
+    type: string // Some well known type (e.g., base64file, chat etc.)
+    [key: string]: JsonPath | NameSpacedPath | OneOfPath;
 };
 ```
 
@@ -719,9 +734,9 @@ at the [widget example](https://github.com/langchain-ai/langserve/tree/main/exam
 
 To define a chat widget, make sure that you pass "type": "chat".
 
-- "input" is JSONPath to the field in the _Request_ that has the new input message.
-- "output" is JSONPath to the field in the _Response_ that has new output message(s).
-- Don't specify these fields if the entire input or output should be used as they are (
+* "input" is JSONPath to the field in the *Request* that has the new input message.
+* "output" is JSONPath to the field in the *Response* that has new output message(s).
+* Don't specify these fields if the entire input or output should be used as they are (
   e.g., if the output is a list of chat messages.)
 
 Here's a snippet:
@@ -761,7 +776,6 @@ add_routes(
 ```
 
 Example widget:
-
 <p align="center">
 <img src="https://github.com/langchain-ai/langserve/assets/3205522/a71ff37b-a6a9-4857-a376-cf27c41d3ca4" width="50%"/>
 </p>

examples/agent/server.py

@@ -20,15 +20,15 @@ Relevant LangChain documentation:
 from typing import Any
 
 from fastapi import FastAPI
-from langchain.agents import AgentExecutor
+from langchain.agents import AgentExecutor, tool
 from langchain.agents.format_scratchpad import format_to_openai_functions
 from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser
+from langchain.chat_models import ChatOpenAI
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
 from langchain.pydantic_v1 import BaseModel
-from langchain_community.vectorstores import FAISS
-from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
-from langchain_core.tools import tool
-from langchain_core.utils.function_calling import format_tool_to_openai_function
-from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+from langchain.tools.render import format_tool_to_openai_function
+from langchain.vectorstores import FAISS
 
 from langserve import add_routes
 

examples/agent_custom_streaming/server.py

@@ -47,15 +47,15 @@ Relevant LangChain documentation:
 from typing import Any, AsyncIterator, List, Literal
 
 from fastapi import FastAPI
-from langchain.agents import AgentExecutor
+from langchain.agents import AgentExecutor, tool
 from langchain.agents.format_scratchpad.openai_tools import (
     format_to_openai_tool_messages,
 )
 from langchain.agents.output_parsers.openai_tools import OpenAIToolsAgentOutputParser
-from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain.prompts import MessagesPlaceholder
+from langchain_community.tools.convert_to_openai import format_tool_to_openai_tool
+from langchain_core.prompts import ChatPromptTemplate
 from langchain_core.runnables import RunnableLambda
-from langchain_core.tools import tool
-from langchain_core.utils.function_calling import format_tool_to_openai_tool
 from langchain_openai import ChatOpenAI
 
 from langserve import add_routes

examples/agent_with_history/server.py

@@ -26,15 +26,15 @@ Relevant LangChain documentation:
 from typing import Any, List, Union
 
 from fastapi import FastAPI
-from langchain.agents import AgentExecutor
+from langchain.agents import AgentExecutor, tool
 from langchain.agents.format_scratchpad.openai_tools import (
     format_to_openai_tool_messages,
 )
 from langchain.agents.output_parsers.openai_tools import OpenAIToolsAgentOutputParser
+from langchain.prompts import MessagesPlaceholder
+from langchain_community.tools.convert_to_openai import format_tool_to_openai_tool
 from langchain_core.messages import AIMessage, FunctionMessage, HumanMessage
-from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
-from langchain_core.tools import tool
-from langchain_core.utils.function_calling import format_tool_to_openai_tool
+from langchain_core.prompts import ChatPromptTemplate
 from langchain_openai import ChatOpenAI
 
 from langserve import add_routes

examples/auth/api_handler/server.py

@@ -35,6 +35,7 @@ from typing import Any, List, Optional, Union
 
 from fastapi import Depends, FastAPI, HTTPException, Request, Response, status
 from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
+from langchain_community.embeddings.openai import OpenAIEmbeddings
 from langchain_community.vectorstores.chroma import Chroma
 from langchain_core.documents import Document
 from langchain_core.runnables import (
@@ -43,7 +44,6 @@ from langchain_core.runnables import (
     RunnableSerializable,
 )
 from langchain_core.vectorstores import VectorStore
-from langchain_openai import OpenAIEmbeddings
 from typing_extensions import Annotated
 
 from langserve import APIHandler

examples/auth/per_req_config_modifier/server.py

@@ -36,6 +36,7 @@ from typing import Any, Dict, List, Optional, Union
 
 from fastapi import Depends, FastAPI, HTTPException, Request, status
 from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
+from langchain_community.embeddings.openai import OpenAIEmbeddings
 from langchain_community.vectorstores.chroma import Chroma
 from langchain_core.documents import Document
 from langchain_core.runnables import (
@@ -44,7 +45,6 @@ from langchain_core.runnables import (
     RunnableSerializable,
 )
 from langchain_core.vectorstores import VectorStore
-from langchain_openai import OpenAIEmbeddings
 from typing_extensions import Annotated
 
 from langserve import add_routes

examples/chat_playground/legacy_input/server.py

@@ -5,7 +5,7 @@ state back and forth between server and client.
 from typing import List, Union
 
 from fastapi import FastAPI
-from langchain_anthropic import ChatAnthropic
+from langchain.chat_models import ChatAnthropic
 from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
 from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
 

examples/chat_with_persistence/server.py

@@ -13,8 +13,8 @@ from pathlib import Path
 from typing import Callable, Union
 
 from fastapi import FastAPI, HTTPException
-from langchain_anthropic import ChatAnthropic
-from langchain_community.chat_message_histories import FileChatMessageHistory
+from langchain.chat_models import ChatAnthropic
+from langchain.memory import FileChatMessageHistory
 from langchain_core.chat_history import BaseChatMessageHistory
 from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
 from langchain_core.runnables.history import RunnableWithMessageHistory

examples/chat_with_persistence_and_user/server.py

@@ -13,13 +13,13 @@ from pathlib import Path
 from typing import Any, Callable, Dict, Union
 
 from fastapi import FastAPI, HTTPException, Request
-from langchain_community.chat_message_histories import FileChatMessageHistory
+from langchain.chat_models import ChatOpenAI
+from langchain.memory import FileChatMessageHistory
+from langchain.schema.runnable.utils import ConfigurableFieldSpec
 from langchain_core import __version__
 from langchain_core.chat_history import BaseChatMessageHistory
 from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
-from langchain_core.runnables import ConfigurableFieldSpec
 from langchain_core.runnables.history import RunnableWithMessageHistory
-from langchain_openai import ChatOpenAI
 from typing_extensions import TypedDict
 
 from langserve import add_routes

examples/configurable_agent_executor/server.py

@@ -17,12 +17,15 @@ on the LLM and use the stream_log endpoint rather than stream endpoint.
 from typing import Any, AsyncIterator, Dict, List, Optional, cast
 
 from fastapi import FastAPI
-from langchain.agents import AgentExecutor
+from langchain.agents import AgentExecutor, tool
 from langchain.agents.format_scratchpad import format_to_openai_functions
 from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser
+from langchain.chat_models import ChatOpenAI
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
 from langchain.pydantic_v1 import BaseModel
-from langchain_community.vectorstores import FAISS
-from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain.tools.render import format_tool_to_openai_function
+from langchain.vectorstores import FAISS
 from langchain_core.runnables import (
     ConfigurableField,
     ConfigurableFieldSpec,
@@ -30,9 +33,6 @@ from langchain_core.runnables import (
     RunnableConfig,
 )
 from langchain_core.runnables.utils import Input, Output
-from langchain_core.tools import tool
-from langchain_core.utils.function_calling import format_tool_to_openai_function
-from langchain_openai import ChatOpenAI, OpenAIEmbeddings
 
 from langserve import add_routes
 

examples/configurable_chain/server.py

@@ -10,10 +10,10 @@ from typing import Any, Dict
 
 from fastapi import FastAPI, HTTPException, Request
 from fastapi.middleware.cors import CORSMiddleware
-from langchain_core.output_parsers import StrOutputParser
-from langchain_core.prompts import PromptTemplate
-from langchain_core.runnables import ConfigurableField
-from langchain_openai import ChatOpenAI
+from langchain.chat_models import ChatOpenAI
+from langchain.prompts import PromptTemplate
+from langchain.schema.output_parser import StrOutputParser
+from langchain.schema.runnable import ConfigurableField
 
 from langserve import add_routes
 

examples/configurable_retrieval/server.py

@@ -3,18 +3,17 @@
 from typing import Any, Iterable, List, Optional, Type
 
 from fastapi import FastAPI
-from langchain.schema.vectorstore import VST
-from langchain_community.vectorstores import FAISS
-from langchain_core.documents import Document
-from langchain_core.embeddings import Embeddings
-from langchain_core.retrievers import BaseRetriever
-from langchain_core.runnables import (
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.schema import Document
+from langchain.schema.embeddings import Embeddings
+from langchain.schema.retriever import BaseRetriever
+from langchain.schema.runnable import (
     ConfigurableFieldSingleOption,
     RunnableConfig,
     RunnableSerializable,
 )
-from langchain_core.vectorstores import VectorStore
-from langchain_openai import OpenAIEmbeddings
+from langchain.schema.vectorstore import VST
+from langchain.vectorstores import FAISS, VectorStore
 
 from langserve import add_routes
 from langserve.pydantic_v1 import BaseModel, Field

examples/conversational_retrieval_chain/server.py

@@ -13,11 +13,14 @@ from operator import itemgetter
 from typing import List, Tuple
 
 from fastapi import FastAPI
-from langchain_community.vectorstores import FAISS
-from langchain_core.output_parsers import StrOutputParser
-from langchain_core.prompts import ChatPromptTemplate, PromptTemplate, format_document
-from langchain_core.runnables import RunnableMap, RunnablePassthrough
-from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+from langchain.chat_models import ChatOpenAI
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.prompts import ChatPromptTemplate
+from langchain.prompts.prompt import PromptTemplate
+from langchain.schema import format_document
+from langchain.schema.output_parser import StrOutputParser
+from langchain.schema.runnable import RunnableMap, RunnablePassthrough
+from langchain.vectorstores import FAISS
 
 from langserve import add_routes
 from langserve.pydantic_v1 import BaseModel, Field

examples/file_processing/server.py

@@ -15,10 +15,10 @@ allowing one to upload a binary file using the langserve playground UI.
 import base64
 
 from fastapi import FastAPI
+from langchain.document_loaders.blob_loaders import Blob
+from langchain.document_loaders.parsers.pdf import PDFMinerParser
 from langchain.pydantic_v1 import Field
-from langchain_community.document_loaders.parsers.pdf import PDFMinerParser
-from langchain_core.document_loaders import Blob
-from langchain_core.runnables import RunnableLambda
+from langchain.schema.runnable import RunnableLambda
 
 from langserve import CustomUserType, add_routes
 

examples/llm/server.py

@@ -2,8 +2,7 @@
 """Example LangChain server exposes multiple runnables (LLMs in this case)."""
 
 from fastapi import FastAPI
-from langchain_anthropic import ChatAnthropic
-from langchain_openai import ChatOpenAI
+from langchain.chat_models import ChatAnthropic, ChatOpenAI
 
 from langserve import add_routes
 
@@ -15,12 +14,12 @@ app = FastAPI(
 
 add_routes(
     app,
-    ChatOpenAI(model="gpt-3.5-turbo-0125"),
+    ChatOpenAI(),
     path="/openai",
 )
 add_routes(
     app,
-    ChatAnthropic(model="claude-3-haiku-20240307"),
+    ChatAnthropic(),
     path="/anthropic",
 )
 

examples/passthrough_dict/server.py

@@ -4,9 +4,9 @@
 from typing import Any, Callable, Dict, List, Optional, TypedDict
 
 from fastapi import FastAPI
-from langchain_core.prompts import ChatPromptTemplate
-from langchain_core.runnables import RunnableParallel, RunnablePassthrough
-from langchain_openai import ChatOpenAI
+from langchain.chat_models import ChatOpenAI
+from langchain.prompts import ChatPromptTemplate
+from langchain.schema.runnable import RunnableMap, RunnablePassthrough
 
 from langserve import add_routes
 
@@ -43,7 +43,7 @@ model = ChatOpenAI()
 
 underlying_chain = prompt | model
 
-wrapped_chain = RunnableParallel(
+wrapped_chain = RunnableMap(
     {
         "output": _create_projection(exclude_keys=["info"]) | underlying_chain,
         "info": _create_projection(include_keys=["info"]),

examples/retrieval/server.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python
 """Example LangChain server exposes a retriever."""
 from fastapi import FastAPI
-from langchain_community.vectorstores import FAISS
-from langchain_openai import OpenAIEmbeddings
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.vectorstores import FAISS
 
 from langserve import add_routes
 

examples/router/server.py

@@ -9,8 +9,7 @@ See more documentation at:
 https://fastapi.tiangolo.com/tutorial/bigger-applications/
 """
 from fastapi import APIRouter, FastAPI
-from langchain_anthropic import ChatAnthropic
-from langchain_openai import ChatOpenAI
+from langchain.chat_models import ChatAnthropic, ChatOpenAI
 
 from langserve import add_routes
 
@@ -21,13 +20,13 @@ router = APIRouter(prefix="/models")
 # Invocations to this router will appear in trace logs as /models/openai
 add_routes(
     router,
-    ChatOpenAI(model="gpt-3.5-turbo-0125"),
+    ChatOpenAI(),
     path="/openai",
 )
 # Invocations to this router will appear in trace logs as /models/anthropic
 add_routes(
     router,
-    ChatAnthropic(model="claude-3-haiku-20240307"),
+    ChatAnthropic(),
     path="/anthropic",
 )
 

examples/widgets/chat/message_list/server.py

@@ -6,7 +6,7 @@ from typing import List, Union
 
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
-from langchain_anthropic import ChatAnthropic
+from langchain.chat_models import ChatAnthropic
 from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
 from langchain_core.output_parsers import StrOutputParser
 from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder

examples/widgets/chat/tuples/server.py

@@ -6,17 +6,18 @@ from typing import Any, Dict, List, Tuple
 
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
+from langchain.chat_models.openai import ChatOpenAI
+from langchain.document_loaders.blob_loaders import Blob
+from langchain.document_loaders.parsers.pdf import PDFMinerParser
 from langchain.pydantic_v1 import BaseModel, Field
-from langchain_community.document_loaders.parsers.pdf import PDFMinerParser
-from langchain_core.document_loaders import Blob
-from langchain_core.messages import (
+from langchain.schema.messages import (
     AIMessage,
     BaseMessage,
     FunctionMessage,
     HumanMessage,
 )
-from langchain_core.runnables import RunnableLambda, RunnableParallel
-from langchain_openai import ChatOpenAI
+from langchain.schema.runnable import RunnableLambda
+from langchain_core.runnables import RunnableParallel
 
 from langserve import CustomUserType
 from langserve.server import add_routes

langserve/api_handler.py

@@ -477,6 +477,11 @@ _MODEL_REGISTRY = {}
 _SEEN_NAMES = set()
 
 
+def _is_scoped_feedback_enabled() -> bool:
+    """Temporary hard-coded as False. Used only to enable during unit tests."""
+    return False
+
+
 class PerKeyFeedbackConfig(TypedDict):
     """Per feedback configuration.
 
@@ -645,8 +650,7 @@ class APIHandler:
         # remember to make relevant updates in the unit tests.
         self._langsmith_client = (
             ls_client.Client()
-            if tracing_is_enabled()
-            and (enable_feedback_endpoint or self._token_feedback_enabled)
+            if tracing_is_enabled() and enable_feedback_endpoint
             else None
         )
 
@@ -852,7 +856,7 @@ class APIHandler:
                     feedback_tokens=[
                         FeedbackToken(
                             key=feedback_key,
-                            token_url=feedback_token.url,
+                            url=feedback_token.url,
                             expires_at=feedback_token.expires_at.isoformat(),
                         )
                     ]
@@ -999,7 +1003,7 @@ class APIHandler:
                     feedback_tokens=[
                         FeedbackToken(
                             key=feedback_key,
-                            token_url=feedback_token.url,
+                            url=feedback_token.url,
                             expires_at=feedback_token.expires_at.isoformat(),
                         )
                     ],
@@ -1081,6 +1085,8 @@ class APIHandler:
                 Originates from the client side. This config must be validated.
             server_config: optional server configuration that will be merged
         """
+        err_event = {}
+        validation_exception: Optional[BaseException] = None
         run_id = None
         try:
             config, input_ = await self._get_config_and_input(
@@ -1090,12 +1096,26 @@ class APIHandler:
                 server_config=server_config,
             )
             run_id = config["run_id"]
-        except BaseException:
-            # Exceptions will be properly translated by default FastAPI middleware
-            # to either 422 (on input validation) or 500 internal server errors.
-            raise
+        except BaseException as e:
+            validation_exception = e
+            if isinstance(e, RequestValidationError):
+                err_event = {
+                    "event": "error",
+                    "data": json.dumps(
+                        {"status_code": 422, "message": repr(e.errors())}
+                    ),
+                }
+            else:
+                err_event = {
+                    "event": "error",
+                    # Do not expose the error message to the client since
+                    # the message may contain sensitive information.
+                    "data": json.dumps(
+                        {"status_code": 500, "message": "Internal Server Error"}
+                    ),
+                }
 
-        if self._token_feedback_enabled:
+        if self._token_feedback_enabled and not validation_exception:
             # Create task to create a presigned feedback token
             feedback_key: Optional[str] = self._token_feedback_config["key_configs"][0][
                 "key"
@@ -1113,6 +1133,15 @@ class APIHandler:
 
         async def _stream() -> AsyncIterator[dict]:
             """Stream the output of the runnable."""
+            if validation_exception:
+                yield err_event
+                if isinstance(validation_exception, RequestValidationError):
+                    return
+                else:
+                    raise AssertionError(
+                        "Internal server error"
+                    ) from validation_exception
+
             try:
                 config_w_callbacks = config.copy()
                 event_aggregator = AsyncEventAggregatorCallback()
@@ -1170,6 +1199,8 @@ class APIHandler:
         View documentation for endpoint at the end of the file.
         It's attached to _stream_log_docs endpoint.
         """
+        err_event = {}
+        validation_exception: Optional[BaseException] = None
         try:
             config, input_ = await self._get_config_and_input(
                 request,
@@ -1177,39 +1208,56 @@ class APIHandler:
                 endpoint="stream_log",
                 server_config=server_config,
             )
-            run_id = config["run_id"]
-        except BaseException:
-            # Exceptions will be properly translated by default FastAPI middleware
-            # to either 422 (on input validation) or 500 internal server errors.
-            raise
+        except BaseException as e:
+            validation_exception = e
+            if isinstance(e, RequestValidationError):
+                err_event = {
+                    "event": "error",
+                    "data": json.dumps(
+                        {"status_code": 422, "message": repr(e.errors())}
+                    ),
+                }
+            else:
+                err_event = {
+                    "event": "error",
+                    # Do not expose the error message to the client since
+                    # the message may contain sensitive information.
+                    "data": json.dumps(
+                        {"status_code": 500, "message": "Internal Server Error"}
+                    ),
+                }
+
         try:
             body = await request.json()
             with _with_validation_error_translation():
                 stream_log_request = StreamLogParameters(**body)
         except json.JSONDecodeError:
-            raise RequestValidationError(errors=["Invalid JSON body"])
-        except RequestValidationError:
-            raise
-
-        feedback_key: Optional[str]
-
-        if self._token_feedback_enabled:
-            # Create task to create a presigned feedback token
-            feedback_key: str = self._token_feedback_config["key_configs"][0]["key"]
-            feedback_coro = run_in_executor(
-                None,
-                self._langsmith_client.create_presigned_feedback_token,
-                run_id,
-                feedback_key,
-            )
-            task: Optional[asyncio.Task] = asyncio.create_task(feedback_coro)
-        else:
-            feedback_key = None
-            task = None
+            # Body as text
+            validation_exception = RequestValidationError(errors=["Invalid JSON body"])
+            err_event = {
+                "event": "error",
+                "data": json.dumps(
+                    {"status_code": 422, "message": "Invalid JSON body"}
+                ),
+            }
+        except RequestValidationError as e:
+            validation_exception = e
+            err_event = {
+                "event": "error",
+                "data": json.dumps({"status_code": 422, "message": repr(e.errors())}),
+            }
 
         async def _stream_log() -> AsyncIterator[dict]:
             """Stream the output of the runnable."""
-            has_sent_metadata = False
+            if validation_exception:
+                yield err_event
+                if isinstance(validation_exception, RequestValidationError):
+                    return
+                else:
+                    raise AssertionError(
+                        "Internal server error"
+                    ) from validation_exception
+
             try:
                 async for chunk in self._runnable.astream_log(
                     input_,
@@ -1243,18 +1291,6 @@ class APIHandler:
                             "data": self._serializer.dumps(data).decode("utf-8"),
                             "event": "data",
                         }
-
-                    # Send a metadata event as soon as possible
-                    if not has_sent_metadata and self._token_feedback_enabled:
-                        if task is None:
-                            raise AssertionError("Feedback token task was not created.")
-                        if not task.done():
-                            continue
-                        feedback_token = task.result()
-                        yield _create_metadata_event(
-                            run_id, feedback_key, feedback_token
-                        )
-                        has_sent_metadata = True
                 yield {"event": "end"}
             except BaseException:
                 yield {
@@ -1278,6 +1314,8 @@ class APIHandler:
         server_config: Optional[RunnableConfig] = None,
     ) -> EventSourceResponse:
         """Stream events from the runnable."""
+        err_event = {}
+        validation_exception: Optional[BaseException] = None
         run_id = None
         try:
             config, input_ = await self._get_config_and_input(
@@ -1287,23 +1325,48 @@ class APIHandler:
                 server_config=server_config,
             )
             run_id = config["run_id"]
-        except BaseException:
-            # Exceptions will be properly translated by default FastAPI middleware
-            # to either 422 (on input validation) or 500 internal server errors.
-            raise
+        except BaseException as e:
+            validation_exception = e
+            if isinstance(e, RequestValidationError):
+                err_event = {
+                    "event": "error",
+                    "data": json.dumps(
+                        {"status_code": 422, "message": repr(e.errors())}
+                    ),
+                }
+            else:
+                err_event = {
+                    "event": "error",
+                    # Do not expose the error message to the client since
+                    # the message may contain sensitive information.
+                    "data": json.dumps(
+                        {"status_code": 500, "message": "Internal Server Error"}
+                    ),
+                }
 
         try:
             body = await request.json()
             with _with_validation_error_translation():
                 stream_events_request = StreamEventsParameters(**body)
         except json.JSONDecodeError:
-            raise RequestValidationError(errors=["Invalid JSON body"])
-        except RequestValidationError:
-            raise
+            # Body as text
+            validation_exception = RequestValidationError(errors=["Invalid JSON body"])
+            err_event = {
+                "event": "error",
+                "data": json.dumps(
+                    {"status_code": 422, "message": "Invalid JSON body"}
+                ),
+            }
+        except RequestValidationError as e:
+            validation_exception = e
+            err_event = {
+                "event": "error",
+                "data": json.dumps({"status_code": 422, "message": repr(e.errors())}),
+            }
 
         feedback_key: Optional[str]
 
-        if self._token_feedback_enabled:
+        if self._token_feedback_enabled and not validation_exception:
             # Create task to create a presigned feedback token
             feedback_key: str = self._token_feedback_config["key_configs"][0]["key"]
             feedback_coro = run_in_executor(
@@ -1324,6 +1387,15 @@ class APIHandler:
                     "Please upgrade langchain-core>=0.1.14 to use astream_events"
                 )
 
+            if validation_exception:
+                yield err_event
+                if isinstance(validation_exception, RequestValidationError):
+                    return
+                else:
+                    raise AssertionError(
+                        "Internal server error"
+                    ) from validation_exception
+
             has_sent_metadata = False
 
             try:
@@ -1354,7 +1426,7 @@ class APIHandler:
                         }
 
                     # Send a metadata event as soon as possible
-                    if not has_sent_metadata and self._token_feedback_enabled:
+                    if not has_sent_metadata and self._enable_feedback_endpoint:
                         if task is None:
                             raise AssertionError("Feedback token task was not created.")
                         if not task.done():

langserve/client.py

@@ -45,7 +45,6 @@ from langserve.serialization import (
     WellKnownLCSerializer,
     load_events,
 )
-from langserve.server_sent_events import aconnect_sse, connect_sse
 
 logger = logging.getLogger(__name__)
 
@@ -524,17 +523,25 @@ class RemoteRunnable(Runnable[Input, Output]):
         }
         endpoint = urljoin(self.url, "stream")
 
+        try:
+            from httpx_sse import connect_sse
+        except ImportError:
+            raise ImportError(
+                "Missing `httpx_sse` dependency to use the stream method. "
+                "Install via `pip install httpx_sse`'"
+            )
+
         try:
             with connect_sse(
                 self.sync_client, "POST", endpoint, json=data
             ) as event_source:
                 for sse in event_source.iter_sse():
-                    if sse["event"] == "data":
-                        chunk = self._lc_serializer.loads(sse["data"])
+                    if sse.event == "data":
+                        chunk = self._lc_serializer.loads(sse.data)
                         if isinstance(chunk, dict):
                             # Any dict returned from streaming end point
                             # is assumed to follow additive semantics
-                            # and will be coverted to an AddableDict
+                            # and will be converted to an AddableDict
                             # automatically
                             chunk = AddableDict(chunk)
                         yield chunk
@@ -556,21 +563,21 @@ class RemoteRunnable(Runnable[Input, Output]):
                                 except TypeError:
                                     final_output = None
                                     final_output_supported = False
-                    elif sse["event"] == "error":
+                    elif sse.event == "error":
                         # This can only be a server side error
                         _raise_exception_from_data(
-                            sse["data"], httpx.Request(method="POST", url=endpoint)
+                            sse.data, httpx.Request(method="POST", url=endpoint)
                         )
-                    elif sse["event"] == "metadata":
+                    elif sse.event == "metadata":
                         # Nothing to do for metadata for the regular remote client.
                         continue
-                    elif sse["event"] == "end":
+                    elif sse.event == "end":
                         break
                     else:
                         _log_error_message_once(
-                            f"Encountered an unsupported event type: `{sse['event']}`. "
+                            f"Encountered an unsupported event type: `{sse.event}`. "
                             f"Try upgrading the remote client to the latest version."
-                            f"Ignoring events of type `{sse['event']}`."
+                            f"Ignoring events of type `{sse.event}`."
                         )
         except BaseException as e:
             run_manager.on_chain_error(e)
@@ -602,13 +609,18 @@ class RemoteRunnable(Runnable[Input, Output]):
         }
         endpoint = urljoin(self.url, "stream")
 
+        try:
+            from httpx_sse import aconnect_sse
+        except ImportError:
+            raise ImportError("You must install `httpx_sse` to use the stream method.")
+
         try:
             async with aconnect_sse(
                 self.async_client, "POST", endpoint, json=data
             ) as event_source:
                 async for sse in event_source.aiter_sse():
-                    if sse["event"] == "data":
-                        chunk = self._lc_serializer.loads(sse["data"])
+                    if sse.event == "data":
+                        chunk = self._lc_serializer.loads(sse.data)
                         if isinstance(chunk, dict):
                             # Any dict returned from streaming end point
                             # is assumed to follow additive semantics
@@ -635,21 +647,21 @@ class RemoteRunnable(Runnable[Input, Output]):
                                     final_output = None
                                     final_output_supported = False
 
-                    elif sse["event"] == "error":
+                    elif sse.event == "error":
                         # This can only be a server side error
                         _raise_exception_from_data(
-                            sse["data"], httpx.Request(method="POST", url=endpoint)
+                            sse.data, httpx.Request(method="POST", url=endpoint)
                         )
-                    elif sse["event"] == "metadata":
+                    elif sse.event == "metadata":
                         # Nothing to do for metadata for the regular remote client.
                         continue
-                    elif sse["event"] == "end":
+                    elif sse.event == "end":
                         break
                     else:
                         _log_error_message_once(
-                            f"Encountered an unsupported event type: `{sse['event']}`. "
+                            f"Encountered an unsupported event type: `{sse.event}`. "
                             f"Try upgrading the remote client to the latest version."
-                            f"Ignoring events of type `{sse['event']}`."
+                            f"Ignoring events of type `{sse.event}`."
                         )
         except BaseException as e:
             await run_manager.on_chain_error(e)
@@ -706,13 +718,18 @@ class RemoteRunnable(Runnable[Input, Output]):
         }
         endpoint = urljoin(self.url, "stream_log")
 
+        try:
+            from httpx_sse import aconnect_sse
+        except ImportError:
+            raise ImportError("You must install `httpx_sse` to use the stream method.")
+
         try:
             async with aconnect_sse(
                 self.async_client, "POST", endpoint, json=data
             ) as event_source:
                 async for sse in event_source.aiter_sse():
-                    if sse["event"] == "data":
-                        data = self._lc_serializer.loads(sse["data"])
+                    if sse.event == "data":
+                        data = self._lc_serializer.loads(sse.data)
                         # Create a copy of the data to yield since underlying
                         # code is using jsonpatch which does some stuff in-place
                         # that can cause unexpected consequences.
@@ -724,18 +741,18 @@ class RemoteRunnable(Runnable[Input, Output]):
                             final_output += chunk
                         else:
                             final_output = chunk
-                    elif sse["event"] == "error":
+                    elif sse.event == "error":
                         # This can only be a server side error
                         _raise_exception_from_data(
-                            sse["data"], httpx.Request(method="POST", url=endpoint)
+                            sse.data, httpx.Request(method="POST", url=endpoint)
                         )
-                    elif sse["event"] == "end":
+                    elif sse.event == "end":
                         break
                     else:
                         _log_error_message_once(
-                            f"Encountered an unsupported event type: `{sse['event']}`. "
+                            f"Encountered an unsupported event type: `{sse.event}`. "
                             f"Try upgrading the remote client to the latest version."
-                            f"Ignoring events of type `{sse['event']}`."
+                            f"Ignoring events of type `{sse.event}`."
                         )
         except BaseException as e:
             await run_manager.on_chain_error(e)
@@ -805,34 +822,36 @@ class RemoteRunnable(Runnable[Input, Output]):
             "exclude_tags": exclude_tags,
         }
         endpoint = urljoin(self.url, "stream_events")
-        headers = kwargs.pop("headers", {})
-        headers["Accept"] = "text/event-stream"
-        headers["Cache-Control"] = "no-store"
+
+        try:
+            from httpx_sse import aconnect_sse
+        except ImportError:
+            raise ImportError("You must install `httpx_sse` to use the stream method.")
 
         try:
             async with aconnect_sse(
                 self.async_client, "POST", endpoint, json=data
             ) as event_source:
                 async for sse in event_source.aiter_sse():
-                    if sse["event"] == "data":
-                        event = self._lc_serializer.loads(sse["data"])
+                    if sse.event == "data":
+                        event = self._lc_serializer.loads(sse.data)
                         # Create a copy of the data to yield since underlying
                         # code is using jsonpatch which does some stuff in-place
                         # that can cause unexpected consequences.
                         yield event
                         events.append(event)
-                    elif sse["event"] == "error":
+                    elif sse.event == "error":
                         # This can only be a server side error
                         _raise_exception_from_data(
-                            sse["data"], httpx.Request(method="POST", url=endpoint)
+                            sse.data, httpx.Request(method="POST", url=endpoint)
                         )
-                    elif sse["event"] == "end":
+                    elif sse.event == "end":
                         break
                     else:
                         _log_error_message_once(
-                            f"Encountered an unsupported event type: `{sse['event']}`. "
+                            f"Encountered an unsupported event type: `{sse.event}`. "
                             f"Try upgrading the remote client to the latest version."
-                            f"Ignoring events of type `{sse['event']}`."
+                            f"Ignoring events of type `{sse.event}`."
                         )
         except BaseException as e:
             await run_manager.on_chain_error(e)

langserve/schema.py

@@ -116,7 +116,7 @@ class BaseFeedback(BaseModel):
 class FeedbackCreateRequestTokenBased(BaseModel):
     """Shared information between create requests of feedback and feedback objects."""
 
-    token_or_url: Union[UUID, str]
+    token_or_url: UUID
     """The associated run ID this feedback is logged for."""
 
     score: Optional[Union[float, int, bool]] = None

langserve/server_sent_events.py

@@ -1,136 +0,0 @@
-"""Adapted from https://github.com/florimondmanca/httpx-sse"""
-from contextlib import asynccontextmanager, contextmanager
-from typing import Any, AsyncIterator, Iterator, List, Optional, TypedDict
-
-import httpx
-
-
-class ServerSentEvent(TypedDict):
-    event: Optional[str]
-    data: Optional[str]
-    id: Optional[str]
-    retry: Optional[int]
-
-
-class SSEDecoder:
-    def __init__(self) -> None:
-        self._event = ""
-        self._data: List[str] = []
-        self._last_event_id = ""
-        self._retry: Optional[int] = None
-
-    def decode(self, line: str) -> Optional[ServerSentEvent]:
-        # See: https://html.spec.whatwg.org/multipage/server-sent-events.html#event-stream-interpretation  # noqa: E501
-        if not line:
-            if (
-                not self._event
-                and not self._data
-                and not self._last_event_id
-                and self._retry is None
-            ):
-                return None
-
-            sse = {
-                "event": self._event,
-                "data": "\n".join(self._data),
-                "id": self._last_event_id,
-                "retry": self._retry,
-            }
-
-            # NOTE: as per the SSE spec, do not reset last_event_id.
-            self._event = ""
-            self._data = []
-            self._retry = None
-
-            return sse
-
-        if line.startswith(":"):
-            return None
-
-        fieldname, _, value = line.partition(":")
-
-        if value.startswith(" "):
-            value = value[1:]
-
-        if fieldname == "event":
-            self._event = value
-        elif fieldname == "data":
-            self._data.append(value)
-        elif fieldname == "id":
-            if "\0" in value:
-                pass
-            else:
-                self._last_event_id = value
-        elif fieldname == "retry":
-            try:
-                self._retry = int(value)
-            except (TypeError, ValueError):
-                pass
-        else:
-            pass  # Field is ignored.
-
-        return None
-
-
-class EventSource:
-    def __init__(self, response: httpx.Response) -> None:
-        self._response = response
-
-    def _check_content_type(self) -> None:
-        """Check that the response content type is 'text/event-stream'."""
-        self._response.raise_for_status()
-        content_type = self._response.headers.get("content-type", "").partition(";")[0]
-        if "text/event-stream" not in content_type:
-            raise AssertionError(
-                "Expected response header Content-Type to contain 'text/event-stream', "
-                f"got {content_type!r}"
-            )
-
-    @property
-    def response(self) -> httpx.Response:
-        return self._response
-
-    def iter_sse(self) -> Iterator[ServerSentEvent]:
-        self._check_content_type()
-        decoder = SSEDecoder()
-        for line in self._response.iter_lines():
-            line = line.rstrip("\n")
-            sse = decoder.decode(line)
-            if sse is not None:
-                yield sse
-
-    async def aiter_sse(self) -> AsyncIterator[ServerSentEvent]:
-        self._check_content_type()
-        decoder = SSEDecoder()
-        async for line in self._response.aiter_lines():
-            line = line.rstrip("\n")
-            sse = decoder.decode(line)
-            if sse is not None:
-                yield sse
-
-
-@contextmanager
-def connect_sse(
-    client: httpx.Client, method: str, url: str, **kwargs: Any
-) -> Iterator[EventSource]:
-    headers = kwargs.pop("headers", {})
-    headers["Accept"] = "text/event-stream"
-    headers["Cache-Control"] = "no-store"
-
-    with client.stream(method, url, headers=headers, **kwargs) as response:
-        yield EventSource(response)
-
-
-@asynccontextmanager
-async def aconnect_sse(
-    client: httpx.AsyncClient,
-    method: str,
-    url: str,
-    **kwargs: Any,
-) -> AsyncIterator[EventSource]:
-    headers = kwargs.pop("headers", {})
-    headers["Accept"] = "text/event-stream"
-    headers["Cache-Control"] = "no-store"
-
-    async with client.stream(method, url, headers=headers, **kwargs) as response:
-        yield EventSource(response)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "langserve"
-version = "0.2.1"
+version = "0.0.51"
 description = ""
 readme = "README.md"
 authors = ["LangChain"]
@@ -14,15 +14,16 @@ python = "^3.8.1"
 httpx = ">=0.23.0" # May be able to decrease this version
 fastapi = {version = ">=0.90.1,<1", optional = true}
 sse-starlette = {version = "^1.3.0", optional = true}
+httpx-sse = {version = ">=0.3.1", optional = true}
 pydantic = ">=1"
-langchain-core = ">=0.1,<0.3"
+langchain-core = "^0.1.0"
 orjson = ">=2"
-pyproject-toml = "^0.0.10"
 
 [tool.poetry.group.dev.dependencies]
 jupyterlab = "^3.6.1"
 fastapi = ">=0.90.1"
 sse-starlette = "^1.3.0"
+httpx-sse = ">=0.3.1"
 
 [tool.poetry.group.typing.dependencies]
 
@@ -44,14 +45,15 @@ openai = "^0.28.0"
 uvicorn = {extras = ["standard"], version = "^0.23.2"}
 fastapi = ">=0.90.1"
 sse-starlette = "^1.3.0"
+httpx-sse = ">=0.3.1"
 
 [tool.poetry.extras]
 # Extras that are used for client
-client = []
+client = ["httpx-sse"]
 # Extras that are used for server
 server = ["sse-starlette", "fastapi"]
 # All
-all = ["sse-starlette", "fastapi"]
+all = ["httpx-sse", "sse-starlette", "fastapi"]
 
 [tool.ruff]
 select = [

tests/unit_tests/test_api_playground.py

@@ -45,6 +45,9 @@ async def test_serve_playground_with_api_router() -> None:
     async with AsyncClient(app=app, base_url="http://localhost:9999") as client:
         response = await client.get("/langserve_runnables/chat/playground/index.html")
         assert response.status_code == 200
+        assert (
+            'src="/langserve_runnables/chat/playground/assets/' in response.content.decode()
+        ), "html should contain reference to valid playground assets path"
 
 
 async def test_serve_playground_with_api_router_in_api_router() -> None:
@@ -67,6 +70,9 @@ async def test_serve_playground_with_api_router_in_api_router() -> None:
     async with AsyncClient(app=app, base_url="http://localhost:9999") as client:
         response = await client.get("/parent/bar/foo/playground/index.html")
         assert response.status_code == 200
+        assert (
+            'src="/parent/bar/foo/playground/assets/' in response.content.decode()
+        ), "html should contain reference to valid playground assets path"
 
 
 async def test_root_path_on_playground() -> None:

tests/unit_tests/test_server_client.py

@@ -71,7 +71,6 @@ from langserve.callbacks import AsyncEventAggregatorCallback
 from langserve.client import RemoteRunnable
 from langserve.lzstring import LZString
 from langserve.schema import CustomUserType
-from tests.unit_tests.utils.stubs import AnyStr
 
 try:
     from pydantic.v1 import BaseModel, Field
@@ -381,18 +380,26 @@ async def test_server_async(app: FastAPI) -> None:
     async with get_async_test_client(app, raise_app_exceptions=True) as async_client:
         # Test bad stream requests
         response = await async_client.post("/stream", data="bad json []")
-        assert response.status_code == 422
+        stream_events = _decode_eventstream(response.text)
+        assert stream_events[0]["type"] == "error"
+        assert stream_events[0]["data"]["status_code"] == 422
 
         response = await async_client.post("/stream", json={})
-        assert response.status_code == 422
+        stream_events = _decode_eventstream(response.text)
+        assert stream_events[0]["type"] == "error"
+        assert stream_events[0]["data"]["status_code"] == 422
 
     # test stream_log bad requests
     async with get_async_test_client(app, raise_app_exceptions=True) as async_client:
         response = await async_client.post("/stream_log", data="bad json []")
-        assert response.status_code == 422
+        stream_events = _decode_eventstream(response.text)
+        assert stream_events[0]["type"] == "error"
+        assert stream_events[0]["data"]["status_code"] == 422
 
         response = await async_client.post("/stream_log", json={})
-        assert response.status_code == 422
+        stream_events = _decode_eventstream(response.text)
+        assert stream_events[0]["type"] == "error"
+        assert stream_events[0]["data"]["status_code"] == 422
 
 
 async def test_server_astream_events(app: FastAPI) -> None:
@@ -448,10 +455,14 @@ async def test_server_astream_events(app: FastAPI) -> None:
     # test stream_events with bad requests
     async with get_async_test_client(app, raise_app_exceptions=True) as async_client:
         response = await async_client.post("/stream_events", data="bad json []")
-        assert response.status_code == 422
+        stream_events = _decode_eventstream(response.text)
+        assert stream_events[0]["type"] == "error"
+        assert stream_events[0]["data"]["status_code"] == 422
 
         response = await async_client.post("/stream_events", json={})
-        assert response.status_code == 422
+        stream_events = _decode_eventstream(response.text)
+        assert stream_events[0]["type"] == "error"
+        assert stream_events[0]["data"]["status_code"] == 422
 
 
 async def test_server_bound_async(app_for_config: FastAPI) -> None:
@@ -802,31 +813,6 @@ async def test_astream_log(async_remote_runnable: RemoteRunnable) -> None:
         }
 
 
-async def test_streaming_with_errors() -> None:
-    from langchain_core.runnables import RunnableGenerator
-
-    async def with_errors(inputs: dict) -> AsyncIterator[int]:
-        yield 1
-        raise ValueError("Error")
-        yield 2
-
-    app = FastAPI()
-    add_routes(app, RunnableGenerator(with_errors), path="/with_errors")
-
-    async with get_async_remote_runnable(
-        app, path="/with_errors", raise_app_exceptions=False
-    ) as runnable:
-        chunks = []
-
-        with pytest.raises(httpx.HTTPStatusError) as e:
-            async for chunk in runnable.astream(1):
-                chunks.append(chunk)
-
-        # Check that first chunk was received
-        assert chunks == [1]
-        assert e.value.response.status_code == 500
-
-
 async def test_astream_log_allowlist(event_loop: AbstractEventLoop) -> None:
     """Test async stream with an allowlist."""
 
@@ -1972,14 +1958,10 @@ async def test_per_request_config_modifier_endpoints(
             else:
                 raise ValueError(f"Unknown endpoint {endpoint}")
 
-            if endpoint in {
-                "invoke",
-                "batch",
-                "stream",
-                "stream_log",
-                "astream_events",
-            }:
+            if endpoint in {"invoke", "batch"}:
                 assert response.status_code == 500
+            elif endpoint in {"stream", "stream_log"}:
+                assert '"status_code": 500' in response.text
             else:
                 assert response.status_code != 500
 
@@ -2559,7 +2541,7 @@ async def test_astream_events_with_prompt_model_parser_chain(
                 "tags": ["seq:step:2"],
             },
             {
-                "data": {"chunk": AIMessageChunk(content="Hello", id=AnyStr())},
+                "data": {"chunk": AIMessageChunk(content="Hello")},
                 "event": "on_chat_model_stream",
                 "name": "GenericFakeChatModel",
                 "tags": ["seq:step:2"],
@@ -2583,7 +2565,7 @@ async def test_astream_events_with_prompt_model_parser_chain(
                 "tags": [],
             },
             {
-                "data": {"chunk": AIMessageChunk(content=" ", id=AnyStr())},
+                "data": {"chunk": AIMessageChunk(content=" ")},
                 "event": "on_chat_model_stream",
                 "name": "GenericFakeChatModel",
                 "tags": ["seq:step:2"],
@@ -2601,7 +2583,7 @@ async def test_astream_events_with_prompt_model_parser_chain(
                 "tags": [],
             },
             {
-                "data": {"chunk": AIMessageChunk(content="World!", id=AnyStr())},
+                "data": {"chunk": AIMessageChunk(content="World!")},
                 "event": "on_chat_model_stream",
                 "name": "GenericFakeChatModel",
                 "tags": ["seq:step:2"],
@@ -2633,9 +2615,7 @@ async def test_astream_events_with_prompt_model_parser_chain(
                             [
                                 ChatGenerationChunk(
                                     text="Hello World!",
-                                    message=AIMessageChunk(
-                                        content="Hello World!", id=AnyStr()
-                                    ),
+                                    message=AIMessageChunk(content="Hello World!"),
                                 )
                             ]
                         ],
@@ -2649,7 +2629,7 @@ async def test_astream_events_with_prompt_model_parser_chain(
             },
             {
                 "data": {
-                    "input": AIMessageChunk(content="Hello World!", id=AnyStr()),
+                    "input": AIMessageChunk(content="Hello World!"),
                     "output": "Hello World!",
                 },
                 "event": "on_parser_end",
@@ -2787,13 +2767,10 @@ async def test_remote_configurable_remote_runnable() -> None:
         result = await chain_with_history.ainvoke(
             {"question": "hi"}, {"configurable": {"session_id": "1"}}
         )
-        assert result == AIMessage(content="Hello World!", id=AnyStr())
+        assert result == AIMessage(content="Hello World!")
         assert store == {
             "1": InMemoryHistory(
-                messages=[
-                    HumanMessage(content="hi"),
-                    AIMessage(content="Hello World!", id=AnyStr()),
-                ]
+                messages=[HumanMessage(content="hi"), AIMessage(content="Hello World!")]
             )
         }
 
@@ -2802,14 +2779,19 @@ async def test_remote_configurable_remote_runnable() -> None:
 async def get_langsmith_client() -> AsyncIterator[MagicMock]:
     """Get a patched langsmith client."""
     with patch("langserve.api_handler.ls_client") as mocked_ls_client_package:
-        with patch("langserve.api_handler.tracing_is_enabled") as tracing_is_enabled:
-            tracing_is_enabled.return_value = True
-            mocked_client = MagicMock(auto_spec=Client)
-            mocked_ls_client_package.Client.return_value = mocked_client
-            yield mocked_client
+        with patch("langserve.api_handler._is_scoped_feedback_enabled") as f:
+            # Enable scoped feedback for now.
+            f.return_value = True
+            with patch(
+                "langserve.api_handler.tracing_is_enabled"
+            ) as tracing_is_enabled:
+                tracing_is_enabled.return_value = True
+                mocked_client = MagicMock(auto_spec=Client)
+                mocked_ls_client_package.Client.return_value = mocked_client
+                yield mocked_client
 
 
-async def test_token_feedback_included_in_responses() -> None:
+async def test_scoped_feedback() -> None:
     """Test that information to leave scoped feedback is passed to the client
     is present in the server response.
     """
@@ -2853,7 +2835,7 @@ async def test_token_feedback_included_in_responses() -> None:
                         {
                             "expires_at": "2023-01-01T00:00:00",
                             "key": "foo",
-                            "token_url": "feedback_id",
+                            "token_url": None,
                         }
                     ],
                     "run_id": run_id,
@@ -2883,7 +2865,7 @@ async def test_token_feedback_included_in_responses() -> None:
                                 {
                                     "expires_at": "2023-01-01T00:00:00",
                                     "key": "foo",
-                                    "token_url": "feedback_id",
+                                    "token_url": None,
                                 }
                             ]
                         },
@@ -2892,7 +2874,7 @@ async def test_token_feedback_included_in_responses() -> None:
                                 {
                                     "expires_at": "2023-01-01T00:00:00",
                                     "key": "foo",
-                                    "token_url": "feedback_id",
+                                    "token_url": None,
                                 }
                             ]
                         },
@@ -2955,35 +2937,6 @@ async def test_token_feedback_included_in_responses() -> None:
                 "type": "metadata",
             }
 
-            # Test astream log
-            response = await async_client.post(
-                "/stream_log",
-                json={"input": "hello"},
-            )
-            events = _decode_eventstream(response.text)
-            for event in events:
-                if "data" in event and "run_id" in event["data"]:
-                    del event["data"]["run_id"]
-
-            # Find the metadata event and pull it out
-            metadata_event = None
-            for event in events:
-                if event["type"] == "metadata":
-                    metadata_event = event
-
-            assert metadata_event == {
-                "data": {
-                    "feedback_tokens": [
-                        {
-                            "expires_at": "2023-01-01T00:00:00",
-                            "key": "foo",
-                            "token_url": "feedback_id",
-                        }
-                    ]
-                },
-                "type": "metadata",
-            }
-
 
 async def test_passing_run_id_from_client() -> None:
     """test that the client can set a run id if server allows it."""
@@ -3044,37 +2997,3 @@ async def test_passing_bad_runnable_to_add_routes() -> None:
         add_routes(FastAPI(), "not a runnable")
 
     assert e.match("Expected a Runnable, got <class 'str'>")
-
-
-async def test_token_feedback_endpoint() -> None:
-    """Tests that the feedback endpoint can accept feedback to langsmith."""
-    async with get_langsmith_client() as client:
-        local_app = FastAPI()
-        add_routes(
-            local_app,
-            RunnableLambda(lambda foo: "hello"),
-            token_feedback_config={
-                "key_configs": [
-                    {
-                        "key": "silliness",
-                    }
-                ]
-            },
-        )
-
-        async with get_async_test_client(
-            local_app, raise_app_exceptions=True
-        ) as async_client:
-            response = await async_client.post(
-                "/token_feedback", json={"token_or_url": "some_url", "score": 3}
-            )
-            assert response.status_code == 200
-
-            call = client.create_feedback_from_token.call_args
-            assert call.args[0] == "some_url"
-            assert call.kwargs == {
-                "comment": None,
-                "metadata": {"from_langserve": True},
-                "score": 3,
-                "value": None,
-            }

tests/unit_tests/utils/stubs.py

@@ -1,6 +0,0 @@
-from typing import Any
-
-
-class AnyStr(str):
-    def __eq__(self, other: Any) -> bool:
-        return isinstance(other, str)

tests/unit_tests/utils/test_fake_chat_model.py

@@ -8,7 +8,6 @@ from langchain_core.messages import AIMessage, AIMessageChunk, BaseMessage
 from langchain_core.outputs import ChatGenerationChunk, GenerationChunk
 
 from tests.unit_tests.utils.llms import GenericFakeChatModel
-from tests.unit_tests.utils.stubs import AnyStr
 
 
 def test_generic_fake_chat_model_invoke() -> None:
@@ -16,11 +15,11 @@ def test_generic_fake_chat_model_invoke() -> None:
     infinite_cycle = cycle([AIMessage(content="hello"), AIMessage(content="goodbye")])
     model = GenericFakeChatModel(messages=infinite_cycle)
     response = model.invoke("meow")
-    assert response == AIMessage(content="hello", id=AnyStr())
+    assert response == AIMessage(content="hello")
     response = model.invoke("kitty")
-    assert response == AIMessage(content="goodbye", id=AnyStr())
+    assert response == AIMessage(content="goodbye")
     response = model.invoke("meow")
-    assert response == AIMessage(content="hello", id=AnyStr())
+    assert response == AIMessage(content="hello")
 
 
 async def test_generic_fake_chat_model_ainvoke() -> None:
@@ -28,11 +27,11 @@ async def test_generic_fake_chat_model_ainvoke() -> None:
     infinite_cycle = cycle([AIMessage(content="hello"), AIMessage(content="goodbye")])
     model = GenericFakeChatModel(messages=infinite_cycle)
     response = await model.ainvoke("meow")
-    assert response == AIMessage(content="hello", id=AnyStr())
+    assert response == AIMessage(content="hello")
     response = await model.ainvoke("kitty")
-    assert response == AIMessage(content="goodbye", id=AnyStr())
+    assert response == AIMessage(content="goodbye")
     response = await model.ainvoke("meow")
-    assert response == AIMessage(content="hello", id=AnyStr())
+    assert response == AIMessage(content="hello")
 
 
 async def test_generic_fake_chat_model_stream() -> None:
@@ -45,28 +44,26 @@ async def test_generic_fake_chat_model_stream() -> None:
     model = GenericFakeChatModel(messages=infinite_cycle)
     chunks = [chunk async for chunk in model.astream("meow")]
     assert chunks == [
-        AIMessageChunk(content="hello", id=AnyStr()),
-        AIMessageChunk(content=" ", id=AnyStr()),
-        AIMessageChunk(content="goodbye", id=AnyStr()),
+        AIMessageChunk(content="hello"),
+        AIMessageChunk(content=" "),
+        AIMessageChunk(content="goodbye"),
     ]
 
     chunks = [chunk for chunk in model.stream("meow")]
     assert chunks == [
-        AIMessageChunk(content="hello", id=AnyStr()),
-        AIMessageChunk(content=" ", id=AnyStr()),
-        AIMessageChunk(content="goodbye", id=AnyStr()),
+        AIMessageChunk(content="hello"),
+        AIMessageChunk(content=" "),
+        AIMessageChunk(content="goodbye"),
     ]
 
     # Test streaming of additional kwargs.
     # Relying on insertion order of the additional kwargs dict
-    message = AIMessage(
-        content="", additional_kwargs={"foo": 42, "bar": 24}, id=AnyStr()
-    )
+    message = AIMessage(content="", additional_kwargs={"foo": 42, "bar": 24})
     model = GenericFakeChatModel(messages=cycle([message]))
     chunks = [chunk async for chunk in model.astream("meow")]
     assert chunks == [
-        AIMessageChunk(content="", additional_kwargs={"foo": 42}, id=AnyStr()),
-        AIMessageChunk(content="", additional_kwargs={"bar": 24}, id=AnyStr()),
+        AIMessageChunk(content="", additional_kwargs={"foo": 42}),
+        AIMessageChunk(content="", additional_kwargs={"bar": 24}),
     ]
 
     message = AIMessage(
@@ -84,28 +81,22 @@ async def test_generic_fake_chat_model_stream() -> None:
 
     assert chunks == [
         AIMessageChunk(
-            content="",
-            additional_kwargs={"function_call": {"name": "move_file"}},
-            id=AnyStr(),
+            content="", additional_kwargs={"function_call": {"name": "move_file"}}
         ),
         AIMessageChunk(
             content="",
             additional_kwargs={
                 "function_call": {"arguments": '{\n  "source_path": "foo"'}
             },
-            id=AnyStr(),
         ),
         AIMessageChunk(
-            content="",
-            additional_kwargs={"function_call": {"arguments": ","}},
-            id=AnyStr(),
+            content="", additional_kwargs={"function_call": {"arguments": ","}}
         ),
         AIMessageChunk(
             content="",
             additional_kwargs={
                 "function_call": {"arguments": '\n  "destination_path": "bar"\n}'}
             },
-            id=AnyStr(),
         ),
     ]
 
@@ -125,7 +116,6 @@ async def test_generic_fake_chat_model_stream() -> None:
                 'destination_path": "bar"\n}',
             }
         },
-        id=AnyStr(),
     )
 
 
@@ -138,9 +128,9 @@ async def test_generic_fake_chat_model_astream_log() -> None:
     ]
     final = log_patches[-1]
     assert final.state["streamed_output"] == [
-        AIMessageChunk(content="hello", id=AnyStr()),
-        AIMessageChunk(content=" ", id=AnyStr()),
-        AIMessageChunk(content="goodbye", id=AnyStr()),
+        AIMessageChunk(content="hello"),
+        AIMessageChunk(content=" "),
+        AIMessageChunk(content="goodbye"),
     ]
 
 
@@ -188,8 +178,8 @@ async def test_callback_handlers() -> None:
     # New model
     results = list(model.stream("meow", {"callbacks": [MyCustomAsyncHandler(tokens)]}))
     assert results == [
-        AIMessageChunk(content="hello", id=AnyStr()),
-        AIMessageChunk(content=" ", id=AnyStr()),
-        AIMessageChunk(content="goodbye", id=AnyStr()),
+        AIMessageChunk(content="hello"),
+        AIMessageChunk(content=" "),
+        AIMessageChunk(content="goodbye"),
     ]
     assert tokens == ["hello", " ", "goodbye"]