mirror of
https://github.com/langchain-ai/learning-langchain.git
synced 2026-07-01 16:06:32 -04:00
update: ch10 retriever and agent evaluation
This commit is contained in:
mayo
@@ -6,7 +6,6 @@ LANGCHAIN_TRACING_V2=true
|
||||
|
||||
LANGCHAIN_PROJECT="learning-langchain"
|
||||
|
||||
|
||||
## Supabase keys are only used in Chapter 9 deployment examples
|
||||
SUPABASE_URL=
|
||||
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
from langchain_core.prompts import ChatPromptTemplate
|
||||
from langchain_core.pydantic_v1 import BaseModel, Field
|
||||
from langchain_openai import ChatOpenAI
|
||||
|
||||
### --- Create a grader for retrieved documents ---
|
||||
|
||||
|
||||
# Data model
|
||||
class GradeDocuments(BaseModel):
|
||||
"""Binary score for relevance check on retrieved documents."""
|
||||
|
||||
binary_score: str = Field(
|
||||
description="Documents are relevant to the question, 'yes' or 'no'"
|
||||
)
|
||||
|
||||
|
||||
# LLM with function call
|
||||
llm = ChatOpenAI(model="gpt-3.5-turbo-0125", temperature=0)
|
||||
structured_llm_grader = llm.with_structured_output(GradeDocuments)
|
||||
|
||||
# Prompt
|
||||
system = """You are a grader assessing relevance of a retrieved document to a user question.
|
||||
If the document contains keyword(s) or semantic meaning related to the question, grade it as relevant.
|
||||
Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question."""
|
||||
grade_prompt = ChatPromptTemplate.from_messages(
|
||||
[
|
||||
("system", system),
|
||||
("human", "Retrieved document: \n\n {document} \n\n User question: {question}"),
|
||||
]
|
||||
)
|
||||
|
||||
retrieval_grader = grade_prompt | structured_llm_grader
|
||||
@@ -1,223 +0,0 @@
|
||||
from typing import List, TypedDict
|
||||
|
||||
from langchain_core.schema import Document
|
||||
from langchain_community.tools.tavily_search import TavilySearchResults
|
||||
from langgraph.graph import END, StateGraph, START
|
||||
|
||||
from .grader import retrieval_grader
|
||||
from .retriever import retriever
|
||||
from .rag import rag_chain
|
||||
|
||||
from langchain_openai import ChatOpenAI
|
||||
from langchain_core.prompts import ChatPromptTemplate
|
||||
from langchain_core.output_parsers import StrOutputParser
|
||||
|
||||
|
||||
|
||||
### Question Re-writer
|
||||
|
||||
# LLM
|
||||
llm = ChatOpenAI(model="gpt-4", temperature=0)
|
||||
|
||||
# Prompt
|
||||
system = """You a question re-writer that converts an input question to a better version that is optimized \n
|
||||
for web search. Look at the input and try to reason about the underlying semantic intent / meaning."""
|
||||
re_write_prompt = ChatPromptTemplate.from_messages(
|
||||
[
|
||||
("system", system),
|
||||
(
|
||||
"human",
|
||||
"Here is the initial question: \n\n {question} \n Formulate an improved question.",
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
question_rewriter = re_write_prompt | llm | StrOutputParser()
|
||||
|
||||
# --- Create the graph ---
|
||||
|
||||
web_search_tool = TavilySearchResults(k=3)
|
||||
|
||||
|
||||
class GraphState(TypedDict):
|
||||
"""
|
||||
Represents the state of our graph.
|
||||
|
||||
Attributes:
|
||||
question: question
|
||||
generation: LLM generation
|
||||
web_search: whether to add search
|
||||
documents: list of documents
|
||||
"""
|
||||
|
||||
question: str
|
||||
generation: str
|
||||
web_search: str
|
||||
documents: List[str]
|
||||
|
||||
|
||||
def retrieve(state):
|
||||
"""
|
||||
Retrieve documents
|
||||
|
||||
Args:
|
||||
state (dict): The current graph state
|
||||
|
||||
Returns:
|
||||
state (dict): New key added to state, documents, that contains retrieved documents
|
||||
"""
|
||||
question = state["question"]
|
||||
|
||||
# Retrieval
|
||||
documents = retriever.get_relevant_documents(question)
|
||||
return {"documents": documents, "question": question}
|
||||
|
||||
|
||||
def generate(state):
|
||||
"""
|
||||
Generate answer
|
||||
|
||||
Args:
|
||||
state (dict): The current graph state
|
||||
|
||||
Returns:
|
||||
state (dict): New key added to state, generation, that contains LLM generation
|
||||
"""
|
||||
question = state["question"]
|
||||
documents = state["documents"]
|
||||
|
||||
# RAG generation
|
||||
generation = rag_chain.invoke({"context": documents, "question": question})
|
||||
return {"documents": documents, "question": question, "generation": generation}
|
||||
|
||||
|
||||
def grade_documents(state):
|
||||
"""
|
||||
Determines whether the retrieved documents are relevant to the question.
|
||||
|
||||
Args:
|
||||
state (dict): The current graph state
|
||||
|
||||
Returns:
|
||||
state (dict): Updates documents key with only filtered relevant documents
|
||||
"""
|
||||
|
||||
question = state["question"]
|
||||
documents = state["documents"]
|
||||
|
||||
# Score each doc
|
||||
filtered_docs = []
|
||||
web_search = "No"
|
||||
for d in documents:
|
||||
score = retrieval_grader.invoke(
|
||||
{"question": question, "document": d.page_content}
|
||||
)
|
||||
grade = score.binary_score
|
||||
if grade == "yes":
|
||||
print("---GRADE: DOCUMENT RELEVANT---")
|
||||
filtered_docs.append(d)
|
||||
else:
|
||||
print("---GRADE: DOCUMENT NOT RELEVANT---")
|
||||
web_search = "Yes"
|
||||
continue
|
||||
return {"documents": filtered_docs, "question": question, "web_search": web_search}
|
||||
|
||||
|
||||
def transform_query(state):
|
||||
"""
|
||||
Transform the query to produce a better question.
|
||||
|
||||
Args:
|
||||
state (dict): The current graph state
|
||||
|
||||
Returns:
|
||||
state (dict): Updates question key with a re-phrased question
|
||||
"""
|
||||
|
||||
question = state["question"]
|
||||
documents = state["documents"]
|
||||
|
||||
# Re-write question
|
||||
better_question = question_rewriter.invoke({"question": question})
|
||||
return {"documents": documents, "question": better_question}
|
||||
|
||||
|
||||
def web_search(state):
|
||||
"""
|
||||
Web search based on the re-phrased question.
|
||||
|
||||
Args:
|
||||
state (dict): The current graph state
|
||||
|
||||
Returns:
|
||||
state (dict): Updates documents key with appended web results
|
||||
"""
|
||||
|
||||
print("---WEB SEARCH---")
|
||||
question = state["question"]
|
||||
documents = state["documents"]
|
||||
|
||||
# Web search
|
||||
docs = web_search_tool.invoke({"query": question})
|
||||
web_results = "\n".join([d["content"] for d in docs])
|
||||
web_results = Document(page_content=web_results)
|
||||
documents.append(web_results)
|
||||
|
||||
return {"documents": documents, "question": question}
|
||||
|
||||
|
||||
### Edges
|
||||
|
||||
|
||||
def decide_to_generate(state):
|
||||
"""
|
||||
Determines whether to generate an answer, or re-generate a question.
|
||||
|
||||
Args:
|
||||
state (dict): The current graph state
|
||||
|
||||
Returns:
|
||||
str: Binary decision for next node to call
|
||||
"""
|
||||
|
||||
state["question"]
|
||||
web_search = state["web_search"]
|
||||
state["documents"]
|
||||
|
||||
if web_search == "Yes":
|
||||
# All documents have been filtered check_relevance
|
||||
# We will re-generate a new query
|
||||
return "transform_query"
|
||||
else:
|
||||
# We have relevant documents, so generate answer
|
||||
return "generate"
|
||||
|
||||
|
||||
### Graph
|
||||
|
||||
workflow = StateGraph(GraphState)
|
||||
|
||||
# Define the nodes
|
||||
workflow.add_node("retrieve", retrieve) # retrieve
|
||||
workflow.add_node("grade_documents", grade_documents) # grade documents
|
||||
workflow.add_node("generate", generate) # generatae
|
||||
workflow.add_node("transform_query", transform_query) # transform_query
|
||||
workflow.add_node("web_search_node", web_search) # web search
|
||||
|
||||
# Build graph
|
||||
workflow.add_edge(START, "retrieve")
|
||||
workflow.add_edge("retrieve", "grade_documents")
|
||||
workflow.add_conditional_edges(
|
||||
"grade_documents",
|
||||
decide_to_generate,
|
||||
{
|
||||
"transform_query": "transform_query",
|
||||
"generate": "generate",
|
||||
},
|
||||
)
|
||||
workflow.add_edge("transform_query", "web_search_node")
|
||||
workflow.add_edge("web_search_node", "generate")
|
||||
workflow.add_edge("generate", END)
|
||||
|
||||
# Compile
|
||||
app = workflow.compile()
|
||||
@@ -1,23 +0,0 @@
|
||||
### Generate
|
||||
|
||||
from langchain import hub
|
||||
from langchain_core.output_parsers import StrOutputParser
|
||||
from langchain_openai import ChatOpenAI
|
||||
|
||||
# question
|
||||
question = "Explain how the different types of agent memory work?"
|
||||
|
||||
# Prompt
|
||||
prompt = hub.pull("rlm/rag-prompt")
|
||||
|
||||
# LLM
|
||||
llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)
|
||||
|
||||
|
||||
# Post-processing
|
||||
def format_docs(docs):
|
||||
return "\n\n".join(doc.page_content for doc in docs)
|
||||
|
||||
|
||||
# Chain
|
||||
rag_chain = prompt | llm | StrOutputParser()
|
||||
@@ -1,28 +0,0 @@
|
||||
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
||||
from langchain_community.document_loaders import WebBaseLoader
|
||||
from langchain_community.vectorstores import Chroma
|
||||
from langchain_openai import OpenAIEmbeddings
|
||||
|
||||
# --- Create an index of documents ---
|
||||
|
||||
urls = [
|
||||
"https://lilianweng.github.io/posts/2023-06-23-agent/",
|
||||
"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/",
|
||||
"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/",
|
||||
]
|
||||
|
||||
docs = [WebBaseLoader(url).load() for url in urls]
|
||||
docs_list = [item for sublist in docs for item in sublist]
|
||||
|
||||
text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
|
||||
chunk_size=250, chunk_overlap=0
|
||||
)
|
||||
doc_splits = text_splitter.split_documents(docs_list)
|
||||
|
||||
# Add to vectorDB
|
||||
vectorstore = Chroma.from_documents(
|
||||
documents=doc_splits,
|
||||
collection_name="rag-chroma",
|
||||
embedding=OpenAIEmbeddings(),
|
||||
)
|
||||
retriever = vectorstore.as_retriever()
|
||||
+71
-2
@@ -1,8 +1,8 @@
|
||||
# Learning LangChain RAG AI Research Agent Deployment Example
|
||||
# RAG AI Research Agent Deployment Example
|
||||
|
||||
In Chapter 9, you learnt about how to deploy a RAG AI Research agent using LangGraph. This chapter contains the full code for the application discussed in the chapter.
|
||||
|
||||
### Environment variables setup
|
||||
### Prerequisites
|
||||
|
||||
First, you need to ensure you have set the environment variables required to run the examples in this repository at the root of the repository (if you haven't already).
|
||||
|
||||
@@ -23,6 +23,68 @@ Supabase is used as the vector store for the examples. To get your supabase keys
|
||||
- In the settings section, navigate to the API section to see your keys.
|
||||
- Copy the project url and `service_role` key and add them to the `.env` file as values for `SUPABASE_URL` and `SUPABASE_SERVICE_ROLE_KEY`.
|
||||
|
||||
Once you’ve created a database, run the following SQL to set up `pgvector` and create the necessary table and functions:
|
||||
|
||||
```sql
|
||||
-- Enable the pgvector extension to work with embedding vectors
|
||||
create extension vector;
|
||||
|
||||
-- Create a table to store your documents
|
||||
create table documents (
|
||||
id bigserial primary key,
|
||||
content text, -- corresponds to Document.pageContent
|
||||
metadata jsonb, -- corresponds to Document.metadata
|
||||
embedding vector(1536) -- 1536 works for OpenAI embeddings, change if needed
|
||||
);
|
||||
|
||||
-- Create a function to search for documents
|
||||
create function match_documents (
|
||||
query_embedding vector(1536),
|
||||
match_count int DEFAULT null,
|
||||
filter jsonb DEFAULT '{}'
|
||||
) returns table (
|
||||
id bigint,
|
||||
content text,
|
||||
metadata jsonb,
|
||||
embedding jsonb,
|
||||
similarity float
|
||||
)
|
||||
language plpgsql
|
||||
as $$
|
||||
#variable_conflict use_column
|
||||
begin
|
||||
return query
|
||||
select
|
||||
id,
|
||||
content,
|
||||
metadata,
|
||||
(embedding::text)::jsonb as embedding,
|
||||
1 - (documents.embedding <=> query_embedding) as similarity
|
||||
from documents
|
||||
where metadata @> filter
|
||||
order by documents.embedding <=> query_embedding
|
||||
limit match_count;
|
||||
end;
|
||||
$$;
|
||||
```
|
||||
|
||||
To test if the `pgvector` extension is set up correctly, you can run the following SQL query in the supabase SQL editor:
|
||||
|
||||
```sql
|
||||
-- Insert test document
|
||||
INSERT INTO documents (content, metadata, embedding)
|
||||
VALUES (
|
||||
'Test document',
|
||||
'{"category": "test", "author": "supabase"}',
|
||||
'[1,1,1]'::vector(1536)
|
||||
);
|
||||
|
||||
-- Search using match_documents function
|
||||
SELECT * FROM match_documents(
|
||||
query_embedding => '[1,1,1]'::vector(1536),
|
||||
match_count => 1
|
||||
);
|
||||
```
|
||||
|
||||
### Quick Start:
|
||||
|
||||
@@ -32,3 +94,10 @@ The python version is in the `py` folder, whilst the javascript version is in th
|
||||
|
||||
- For python, open the `py` folder and follow the instructions in the `README.md` file.
|
||||
- For javascript, open the `js` folder and follow the instructions in the `README.md` file.
|
||||
|
||||
### Deployment options
|
||||
|
||||
After duplicating app logic for your language of choice, you can deploy the agent on the LangGraph Platform or self-host it.
|
||||
|
||||
- If you're deploying the agent on the LangGraph Platform, you can follow the guide [here](https://langchain-ai.github.io/langgraph/cloud/deployment/cloud/).
|
||||
- If you're self-hosting the agent, you can follow the guide [here](https://langchain-ai.github.io/langgraph/concepts/self_hosted/).
|
||||
|
||||
@@ -18,6 +18,8 @@ dependencies = [
|
||||
"pypdf>=5.1.0",
|
||||
"psycopg[binary]>=3.2.4", # Updated to include [binary] extra
|
||||
"setuptools>=75.8.0",
|
||||
"langsmith>=0.3.2",
|
||||
"langgraph-checkpoint-sqlite>=2.0.3",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
|
||||
Reference in New Issue
Block a user