mirror of
https://github.com/langchain-ai/auto-evaluator.git
synced 2026-07-01 20:44:05 -04:00
Merge pull request #98 from langchain-ai/streamlit_app
Streamlit app for Pinecone
This commit is contained in:
Lance Martin
@@ -0,0 +1,51 @@
|
||||
# `VectorDB Auto-evaluator` :brain: :memo:
|
||||
|
||||
This is a lightweight evaluation tool connected to a Pinecone vectorDB that uses Langchain to test different question answering configurations:
|
||||
|
||||
- Pass in a test set of question/answer pairs: see `eval_sets/lex-pod-eval.json`
|
||||
|
||||
- Specify a retrieval method, the number of chunks to retrieve, VectorDB embeddings, a model for answer synthesis, and the model-graded evaluation prompt
|
||||
|
||||
- The app will use an LLM (`GPT-4`) to score the response relative to the correct `answer`
|
||||
|
||||
**Run as Streamlit app**
|
||||
|
||||
`pip install -r requirements.txt`
|
||||
|
||||
`streamlit run auto-evaluator.py`
|
||||
|
||||
**Inputs**
|
||||
|
||||
`retriever_type` - Chunk retrieval method
|
||||
|
||||
`num_neighbors` - Neighbors for retrieval
|
||||
|
||||
`embeddings` - Embeddings in your Pinecone vectorDB
|
||||
|
||||
`model` - LLM for summarization of retrieved chunks
|
||||
|
||||
`grade_prompt` - Prompt choice for model self-grading
|
||||
|
||||
**Retrievers**
|
||||
|
||||
`Pinecone w/ metadata filtering`
|
||||
|
||||
* Hard-code the metadata fileter you want to use: `metadata_filter = {'id':"0252"}`
|
||||
|
||||
`Pinecone w/ self-querying`
|
||||
|
||||
* Specify the `metadata_field_info` (see: `self_query_retriever_lex.py`) and the `SelfQueryRetriever` will try to extract metadata filters from the question
|
||||
|
||||
`Kor filtering`
|
||||
|
||||
* Specify the `schema` for Kor (see: `kor_retriever_lex.py`) and define a `kor_retriever` that will try to extract metadata filters from the question
|
||||
|
||||
**Blog**
|
||||
|
||||
https://rlancemartin.notion.site/Auto-Evaluation-of-Metadata-Filtering-18502448c85240828f33716740f9574b
|
||||
|
||||
**UI**
|
||||
|
||||
**Disclaimer**
|
||||
|
||||
```You will need an OpenAI API key with access to `GPT-4` for the default grading, but this can be modified in grade_model_retrieval and grade_model_answer if needed.```
|
||||
@@ -0,0 +1,349 @@
|
||||
import json
|
||||
import time
|
||||
import pinecone
|
||||
import pandas as pd
|
||||
import altair as alt
|
||||
import streamlit as st
|
||||
from typing import List
|
||||
from langchain.vectorstores import Pinecone
|
||||
from langchain.llms import Anthropic
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langchain.evaluation.qa import QAEvalChain
|
||||
from langchain.embeddings import HuggingFaceEmbeddings
|
||||
from langchain.embeddings.openai import OpenAIEmbeddings
|
||||
from langchain.chains.question_answering import load_qa_chain
|
||||
from langchain.retrievers.self_query.base import SelfQueryRetriever
|
||||
from kor_retriever_lex import kor_retriever
|
||||
from langchain.docstore.document import Document
|
||||
from self_query_retriever_lex import metadata_field_info, document_content_description
|
||||
from prompts import GRADE_DOCS_PROMPT, GRADE_ANSWER_PROMPT, GRADE_ANSWER_PROMPT_FAST, GRADE_ANSWER_PROMPT_BIAS_CHECK, GRADE_ANSWER_PROMPT_OPENAI, QA_CHAIN_PROMPT_LEX, QA_CHAIN_PROMPT_TRAVEL
|
||||
|
||||
# Keep dataframe in memory to accumulate experimental results
|
||||
if "existing_df" not in st.session_state:
|
||||
summary = pd.DataFrame(columns=['model',
|
||||
'retriever',
|
||||
'embedding',
|
||||
'num_neighbors',
|
||||
'Latency',
|
||||
'Retrieval score',
|
||||
'Answer score'])
|
||||
st.session_state.existing_df = summary
|
||||
else:
|
||||
summary = st.session_state.existing_df
|
||||
|
||||
@st.cache_resource
|
||||
def make_llm(model_version: str):
|
||||
"""
|
||||
Make LLM from model version
|
||||
@param model_version: model_version
|
||||
@return: LLN
|
||||
"""
|
||||
if (model_version == "gpt-3.5-turbo") or (model_version == "gpt-4"):
|
||||
chosen_model = ChatOpenAI(model_name=model_version, temperature=0)
|
||||
elif model_version == "anthropic":
|
||||
chosen_model = Anthropic(temperature=0)
|
||||
else:
|
||||
st.warning("`Model version not recognized. Using gpt-3.5-turbo`", icon="⚠️")
|
||||
chosen_model = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)
|
||||
return chosen_model
|
||||
|
||||
@st.cache_resource
|
||||
def make_retriever(retriever_type,embedding_type,pc_api_key,pc_region,pc_index):
|
||||
"""
|
||||
Make document retriever
|
||||
@param retriever_type: retriever type
|
||||
@param embedding_type: embedding type
|
||||
@param num_neighbors: number of neighbors for retrieval
|
||||
@return: Pinecone
|
||||
"""
|
||||
st.info("`Connecting to Pinecone ...`")
|
||||
|
||||
# Retriver type
|
||||
if retriever_type in ("Pinecone","Pinecone w/ metadata filtering"):
|
||||
return p
|
||||
elif retriever_type == "Pinecone w/ self-querying":
|
||||
return SelfQueryRetriever.from_llm(ChatOpenAI(model_name="gpt-3.5-turbo",temperature=0), p, document_content_description, metadata_field_info, verbose=True, k=10)
|
||||
elif retriever_type == "Kor filtering":
|
||||
return kor_retriever
|
||||
|
||||
def make_chain(llm):
|
||||
"""
|
||||
Make retrieval chain
|
||||
@param retriever: retriever
|
||||
@param retriever_type: retriever type
|
||||
@return: QA chain
|
||||
"""
|
||||
|
||||
qa_chain = load_qa_chain(llm, chain_type="stuff", prompt=QA_CHAIN_PROMPT_LEX)
|
||||
|
||||
return qa_chain
|
||||
|
||||
|
||||
def grade_model_answer(predicted_dataset: List, predictions: List, grade_answer_prompt: str) -> List:
|
||||
"""
|
||||
Grades the distilled answer based on ground truth and model predictions.
|
||||
@param predicted_dataset: A list of dictionaries containing ground truth questions and answers.
|
||||
@param predictions: A list of dictionaries containing model predictions for the questions.
|
||||
@param grade_answer_prompt: The prompt level for the grading. Either "Fast" or "Full".
|
||||
@return: A list of scores for the distilled answers.
|
||||
"""
|
||||
# Grade the distilled answer
|
||||
st.info("`Grading model answer ...`")
|
||||
# Set the grading prompt based on the grade_answer_prompt parameter
|
||||
if grade_answer_prompt == "Fast":
|
||||
prompt = GRADE_ANSWER_PROMPT_FAST
|
||||
elif grade_answer_prompt == "Descriptive w/ bias check":
|
||||
prompt = GRADE_ANSWER_PROMPT_BIAS_CHECK
|
||||
elif grade_answer_prompt == "OpenAI grading prompt":
|
||||
prompt = GRADE_ANSWER_PROMPT_OPENAI
|
||||
else:
|
||||
prompt = GRADE_ANSWER_PROMPT
|
||||
|
||||
# Create an evaluation chain
|
||||
eval_chain = QAEvalChain.from_llm(
|
||||
llm=ChatOpenAI(model_name="gpt-4", temperature=0),
|
||||
prompt=prompt
|
||||
)
|
||||
|
||||
# Evaluate the predictions and ground truth using the evaluation chain
|
||||
graded_outputs = eval_chain.evaluate(
|
||||
predicted_dataset,
|
||||
predictions,
|
||||
question_key="question",
|
||||
prediction_key="result"
|
||||
)
|
||||
|
||||
return graded_outputs
|
||||
|
||||
|
||||
def grade_model_retrieval(gt_dataset: List, predictions: List, grade_docs_prompt: str):
|
||||
"""
|
||||
Grades the relevance of retrieved documents based on ground truth and model predictions.
|
||||
@param gt_dataset: list of dictionaries containing ground truth questions and answers.
|
||||
@param predictions: list of dictionaries containing model predictions for the questions
|
||||
@param grade_docs_prompt: prompt level for the grading. Either "Fast" or "Full"
|
||||
@return: list of scores for the retrieved documents.
|
||||
"""
|
||||
# Grade the docs retrieval
|
||||
st.info("`Grading relevance of retrieved docs ...`")
|
||||
|
||||
# Set the grading prompt based on the grade_docs_prompt parameter
|
||||
prompt = GRADE_DOCS_PROMPT
|
||||
|
||||
# Create an evaluation chain
|
||||
eval_chain = QAEvalChain.from_llm(
|
||||
llm=ChatOpenAI(model_name="gpt-4", temperature=0),
|
||||
prompt=prompt
|
||||
)
|
||||
|
||||
# Evaluate the predictions and ground truth using the evaluation chain
|
||||
graded_outputs = eval_chain.evaluate(
|
||||
gt_dataset,
|
||||
predictions,
|
||||
question_key="question",
|
||||
prediction_key="result"
|
||||
)
|
||||
return graded_outputs
|
||||
|
||||
|
||||
def run_evaluation(chain, retriever, eval_set, grade_prompt, retriever_type, num_neighbors):
|
||||
"""
|
||||
Runs evaluation on a model's performance on a given evaluation dataset.
|
||||
@param chain: Model chain used for answering questions
|
||||
@param retriever: Document retriever used for retrieving relevant documents
|
||||
@param eval_set: List of dictionaries containing questions and corresponding ground truth answers
|
||||
@param grade_prompt: String prompt used for grading model's performance
|
||||
@param retriever_type: String specifying the type of retriever used
|
||||
@param num_neighbors: Number of neighbors to retrieve using the retriever
|
||||
@return: A tuple of four items:
|
||||
- answers_grade: A dictionary containing scores for the model's answers.
|
||||
- retrieval_grade: A dictionary containing scores for the model's document retrieval.
|
||||
- latencies_list: A list of latencies in seconds for each question answered.
|
||||
- predictions_list: A list of dictionaries containing the model's predicted answers and relevant documents for each question.
|
||||
"""
|
||||
st.info("`Running evaluation ...`")
|
||||
predictions_list = []
|
||||
retrieved_docs = []
|
||||
gt_dataset = []
|
||||
latencies_list = []
|
||||
|
||||
for data in eval_set:
|
||||
|
||||
# Get answer and log latency
|
||||
start_time = time.time()
|
||||
|
||||
# Get docs
|
||||
if retriever_type == "Pinecone w/ self-querying":
|
||||
docs = retriever.get_relevant_documents(data["question"])
|
||||
|
||||
elif retriever_type == "Pinecone w/ metadata filtering":
|
||||
### Set metadata here ###
|
||||
metadata_filter = {'id':"0252"}
|
||||
docs = retriever.similarity_search(query=data["question"],k=num_neighbors,filter=metadata_filter)
|
||||
|
||||
elif retriever_type == "Kor filtering":
|
||||
docs = retriever(p,data["question"])
|
||||
|
||||
else:
|
||||
docs = retriever.similarity_search(query=data["question"],k=num_neighbors)
|
||||
|
||||
print("--DOCS--")
|
||||
if not docs:
|
||||
docs=[Document(page_content="I was unable to recover any information about the question!")]
|
||||
print(docs)
|
||||
|
||||
# Get answer
|
||||
answer = chain.run(input_documents=docs,question=data["question"])
|
||||
predictions_list.append({"question": data["question"], "answer": data["answer"], "result": answer})
|
||||
gt_dataset.append(data)
|
||||
end_time = time.time()
|
||||
elapsed_time = end_time - start_time
|
||||
latencies_list.append(elapsed_time)
|
||||
|
||||
# Get doc text
|
||||
retrieved_doc_text = ""
|
||||
for i, doc in enumerate(docs):
|
||||
retrieved_doc_text += "Doc %s: " % str(i + 1) + doc.page_content + " "
|
||||
retrieved = {"question": data["question"], "answer": data["answer"], "result": retrieved_doc_text}
|
||||
retrieved_docs.append(retrieved)
|
||||
|
||||
# Grade docs and answer
|
||||
answers_grade = grade_model_answer(gt_dataset, predictions_list, grade_prompt)
|
||||
retrieval_grade = grade_model_retrieval(gt_dataset, retrieved_docs, grade_prompt)
|
||||
return answers_grade, retrieval_grade, latencies_list, predictions_list
|
||||
|
||||
# Auth
|
||||
st.sidebar.image("img/diagnostic.jpg")
|
||||
|
||||
with st.sidebar.form("user_input"):
|
||||
|
||||
# Pinecone params
|
||||
pc_api_key = st.text_input("`Pinecone API Key:`", type="password").strip()
|
||||
pc_region = st.text_input("`Pinecone region:`", type="password").strip()
|
||||
pc_index = st.text_input("`Pinecone index:`", type="password").strip()
|
||||
|
||||
retriever_type = st.radio("`Choose retriever`",
|
||||
("Pinecone",
|
||||
"Pinecone w/ self-querying",
|
||||
"Pinecone w/ metadata filtering",
|
||||
"Kor filtering"),
|
||||
index=0)
|
||||
|
||||
num_neighbors = st.select_slider("`Choose # chunks to retrieve`",
|
||||
options=[3, 4, 5, 6, 7, 8])
|
||||
|
||||
embeddings = st.radio("`Choose embeddings`",
|
||||
("HuggingFace",
|
||||
"OpenAI"),
|
||||
index=1)
|
||||
|
||||
model = st.radio("`Choose model`",
|
||||
("gpt-3.5-turbo",
|
||||
"gpt-4",
|
||||
"anthropic",),
|
||||
index=0)
|
||||
|
||||
grade_prompt = st.radio("`Grading style prompt`",
|
||||
("Fast",
|
||||
"Descriptive",
|
||||
"Descriptive w/ bias check",
|
||||
"OpenAI grading prompt"),
|
||||
index=3)
|
||||
|
||||
submitted = st.form_submit_button("Submit evaluation")
|
||||
|
||||
# App
|
||||
st.header("`VectorDB auto-evaluator`")
|
||||
st.info(
|
||||
"`I am an evaluation tool for question-answering using an existing vectorDB (currently Pinecone is supported) and an eval set. "
|
||||
"I will generate and grade an answer to each eval set question with the user-specific retrival setting, such as metadata filtering or self-querying retrieval."
|
||||
" Experiments with different configurations are logged. For an example eval set, see eval_sets/lex-pod-eval.json.`")
|
||||
|
||||
with st.form(key='file_inputs'):
|
||||
|
||||
uploaded_eval_set = st.file_uploader("`Please upload eval set (.json):` ",
|
||||
type=['json'],
|
||||
accept_multiple_files=False)
|
||||
|
||||
submitted = st.form_submit_button("Submit files")
|
||||
|
||||
# Build an index from the supplied docs
|
||||
if uploaded_eval_set and pc_api_key and pc_region and pc_index:
|
||||
|
||||
# Set embeddings (must match your Pinecone DB)
|
||||
if embeddings == "OpenAI":
|
||||
embedding = OpenAIEmbeddings()
|
||||
elif embeddings == "HuggingFace":
|
||||
embedding = HuggingFaceEmbeddings()
|
||||
|
||||
# Set Pinecone
|
||||
pinecone.init(api_key=str(pc_api_key), environment=str(pc_region))
|
||||
p = Pinecone.from_existing_index(index_name=str(pc_index), embedding=embedding)
|
||||
|
||||
# Eval set
|
||||
eval_set = json.loads(uploaded_eval_set.read())
|
||||
|
||||
# Make LLM
|
||||
llm = make_llm(model)
|
||||
|
||||
# Make retriver
|
||||
retriever = make_retriever(retriever_type,embeddings,pc_api_key,pc_region,pc_index)
|
||||
|
||||
# Make chain
|
||||
qa_chain = make_chain(llm)
|
||||
|
||||
# Grade model
|
||||
graded_answers, graded_retrieval, latency, predictions = run_evaluation(qa_chain, retriever, eval_set, grade_prompt,
|
||||
retriever_type, num_neighbors)
|
||||
|
||||
# Assemble outputs
|
||||
d = pd.DataFrame(predictions)
|
||||
d['answer score'] = [g['text'] for g in graded_answers]
|
||||
d['docs score'] = [g['text'] for g in graded_retrieval]
|
||||
d['latency'] = latency
|
||||
|
||||
# Summary statistics
|
||||
mean_latency = d['latency'].mean()
|
||||
correct_answer_count = len([text for text in d['answer score'] if "Incorrect" not in text])
|
||||
correct_docs_count = len([text for text in d['docs score'] if "Incorrect" not in text])
|
||||
percentage_answer = (correct_answer_count / len(graded_answers)) * 100
|
||||
percentage_docs = (correct_docs_count / len(graded_retrieval)) * 100
|
||||
|
||||
st.subheader("`Run Results`")
|
||||
st.info(
|
||||
"`I will grade the chain based on: 1/ the relevance of the retrived documents relative to the question and 2/ "
|
||||
"the summarized answer relative to the ground truth answer. You can see (and change) to prompts used for "
|
||||
"grading in text_utils`")
|
||||
st.dataframe(data=d, use_container_width=True)
|
||||
|
||||
# Accumulate results
|
||||
st.subheader("`Aggregate Results`")
|
||||
st.info(
|
||||
"`Retrieval and answer scores are percentage of retrived documents deemed relevant by the LLM grader ("
|
||||
"relative to the question) and percentage of summarized answers deemed relevant (relative to ground truth "
|
||||
"answer), respectively. The size of point correponds to the latency (in seconds) of retrieval + answer "
|
||||
"summarization (larger circle = slower).`")
|
||||
new_row = pd.DataFrame({'model': [model],
|
||||
'retriever': [retriever_type],
|
||||
'embedding': [embeddings],
|
||||
'num_neighbors': [num_neighbors],
|
||||
'Latency': [mean_latency],
|
||||
'Retrieval score': [percentage_docs],
|
||||
'Answer score': [percentage_answer]})
|
||||
summary = pd.concat([summary, new_row], ignore_index=True)
|
||||
st.dataframe(data=summary, use_container_width=True)
|
||||
st.session_state.existing_df = summary
|
||||
|
||||
# Dataframe for visualization
|
||||
show = summary.reset_index().copy()
|
||||
show.columns = ['expt number', 'model', 'retriever', 'embedding', 'num_neighbors', 'Latency', 'Retrieval score','Answer score']
|
||||
show['expt number'] = show['expt number'].apply(lambda x: "Expt #: " + str(x + 1))
|
||||
c = alt.Chart(show).mark_circle().encode(x='Retrieval score',
|
||||
y='Answer score',
|
||||
size=alt.Size('Latency'),
|
||||
color='expt number',
|
||||
tooltip=['expt number', 'Retrieval score', 'Latency', 'Answer score'])
|
||||
st.altair_chart(c, use_container_width=True, theme="streamlit")
|
||||
|
||||
else:
|
||||
st.warning('Please specify a Pinecone index and add an eval set.', icon="⚠️")
|
||||
@@ -0,0 +1,4 @@
|
||||
[
|
||||
{"question": "What does Elon Musk say about the self driving problem in episode 252?",
|
||||
"answer": "Elon mentions that the self-driving problem is harder than he thought because you need to build a silicon equivalent of vision that maps from camera to vector space. But, he also mentions that the disengagements has been dropping rapidly and the probability of an accident on FSD will be less than that of the average human within a year."}
|
||||
]
|
||||
@@ -0,0 +1,52 @@
|
||||
import requests
|
||||
from kor.extraction import create_extraction_chain
|
||||
from kor.nodes import Object, Text, Number
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langchain.docstore.document import Document
|
||||
|
||||
# Extraction schema -
|
||||
schema = Object(
|
||||
id = "episode_id",
|
||||
description = "An ID for each Lex Fridman podcast episode",
|
||||
attributes = [
|
||||
Text(
|
||||
id="episode_id",
|
||||
description="The podcast ID.",
|
||||
)
|
||||
],
|
||||
examples = [
|
||||
("What does episode 333 say about AI?", [{"episode_id": "0333"}]),
|
||||
("What does episode #231 say about dogs?", [{"episode_id": "0231"}]),
|
||||
("What is the summary of episode 50?",[{"episode_id": "050"}])
|
||||
],
|
||||
many = True,
|
||||
)
|
||||
|
||||
# Retriever -
|
||||
def kor_retriever(p,query):
|
||||
|
||||
# LLM -
|
||||
llm = ChatOpenAI(
|
||||
model_name="gpt-3.5-turbo",
|
||||
temperature=0,
|
||||
)
|
||||
|
||||
# Chain -
|
||||
chain = create_extraction_chain(llm, schema)
|
||||
|
||||
# City extraction -
|
||||
results = chain.predict_and_parse(text=(query.strip()))["data"]['episode_id']
|
||||
print("RESULTS in KOR")
|
||||
print(results)
|
||||
|
||||
# Get results -
|
||||
if results:
|
||||
metadata_filter = {'id':results[0]['episode_id']}
|
||||
docs = p.similarity_search(query=query,k=3,filter=metadata_filter)
|
||||
print("DOCS in KOR")
|
||||
print(docs)
|
||||
return docs
|
||||
|
||||
else:
|
||||
print("No results Kor retrieval!")
|
||||
return None
|
||||
@@ -0,0 +1,146 @@
|
||||
from langchain.prompts import PromptTemplate
|
||||
|
||||
template = """You are a teacher grading a quiz.
|
||||
You are given a question, the student's answer, and the true answer, and are asked to score the student answer as either Correct or Incorrect.
|
||||
|
||||
Example Format:
|
||||
QUESTION: question here
|
||||
STUDENT ANSWER: student's answer here
|
||||
TRUE ANSWER: true answer here
|
||||
GRADE: Correct or Incorrect here
|
||||
|
||||
Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. If the student answers that there is no specific information provided in the context, then the answer is Incorrect. Begin!
|
||||
|
||||
QUESTION: {query}
|
||||
STUDENT ANSWER: {result}
|
||||
TRUE ANSWER: {answer}
|
||||
GRADE:
|
||||
|
||||
Your response should be as follows:
|
||||
|
||||
GRADE: (Correct or Incorrect)
|
||||
(line break)
|
||||
JUSTIFICATION: (Without mentioning the student/teacher framing of this prompt, explain why the STUDENT ANSWER is Correct or Incorrect. Use one or two sentences maximum. Keep the answer as concise as possible.)
|
||||
"""
|
||||
|
||||
GRADE_ANSWER_PROMPT = PromptTemplate(input_variables=["query", "result", "answer"], template=template)
|
||||
|
||||
template = """You are a teacher grading a quiz.
|
||||
You are given a question, the student's answer, and the true answer, and are asked to score the student answer as either Correct or Incorrect.
|
||||
|
||||
Example Format:
|
||||
QUESTION: question here
|
||||
STUDENT ANSWER: student's answer here
|
||||
TRUE ANSWER: true answer here
|
||||
GRADE: Correct or Incorrect here
|
||||
|
||||
Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. If the student answers that there is no specific information provided in the context, then the answer is Incorrect. Begin!
|
||||
|
||||
QUESTION: {query}
|
||||
STUDENT ANSWER: {result}
|
||||
TRUE ANSWER: {answer}
|
||||
GRADE:"""
|
||||
|
||||
GRADE_ANSWER_PROMPT_FAST = PromptTemplate(input_variables=["query", "result", "answer"], template=template)
|
||||
|
||||
template = """You are a teacher grading a quiz.
|
||||
You are given a question, the student's answer, and the true answer, and are asked to score the student answer as either Correct or Incorrect.
|
||||
You are also asked to identify potential sources of bias in the question and in the true answer.
|
||||
|
||||
Example Format:
|
||||
QUESTION: question here
|
||||
STUDENT ANSWER: student's answer here
|
||||
TRUE ANSWER: true answer here
|
||||
GRADE: Correct or Incorrect here
|
||||
|
||||
Grade the student answers based ONLY on their factual accuracy. Ignore differences in punctuation and phrasing between the student answer and true answer. It is OK if the student answer contains more information than the true answer, as long as it does not contain any conflicting statements. If the student answers that there is no specific information provided in the context, then the answer is Incorrect. Begin!
|
||||
|
||||
QUESTION: {query}
|
||||
STUDENT ANSWER: {result}
|
||||
TRUE ANSWER: {answer}
|
||||
GRADE:
|
||||
|
||||
Your response should be as follows:
|
||||
|
||||
GRADE: (Correct or Incorrect)
|
||||
(line break)
|
||||
JUSTIFICATION: (Without mentioning the student/teacher framing of this prompt, explain why the STUDENT ANSWER is Correct or Incorrect, identify potential sources of bias in the QUESTION, and identify potential sources of bias in the TRUE ANSWER. Use one or two sentences maximum. Keep the answer as concise as possible.)
|
||||
"""
|
||||
|
||||
GRADE_ANSWER_PROMPT_BIAS_CHECK = PromptTemplate(input_variables=["query", "result", "answer"], template=template)
|
||||
|
||||
template = """You are assessing a submitted student answer to a question relative to the true answer based on the provided criteria:
|
||||
|
||||
***
|
||||
QUESTION: {query}
|
||||
***
|
||||
STUDENT ANSWER: {result}
|
||||
***
|
||||
TRUE ANSWER: {answer}
|
||||
***
|
||||
Criteria:
|
||||
relevance: Is the submission referring to a real quote from the text?"
|
||||
conciseness: Is the answer concise and to the point?"
|
||||
correct: Is the answer correct?"
|
||||
***
|
||||
Does the submission meet the criterion? First, write out in a step by step manner your reasoning about the criterion to be sure that your conclusion is correct. Avoid simply stating the correct answers at the outset. Then print "Correct" or "Incorrect" (without quotes or punctuation) on its own line corresponding to the correct answer.
|
||||
Reasoning:
|
||||
"""
|
||||
|
||||
GRADE_ANSWER_PROMPT_OPENAI = PromptTemplate(input_variables=["query", "result", "answer"], template=template)
|
||||
|
||||
template = """
|
||||
You are a grader trying to determine if a set of retrieved documents will help a student answer a question. \n
|
||||
|
||||
Here is the question: \n
|
||||
{query}
|
||||
|
||||
Here are the documents retrieved to answer question: \n
|
||||
{result}
|
||||
|
||||
Here is the correct answer to the question: \n
|
||||
{answer}
|
||||
|
||||
Criteria:
|
||||
relevance: Do all of the documents contain information that will help the student arrive that the correct answer to the question?"
|
||||
|
||||
Your response should be as follows:
|
||||
|
||||
GRADE: (Correct or Incorrect, depending if all of the documents retrieved meet the criterion)
|
||||
(line break)
|
||||
JUSTIFICATION: (Write out in a step by step manner your reasoning about the criterion to be sure that your conclusion is correct. Use three sentences maximum. Keep the answer as concise as possible.)
|
||||
"""
|
||||
|
||||
GRADE_DOCS_PROMPT = PromptTemplate(input_variables=['result', 'answer', 'query'], template=template)
|
||||
|
||||
template = """You are an AI travel assistant for a company called Osito that answers questions using the provided context.
|
||||
|
||||
Your job is to help the customer decide on which cities might be good for an event they are planning, and then optionally to help them pick a shortlist of hotels within those cities to request proposals from.
|
||||
|
||||
If you need more information at any point, you can ask the customer follow up questions about where their attendees are coming from ("origin cities"), what vibe they are looking for, their budget, what hotel class they want (2-star through 5-star), etc.
|
||||
|
||||
Question:
|
||||
{question}
|
||||
|
||||
Here is some context that might be helpful:
|
||||
{context}
|
||||
|
||||
Answer:"""
|
||||
|
||||
QA_CHAIN_PROMPT_TRAVEL = PromptTemplate(input_variables=["context", "question"],template=template,)
|
||||
|
||||
template = """You are an assistant providing summary answers about the Lex Fridman podcast.
|
||||
|
||||
If the user asks a question about a specific episode, just answeer using information in the below context.
|
||||
|
||||
Be concise and truthful. Think step by step.
|
||||
|
||||
Question:
|
||||
{question}
|
||||
|
||||
Here is some context that might be helpful:
|
||||
{context}
|
||||
|
||||
Answer:"""
|
||||
|
||||
QA_CHAIN_PROMPT_LEX = PromptTemplate(input_variables=["context", "question"],template=template,)
|
||||
@@ -0,0 +1,7 @@
|
||||
langchain==0.0.164
|
||||
openai==0.27.0
|
||||
altair==4.2.2
|
||||
scikit-learn==1.2.1
|
||||
streamlit==1.21.0
|
||||
kor==0.9.2
|
||||
transformers==4.28.1
|
||||
@@ -0,0 +1,12 @@
|
||||
from langchain.chains.query_constructor.base import AttributeInfo
|
||||
|
||||
metadata_field_info=[
|
||||
|
||||
AttributeInfo(
|
||||
name="id",
|
||||
description="The ID of the episode",
|
||||
type="string",
|
||||
),
|
||||
]
|
||||
|
||||
document_content_description = "Information about Lex Fridman podcast episodes"
|
||||
Reference in New Issue
Block a user