mirror of
https://github.com/deepset-ai/haystack.git
synced 2026-10-12 02:12:50 +00:00
Remove eval end to end tests (#7093)
This commit is contained in:
Silvano Cerza
@@ -1,84 +0,0 @@
|
||||
from haystack import Pipeline
|
||||
from haystack.components.converters import PyPDFToDocument, TextFileToDocument
|
||||
from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder
|
||||
from haystack.components.preprocessors import DocumentCleaner, DocumentSplitter
|
||||
from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever
|
||||
from haystack.components.routers import FileTypeRouter
|
||||
from haystack.components.joiners import DocumentJoiner
|
||||
from haystack.components.writers import DocumentWriter
|
||||
from haystack.dataclasses import Document
|
||||
from haystack.document_stores.in_memory import InMemoryDocumentStore
|
||||
from haystack.evaluation.eval import eval
|
||||
|
||||
|
||||
def test_dense_doc_search_pipeline(samples_path):
|
||||
# Create the indexing pipeline
|
||||
indexing_pipeline = Pipeline()
|
||||
indexing_pipeline.add_component(
|
||||
instance=FileTypeRouter(mime_types=["text/plain", "application/pdf"]), name="file_type_router"
|
||||
)
|
||||
indexing_pipeline.add_component(instance=TextFileToDocument(), name="text_file_converter")
|
||||
indexing_pipeline.add_component(instance=PyPDFToDocument(), name="pdf_file_converter")
|
||||
indexing_pipeline.add_component(instance=DocumentJoiner(), name="joiner")
|
||||
indexing_pipeline.add_component(instance=DocumentCleaner(), name="cleaner")
|
||||
indexing_pipeline.add_component(
|
||||
instance=DocumentSplitter(split_by="sentence", split_length=250, split_overlap=30), name="splitter"
|
||||
)
|
||||
indexing_pipeline.add_component(
|
||||
instance=SentenceTransformersDocumentEmbedder(model="sentence-transformers/all-MiniLM-L6-v2"), name="embedder"
|
||||
)
|
||||
indexing_pipeline.add_component(instance=DocumentWriter(document_store=InMemoryDocumentStore()), name="writer")
|
||||
|
||||
indexing_pipeline.connect("file_type_router.text/plain", "text_file_converter.sources")
|
||||
indexing_pipeline.connect("file_type_router.application/pdf", "pdf_file_converter.sources")
|
||||
indexing_pipeline.connect("text_file_converter.documents", "joiner.documents")
|
||||
indexing_pipeline.connect("pdf_file_converter.documents", "joiner.documents")
|
||||
indexing_pipeline.connect("joiner.documents", "cleaner.documents")
|
||||
indexing_pipeline.connect("cleaner.documents", "splitter.documents")
|
||||
indexing_pipeline.connect("splitter.documents", "embedder.documents")
|
||||
indexing_pipeline.connect("embedder.documents", "writer.documents")
|
||||
|
||||
indexing_pipeline.run({"file_type_router": {"sources": list(samples_path.iterdir())}})
|
||||
filled_document_store = indexing_pipeline.get_component("writer").document_store
|
||||
|
||||
# Create the querying pipeline
|
||||
query_pipeline = Pipeline()
|
||||
query_pipeline.add_component(
|
||||
instance=SentenceTransformersTextEmbedder(model="sentence-transformers/all-MiniLM-L6-v2"), name="text_embedder"
|
||||
)
|
||||
query_pipeline.add_component(
|
||||
instance=InMemoryEmbeddingRetriever(document_store=filled_document_store, top_k=20), name="embedding_retriever"
|
||||
)
|
||||
query_pipeline.connect("text_embedder", "embedding_retriever")
|
||||
|
||||
inputs = [{"text_embedder": {"text": "Who lives in Rome?"}}]
|
||||
expected_outputs = [
|
||||
{
|
||||
"embedding_retriever": {
|
||||
"documents": [
|
||||
Document(
|
||||
id="d219162e5d0b8e5eab901e32ce0d9c12d24e5ea26a92780442fcfa560eb0b7d6",
|
||||
content="My name is Giorgio and I live in Rome.",
|
||||
meta={
|
||||
"file_path": "/home/ashwin/data_science/0ashwin/opensource/haystack/e2e/samples/doc_1.txt",
|
||||
"source_id": "0366ae1654f4573564e29184cd4a2232286a93f4f25d6790ce703ae7d4d7d63c",
|
||||
},
|
||||
score=0.627746287158654,
|
||||
),
|
||||
Document(
|
||||
id="2dcf2bc0307ba21fbb7e97a307d987a05297e577a44f170081acdbab9fc4b95f",
|
||||
content="A sample PDF file History and standardizationFormat (PDF) Adobe Systems made the PDF specification ava...",
|
||||
meta={"source_id": "ec1ac6c430ecd0cc74ae56f3e2d84f93fef3f5393de6901fe8aa01e494ebcdbe"},
|
||||
score=-0.060180130727963355,
|
||||
),
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
eval_result = eval(query_pipeline, inputs=inputs, expected_outputs=expected_outputs)
|
||||
|
||||
assert eval_result.inputs == inputs
|
||||
assert eval_result.expected_outputs == expected_outputs
|
||||
assert len(eval_result.outputs) == len(expected_outputs) == len(inputs)
|
||||
assert eval_result.runnable.to_dict() == query_pipeline.to_dict()
|
||||
@@ -1,166 +0,0 @@
|
||||
import json
|
||||
import pytest
|
||||
|
||||
from haystack import Pipeline
|
||||
from haystack.components.readers import ExtractiveReader
|
||||
from haystack.components.retrievers.in_memory import InMemoryBM25Retriever
|
||||
from haystack.dataclasses import Document, ExtractedAnswer
|
||||
from haystack.document_stores.in_memory import InMemoryDocumentStore
|
||||
from haystack.evaluation.eval import eval
|
||||
from haystack.evaluation.metrics import Metric
|
||||
|
||||
|
||||
def test_extractive_qa_pipeline(tmp_path):
|
||||
# Create the pipeline
|
||||
qa_pipeline = Pipeline()
|
||||
qa_pipeline.add_component(instance=InMemoryBM25Retriever(document_store=InMemoryDocumentStore()), name="retriever")
|
||||
qa_pipeline.add_component(instance=ExtractiveReader(model="deepset/tinyroberta-squad2"), name="reader")
|
||||
qa_pipeline.connect("retriever", "reader")
|
||||
|
||||
# Populate the document store
|
||||
documents = [
|
||||
Document(content="My name is Jean and I live in Paris."),
|
||||
Document(content="My name is Mark and I live in Berlin."),
|
||||
Document(content="My name is Giorgio and I live in Rome."),
|
||||
]
|
||||
qa_pipeline.get_component("retriever").document_store.write_documents(documents)
|
||||
|
||||
# Query and assert
|
||||
questions = ["Who lives in Paris?", "Who lives in Berlin?", "Who lives in Rome?"]
|
||||
inputs = [{"retriever": {"query": question}, "reader": {"query": question, "top_k": 1}} for question in questions]
|
||||
expected_outputs = [
|
||||
{
|
||||
"reader": {
|
||||
"answers": [
|
||||
ExtractedAnswer(
|
||||
query="Who lives in Paris?",
|
||||
score=0.7713339924812317,
|
||||
data="Jean and I",
|
||||
document=Document(content="My name is Jean and I live in Paris.", score=0.33144005810482535),
|
||||
context=None,
|
||||
document_offset=ExtractedAnswer.Span(start=11, end=21),
|
||||
context_offset=None,
|
||||
meta={},
|
||||
),
|
||||
ExtractedAnswer(
|
||||
query="Who lives in Paris?",
|
||||
score=0.2286660075187683,
|
||||
data=None,
|
||||
document=None,
|
||||
context=None,
|
||||
document_offset=None,
|
||||
context_offset=None,
|
||||
meta={},
|
||||
),
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"reader": {
|
||||
"answers": [
|
||||
ExtractedAnswer(
|
||||
query="Who lives in Berlin?",
|
||||
score=0.7047999501228333,
|
||||
data="Mark and I",
|
||||
document=Document(content="My name is Mark and I live in Berlin.", score=0.33144005810482535),
|
||||
context=None,
|
||||
document_offset=ExtractedAnswer.Span(start=11, end=21),
|
||||
context_offset=None,
|
||||
meta={},
|
||||
),
|
||||
ExtractedAnswer(
|
||||
query="Who lives in Berlin?",
|
||||
score=0.29520004987716675,
|
||||
data=None,
|
||||
document=None,
|
||||
context=None,
|
||||
document_offset=None,
|
||||
context_offset=None,
|
||||
meta={},
|
||||
),
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"reader": {
|
||||
"answers": [
|
||||
ExtractedAnswer(
|
||||
query="Who lives in Rome?",
|
||||
score=0.7661304473876953,
|
||||
data="Giorgio and I",
|
||||
document=Document(content="My name is Giorgio and I live in Rome.", score=0.33144005810482535),
|
||||
context=None,
|
||||
document_offset=ExtractedAnswer.Span(start=11, end=24),
|
||||
context_offset=None,
|
||||
meta={},
|
||||
),
|
||||
ExtractedAnswer(
|
||||
query="Who lives in Rome?",
|
||||
score=0.2338695526123047,
|
||||
data=None,
|
||||
document=None,
|
||||
context=None,
|
||||
document_offset=None,
|
||||
context_offset=None,
|
||||
meta={},
|
||||
),
|
||||
]
|
||||
}
|
||||
},
|
||||
]
|
||||
|
||||
eval_result = eval(qa_pipeline, inputs=inputs, expected_outputs=expected_outputs)
|
||||
|
||||
assert eval_result.inputs == inputs
|
||||
assert eval_result.expected_outputs == expected_outputs
|
||||
assert len(eval_result.outputs) == len(expected_outputs) == len(inputs)
|
||||
assert eval_result.runnable.to_dict() == qa_pipeline.to_dict()
|
||||
|
||||
# Test Exact Match
|
||||
em_default = eval_result.calculate_metrics(Metric.EM, output_key="answers")
|
||||
em_custom_parameters = eval_result.calculate_metrics(
|
||||
Metric.EM, output_key="answers", ignore_case=True, ignore_punctuation=True, ignore_numbers=True
|
||||
)
|
||||
# Save EM metric results to json
|
||||
em_default.save(tmp_path / "exact_match_score.json")
|
||||
|
||||
assert em_default["exact_match"] == 1.0
|
||||
assert em_custom_parameters["exact_match"] == 1.0
|
||||
with open(tmp_path / "exact_match_score.json", "r") as f:
|
||||
assert em_default == json.load(f)
|
||||
|
||||
# Test F1
|
||||
f1_default = eval_result.calculate_metrics(Metric.F1, output_key="answers")
|
||||
f1_custom_parameters = eval_result.calculate_metrics(
|
||||
Metric.F1, output_key="answers", ignore_case=True, ignore_punctuation=True, ignore_numbers=True
|
||||
)
|
||||
# Save F1 metric results to json
|
||||
f1_default.save(tmp_path / "f1_score.json")
|
||||
|
||||
assert f1_default["f1"] == 1.0
|
||||
assert f1_custom_parameters["f1"] == 1.0
|
||||
with open(tmp_path / "f1_score.json", "r") as f:
|
||||
assert f1_default == json.load(f)
|
||||
|
||||
# Test SAS
|
||||
sas_default = eval_result.calculate_metrics(
|
||||
Metric.SAS, output_key="answers", model="sentence-transformers/paraphrase-multilingual-mpnet-base-v2"
|
||||
)
|
||||
sas_custom_parameters = eval_result.calculate_metrics(
|
||||
Metric.SAS,
|
||||
output_key="answers",
|
||||
ignore_case=True,
|
||||
ignore_punctuation=True,
|
||||
ignore_numbers=True,
|
||||
model="cross-encoder/ms-marco-MiniLM-L-6-v2",
|
||||
)
|
||||
# Save SAS metric results to json
|
||||
sas_default.save(tmp_path / "sas_score.json")
|
||||
|
||||
assert sas_default["sas"] == pytest.approx(1.0)
|
||||
assert sas_default["scores"] == pytest.approx([1.0, 1.0, 1.0])
|
||||
assert sas_custom_parameters["sas"] == pytest.approx(0.9996823, abs=1e-5)
|
||||
assert sas_custom_parameters["scores"] == pytest.approx([0.999672, 0.999608, 0.999767])
|
||||
|
||||
with open(tmp_path / "sas_score.json", "r") as f:
|
||||
assert sas_default == json.load(f)
|
||||
@@ -1,99 +0,0 @@
|
||||
from haystack import Document, Pipeline
|
||||
from haystack.components.embedders import SentenceTransformersTextEmbedder
|
||||
from haystack.components.rankers import TransformersSimilarityRanker
|
||||
from haystack.components.retrievers.in_memory import InMemoryBM25Retriever, InMemoryEmbeddingRetriever
|
||||
from haystack.components.joiners.document_joiner import DocumentJoiner
|
||||
from haystack.document_stores.in_memory import InMemoryDocumentStore
|
||||
from haystack.evaluation.eval import eval
|
||||
|
||||
|
||||
def test_hybrid_doc_search_pipeline():
|
||||
# Create the pipeline
|
||||
document_store = InMemoryDocumentStore()
|
||||
hybrid_pipeline = Pipeline()
|
||||
hybrid_pipeline.add_component(instance=InMemoryBM25Retriever(document_store=document_store), name="bm25_retriever")
|
||||
hybrid_pipeline.add_component(
|
||||
instance=SentenceTransformersTextEmbedder(model="sentence-transformers/all-MiniLM-L6-v2"), name="text_embedder"
|
||||
)
|
||||
hybrid_pipeline.add_component(
|
||||
instance=InMemoryEmbeddingRetriever(document_store=document_store), name="embedding_retriever"
|
||||
)
|
||||
hybrid_pipeline.add_component(instance=DocumentJoiner(), name="joiner")
|
||||
hybrid_pipeline.add_component(instance=TransformersSimilarityRanker(top_k=2), name="ranker")
|
||||
|
||||
hybrid_pipeline.connect("bm25_retriever", "joiner")
|
||||
hybrid_pipeline.connect("text_embedder", "embedding_retriever")
|
||||
hybrid_pipeline.connect("embedding_retriever", "joiner")
|
||||
hybrid_pipeline.connect("joiner", "ranker")
|
||||
|
||||
# Populate the document store
|
||||
documents = [
|
||||
Document(content="My name is Jean and I live in Paris."),
|
||||
Document(content="My name is Mark and I live in Berlin."),
|
||||
Document(content="My name is Mario and I live in the capital of Italy."),
|
||||
Document(content="My name is Giorgio and I live in Rome."),
|
||||
]
|
||||
hybrid_pipeline.get_component("bm25_retriever").document_store.write_documents(documents)
|
||||
|
||||
questions = ["Who lives in Paris?", "Who lives in Berlin?", "Who lives in Rome?"]
|
||||
inputs = [
|
||||
{"bm25_retriever": {"query": question}, "text_embedder": {"text": question}, "ranker": {"query": question}}
|
||||
for question in questions
|
||||
]
|
||||
expected_outputs = [
|
||||
{
|
||||
"ranker": {
|
||||
"documents": [
|
||||
Document(
|
||||
id="6c90b78ad94e4e634e2a067b5fe2d26d4ce95405ec222cbaefaeb09ab4dce81e",
|
||||
content="My name is Jean and I live in Paris.",
|
||||
score=2.2277960777282715,
|
||||
),
|
||||
Document(
|
||||
id="10a183e965c2e107e20507c717f16559c58a8ba4bc7c577ea8dc32a8d6ca7a20",
|
||||
content="My name is Mark and I live in Berlin.",
|
||||
score=-7.304897308349609,
|
||||
),
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"ranker": {
|
||||
"documents": [
|
||||
Document(
|
||||
id="10a183e965c2e107e20507c717f16559c58a8ba4bc7c577ea8dc32a8d6ca7a20",
|
||||
content="My name is Mark and I live in Berlin.",
|
||||
score=3.694173812866211,
|
||||
),
|
||||
Document(
|
||||
id="f7533b5c6c968680d0ef8e38f366d4e68b7ac0d7238f1b1b366d15cb9c33efd8",
|
||||
content="My name is Mario and I live in the capital of Italy.",
|
||||
score=-9.008655548095703,
|
||||
),
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"ranker": {
|
||||
"documents": [
|
||||
Document(
|
||||
id="fb0f1efe94b3c78aa1c4e5a17a5ef8270f70e89d36a3665c8362675e8a769a27",
|
||||
content="My name is Giorgio and I live in Rome.",
|
||||
score=3.487802028656006,
|
||||
),
|
||||
Document(
|
||||
id="f7533b5c6c968680d0ef8e38f366d4e68b7ac0d7238f1b1b366d15cb9c33efd8",
|
||||
content="My name is Mario and I live in the capital of Italy.",
|
||||
score=-2.873128890991211,
|
||||
),
|
||||
]
|
||||
}
|
||||
},
|
||||
]
|
||||
|
||||
eval_result = eval(hybrid_pipeline, inputs=inputs, expected_outputs=expected_outputs)
|
||||
|
||||
assert eval_result.inputs == inputs
|
||||
assert eval_result.expected_outputs == expected_outputs
|
||||
assert len(eval_result.outputs) == len(expected_outputs) == len(inputs)
|
||||
assert eval_result.runnable.to_dict() == hybrid_pipeline.to_dict()
|
||||
@@ -1,336 +0,0 @@
|
||||
import json
|
||||
import pytest
|
||||
|
||||
from haystack import Pipeline
|
||||
from haystack.components.builders.answer_builder import AnswerBuilder
|
||||
from haystack.components.builders.prompt_builder import PromptBuilder
|
||||
from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder
|
||||
from haystack.components.generators import HuggingFaceLocalGenerator
|
||||
from haystack.components.retrievers.in_memory import InMemoryBM25Retriever, InMemoryEmbeddingRetriever
|
||||
from haystack.components.writers import DocumentWriter
|
||||
from haystack.dataclasses import Document, GeneratedAnswer
|
||||
from haystack.document_stores.in_memory import InMemoryDocumentStore
|
||||
from haystack.evaluation.eval import eval
|
||||
from haystack.evaluation.metrics import Metric
|
||||
|
||||
|
||||
def test_bm25_rag_pipeline(tmp_path):
|
||||
prompt_template = """
|
||||
Given these documents, answer the question.\nDocuments:
|
||||
{% for doc in documents %}
|
||||
{{ doc.content }}
|
||||
{% endfor %}
|
||||
|
||||
\nQuestion: {{question}}
|
||||
\nAnswer:
|
||||
"""
|
||||
rag_pipeline = Pipeline()
|
||||
rag_pipeline.add_component(instance=InMemoryBM25Retriever(document_store=InMemoryDocumentStore()), name="retriever")
|
||||
rag_pipeline.add_component(instance=PromptBuilder(template=prompt_template), name="prompt_builder")
|
||||
rag_pipeline.add_component(
|
||||
instance=HuggingFaceLocalGenerator(
|
||||
model="google/flan-t5-small",
|
||||
task="text2text-generation",
|
||||
generation_kwargs={"max_new_tokens": 100, "temperature": 0.5, "do_sample": True},
|
||||
),
|
||||
name="llm",
|
||||
)
|
||||
rag_pipeline.add_component(instance=AnswerBuilder(), name="answer_builder")
|
||||
rag_pipeline.connect("retriever", "prompt_builder.documents")
|
||||
rag_pipeline.connect("prompt_builder", "llm")
|
||||
rag_pipeline.connect("llm.replies", "answer_builder.replies")
|
||||
rag_pipeline.connect("retriever", "answer_builder.documents")
|
||||
|
||||
# Populate the document store
|
||||
documents = [
|
||||
Document(content="My name is Jean and I live in Paris."),
|
||||
Document(content="My name is Mark and I live in Berlin."),
|
||||
Document(content="My name is Giorgio and I live in Rome."),
|
||||
]
|
||||
rag_pipeline.get_component("retriever").document_store.write_documents(documents)
|
||||
|
||||
questions = ["Who lives in Paris?", "Who lives in Berlin?", "Who lives in Rome?"]
|
||||
inputs = [
|
||||
{
|
||||
"retriever": {"query": question},
|
||||
"prompt_builder": {"question": question},
|
||||
"answer_builder": {"query": question},
|
||||
}
|
||||
for question in questions
|
||||
]
|
||||
|
||||
expected_outputs = [
|
||||
{
|
||||
"answer_builder": {
|
||||
"answers": [
|
||||
GeneratedAnswer(
|
||||
data="Jean",
|
||||
query="Who lives in Paris?",
|
||||
documents=[
|
||||
Document(content="My name is Jean and I live in Paris.", score=0.33144005810482535),
|
||||
Document(content="My name is Giorgio and I live in Rome.", score=-0.17938556566116537),
|
||||
Document(content="My name is Mark and I live in Berlin.", score=-0.17938556566116537),
|
||||
],
|
||||
meta={},
|
||||
)
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"answer_builder": {
|
||||
"answers": [
|
||||
GeneratedAnswer(
|
||||
data="Mark",
|
||||
query="Who lives in Berlin?",
|
||||
documents=[
|
||||
Document(content="My name is Mark and I live in Berlin.", score=0.33144005810482535),
|
||||
Document(content="My name is Giorgio and I live in Rome.", score=-0.17938556566116537),
|
||||
Document(content="My name is Jean and I live in Paris.", score=-0.17938556566116537),
|
||||
],
|
||||
meta={},
|
||||
)
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"answer_builder": {
|
||||
"answers": [
|
||||
GeneratedAnswer(
|
||||
data="Giorgio",
|
||||
query="Who lives in Rome?",
|
||||
documents=[
|
||||
Document(content="My name is Giorgio and I live in Rome.", score=0.33144005810482535),
|
||||
Document(content="My name is Mark and I live in Berlin.", score=-0.17938556566116537),
|
||||
Document(content="My name is Jean and I live in Paris.", score=-0.17938556566116537),
|
||||
],
|
||||
meta={},
|
||||
)
|
||||
]
|
||||
}
|
||||
},
|
||||
]
|
||||
|
||||
eval_result = eval(rag_pipeline, inputs=inputs, expected_outputs=expected_outputs)
|
||||
|
||||
assert eval_result.inputs == inputs
|
||||
assert eval_result.expected_outputs == expected_outputs
|
||||
assert len(eval_result.outputs) == len(expected_outputs) == len(inputs)
|
||||
assert eval_result.runnable.to_dict() == rag_pipeline.to_dict()
|
||||
|
||||
# Test Exact Match
|
||||
em_default = eval_result.calculate_metrics(Metric.EM, output_key="answers")
|
||||
em_custom_parameters = eval_result.calculate_metrics(
|
||||
Metric.EM, output_key="answers", ignore_case=True, ignore_punctuation=True, ignore_numbers=True
|
||||
)
|
||||
# Save EM metric results to json
|
||||
em_default.save(tmp_path / "exact_match_score.json")
|
||||
|
||||
assert em_default["exact_match"] == 1.0
|
||||
assert em_custom_parameters["exact_match"] == 1.0
|
||||
with open(tmp_path / "exact_match_score.json", "r") as f:
|
||||
assert em_default == json.load(f)
|
||||
|
||||
# Test F1
|
||||
f1_default = eval_result.calculate_metrics(Metric.F1, output_key="answers")
|
||||
f1_custom_parameters = eval_result.calculate_metrics(
|
||||
Metric.F1, output_key="answers", ignore_case=True, ignore_punctuation=True, ignore_numbers=True
|
||||
)
|
||||
# Save F1 metric results to json
|
||||
f1_default.save(tmp_path / "f1_score.json")
|
||||
|
||||
assert f1_default["f1"] == 1.0
|
||||
assert f1_custom_parameters["f1"] == 1.0
|
||||
with open(tmp_path / "f1_score.json", "r") as f:
|
||||
assert f1_default == json.load(f)
|
||||
|
||||
# Test SAS
|
||||
sas_default = eval_result.calculate_metrics(
|
||||
Metric.SAS, output_key="answers", model="sentence-transformers/paraphrase-multilingual-mpnet-base-v2"
|
||||
)
|
||||
sas_custom_parameters = eval_result.calculate_metrics(
|
||||
Metric.SAS,
|
||||
output_key="answers",
|
||||
ignore_case=True,
|
||||
ignore_punctuation=True,
|
||||
ignore_numbers=True,
|
||||
model="cross-encoder/ms-marco-MiniLM-L-6-v2",
|
||||
)
|
||||
# Save SAS metric results to json
|
||||
sas_default.save(tmp_path / "sas_score.json")
|
||||
|
||||
assert sas_default["sas"] == pytest.approx(1.0)
|
||||
assert sas_default["scores"] == pytest.approx([1.0, 1.0, 1.0])
|
||||
assert sas_custom_parameters["sas"] == pytest.approx(0.9769593, abs=1e-5)
|
||||
assert sas_custom_parameters["scores"] == pytest.approx([0.975823, 0.957218, 0.997837], abs=1e-5)
|
||||
|
||||
with open(tmp_path / "sas_score.json", "r") as f:
|
||||
assert sas_default == json.load(f)
|
||||
|
||||
|
||||
def test_embedding_retrieval_rag_pipeline(tmp_path):
|
||||
# Create the RAG pipeline
|
||||
prompt_template = """
|
||||
Given these documents, answer the question.\nDocuments:
|
||||
{% for doc in documents %}
|
||||
{{ doc.content }}
|
||||
{% endfor %}
|
||||
|
||||
\nQuestion: {{question}}
|
||||
\nAnswer:
|
||||
"""
|
||||
rag_pipeline = Pipeline()
|
||||
rag_pipeline.add_component(
|
||||
instance=SentenceTransformersTextEmbedder(model="sentence-transformers/all-MiniLM-L6-v2"), name="text_embedder"
|
||||
)
|
||||
rag_pipeline.add_component(
|
||||
instance=InMemoryEmbeddingRetriever(document_store=InMemoryDocumentStore()), name="retriever"
|
||||
)
|
||||
rag_pipeline.add_component(instance=PromptBuilder(template=prompt_template), name="prompt_builder")
|
||||
rag_pipeline.add_component(
|
||||
instance=HuggingFaceLocalGenerator(
|
||||
model="google/flan-t5-small",
|
||||
task="text2text-generation",
|
||||
generation_kwargs={"max_new_tokens": 100, "temperature": 0.5, "do_sample": True},
|
||||
),
|
||||
name="llm",
|
||||
)
|
||||
rag_pipeline.add_component(instance=AnswerBuilder(), name="answer_builder")
|
||||
rag_pipeline.connect("text_embedder", "retriever")
|
||||
rag_pipeline.connect("retriever", "prompt_builder.documents")
|
||||
rag_pipeline.connect("prompt_builder", "llm")
|
||||
rag_pipeline.connect("llm.replies", "answer_builder.replies")
|
||||
rag_pipeline.connect("retriever", "answer_builder.documents")
|
||||
|
||||
# Populate the document store
|
||||
documents = [
|
||||
Document(content="My name is Jean and I live in Paris."),
|
||||
Document(content="My name is Mark and I live in Berlin."),
|
||||
Document(content="My name is Giorgio and I live in Rome."),
|
||||
]
|
||||
document_store = rag_pipeline.get_component("retriever").document_store
|
||||
indexing_pipeline = Pipeline()
|
||||
indexing_pipeline.add_component(
|
||||
instance=SentenceTransformersDocumentEmbedder(model="sentence-transformers/all-MiniLM-L6-v2"),
|
||||
name="document_embedder",
|
||||
)
|
||||
indexing_pipeline.add_component(instance=DocumentWriter(document_store=document_store), name="document_writer")
|
||||
indexing_pipeline.connect("document_embedder", "document_writer")
|
||||
indexing_pipeline.run({"document_embedder": {"documents": documents}})
|
||||
|
||||
# Query and assert
|
||||
questions = ["Who lives in Paris?", "Who lives in Berlin?", "Who lives in Rome?"]
|
||||
inputs = [
|
||||
{
|
||||
"prompt_builder": {"question": question},
|
||||
"text_embedder": {"text": question},
|
||||
"answer_builder": {"query": question},
|
||||
}
|
||||
for question in questions
|
||||
]
|
||||
|
||||
expected_outputs = [
|
||||
{
|
||||
"answer_builder": {
|
||||
"answers": [
|
||||
GeneratedAnswer(
|
||||
data="Jean",
|
||||
query="Who lives in Paris?",
|
||||
documents=[
|
||||
Document(content="My name is Jean and I live in Paris.", score=0.33144005810482535),
|
||||
Document(content="My name is Giorgio and I live in Rome.", score=-0.17938556566116537),
|
||||
Document(content="My name is Mark and I live in Berlin.", score=-0.17938556566116537),
|
||||
],
|
||||
meta={},
|
||||
)
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"answer_builder": {
|
||||
"answers": [
|
||||
GeneratedAnswer(
|
||||
data="Mark",
|
||||
query="Who lives in Berlin?",
|
||||
documents=[
|
||||
Document(content="My name is Mark and I live in Berlin.", score=0.33144005810482535),
|
||||
Document(content="My name is Giorgio and I live in Rome.", score=-0.17938556566116537),
|
||||
Document(content="My name is Jean and I live in Paris.", score=-0.17938556566116537),
|
||||
],
|
||||
meta={},
|
||||
)
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"answer_builder": {
|
||||
"answers": [
|
||||
GeneratedAnswer(
|
||||
data="Giorgio",
|
||||
query="Who lives in Rome?",
|
||||
documents=[
|
||||
Document(content="My name is Giorgio and I live in Rome.", score=0.33144005810482535),
|
||||
Document(content="My name is Mark and I live in Berlin.", score=-0.17938556566116537),
|
||||
Document(content="My name is Jean and I live in Paris.", score=-0.17938556566116537),
|
||||
],
|
||||
meta={},
|
||||
)
|
||||
]
|
||||
}
|
||||
},
|
||||
]
|
||||
|
||||
eval_result = eval(rag_pipeline, inputs=inputs, expected_outputs=expected_outputs)
|
||||
|
||||
assert eval_result.inputs == inputs
|
||||
assert eval_result.expected_outputs == expected_outputs
|
||||
assert len(eval_result.outputs) == len(expected_outputs) == len(inputs)
|
||||
assert eval_result.runnable.to_dict() == rag_pipeline.to_dict()
|
||||
|
||||
# Test Exact Match
|
||||
em_default = eval_result.calculate_metrics(Metric.EM, output_key="answers")
|
||||
em_custom_parameters = eval_result.calculate_metrics(
|
||||
Metric.EM, output_key="answers", ignore_case=True, ignore_punctuation=True, ignore_numbers=True
|
||||
)
|
||||
# Save EM metric results to json
|
||||
em_default.save(tmp_path / "exact_match_score.json")
|
||||
|
||||
assert em_default["exact_match"] == 1.0
|
||||
assert em_custom_parameters["exact_match"] == 1.0
|
||||
with open(tmp_path / "exact_match_score.json", "r") as f:
|
||||
assert em_default == json.load(f)
|
||||
|
||||
# Test F1
|
||||
f1_default = eval_result.calculate_metrics(Metric.F1, output_key="answers")
|
||||
f1_custom_parameters = eval_result.calculate_metrics(
|
||||
Metric.F1, output_key="answers", ignore_case=True, ignore_punctuation=True, ignore_numbers=True
|
||||
)
|
||||
# Save F1 metric results to json
|
||||
f1_default.save(tmp_path / "f1_score.json")
|
||||
|
||||
assert f1_default["f1"] == 1.0
|
||||
assert f1_custom_parameters["f1"] == 1.0
|
||||
with open(tmp_path / "f1_score.json", "r") as f:
|
||||
assert f1_default == json.load(f)
|
||||
|
||||
# Test SAS
|
||||
sas_default = eval_result.calculate_metrics(
|
||||
Metric.SAS, output_key="answers", model="sentence-transformers/paraphrase-multilingual-mpnet-base-v2"
|
||||
)
|
||||
sas_custom_parameters = eval_result.calculate_metrics(
|
||||
Metric.SAS,
|
||||
output_key="answers",
|
||||
ignore_case=True,
|
||||
ignore_punctuation=True,
|
||||
ignore_numbers=True,
|
||||
model="cross-encoder/ms-marco-MiniLM-L-6-v2",
|
||||
)
|
||||
# Save SAS metric results to json
|
||||
sas_default.save(tmp_path / "sas_score.json")
|
||||
|
||||
assert sas_default["sas"] == pytest.approx(1.0)
|
||||
assert sas_default["scores"] == pytest.approx([1.0, 1.0, 1.0])
|
||||
assert sas_custom_parameters["sas"] == pytest.approx(0.9769593, abs=1e-5)
|
||||
assert sas_custom_parameters["scores"] == pytest.approx([0.975823, 0.957218, 0.997837], abs=1e-5)
|
||||
|
||||
with open(tmp_path / "sas_score.json", "r") as f:
|
||||
assert sas_default == json.load(f)
|
||||
Reference in New Issue
Block a user