From de76d20f12a725ace99f43ea1fcb1b971939016d Mon Sep 17 00:00:00 2001 From: "David S. Batista" Date: Tue, 18 Mar 2025 10:52:05 +0100 Subject: [PATCH] fix: updating end2end evaluation tests (#9053) * updating tests * fixing tests, default now is JSON object and no longer dataframe * cleaning up leftovers --- e2e/pipelines/test_evaluation_pipeline.py | 50 +++++++++++------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/e2e/pipelines/test_evaluation_pipeline.py b/e2e/pipelines/test_evaluation_pipeline.py index 1cd41fa4d8..32412f38cd 100644 --- a/e2e/pipelines/test_evaluation_pipeline.py +++ b/e2e/pipelines/test_evaluation_pipeline.py @@ -8,7 +8,7 @@ from typing import List import pytest from haystack import Document, Pipeline -from haystack.components.builders import AnswerBuilder, PromptBuilder +from haystack.components.builders import AnswerBuilder, ChatPromptBuilder from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder from haystack.components.evaluators import ( ContextRelevanceEvaluator, @@ -19,12 +19,13 @@ from haystack.components.evaluators import ( SASEvaluator, ) from haystack.components.evaluators.document_recall import RecallMode -from haystack.components.generators import OpenAIGenerator +from haystack.components.generators.chat import OpenAIChatGenerator from haystack.components.retrievers import InMemoryEmbeddingRetriever from haystack.components.writers import DocumentWriter from haystack.document_stores.in_memory import InMemoryDocumentStore from haystack.document_stores.types import DuplicatePolicy from haystack.evaluation import EvaluationRunResult +from haystack.dataclasses import ChatMessage EMBEDDINGS_MODEL = "sentence-transformers/all-MiniLM-L6-v2" @@ -44,28 +45,29 @@ def indexing_pipeline(documents: List[Document]): def rag_pipeline(document_store: InMemoryDocumentStore, top_k: int): # type: ignore """RAG pipeline""" - template = """ - You have to answer the following question based on the given context information only. + template = [ + ChatMessage.from_system( + text="You have to answer the following question based on the given context information only." + ), + ChatMessage.from_user( + text="""Context: + {% for document in documents %} + {{ document.content }} + {% endfor %} - Context: - {% for document in documents %} - {{ document.content }} - {% endfor %} - - Question: {{question}} - Answer: - """ + Question: {{question}}""" + ), + ] rag = Pipeline() rag.add_component("embedder", SentenceTransformersTextEmbedder(model=EMBEDDINGS_MODEL, progress_bar=False)) # type: ignore rag.add_component("retriever", InMemoryEmbeddingRetriever(document_store, top_k=top_k)) # type: ignore - rag.add_component("prompt_builder", PromptBuilder(template=template)) # type: ignore - rag.add_component("generator", OpenAIGenerator(model="gpt-4o-mini")) # type: ignore + rag.add_component("prompt_builder", ChatPromptBuilder(template=template)) # type: ignore + rag.add_component("generator", OpenAIChatGenerator(model="gpt-4o-mini")) # type: ignore rag.add_component("answer_builder", AnswerBuilder()) # type: ignore rag.connect("embedder", "retriever.query_embedding") rag.connect("retriever", "prompt_builder.documents") rag.connect("prompt_builder", "generator") rag.connect("generator.replies", "answer_builder.replies") - rag.connect("generator.meta", "answer_builder.meta") rag.connect("retriever", "answer_builder.documents") return rag @@ -220,12 +222,12 @@ def test_evaluation_pipeline(samples_path): } results_a = built_input_for_results_eval(results_rag_a) evaluation_result_a = EvaluationRunResult(run_name="rag_pipeline_a", results=results_a, inputs=inputs_a) - df_score_report = evaluation_result_a.score_report() + aggregated_score_report_json = evaluation_result_a.aggregated_report() # assert the score report has all the metrics - assert len(df_score_report) == 7 - assert list(df_score_report.columns) == ["metrics", "score"] - assert list(df_score_report.metrics) == [ + assert len(aggregated_score_report_json["metrics"]) == 7 + assert list(aggregated_score_report_json.keys()) == ["metrics", "score"] + assert list(aggregated_score_report_json["metrics"]) == [ "Mean Reciprocal Rank", "Semantic Answer Similarity", "Faithfulness", @@ -236,8 +238,8 @@ def test_evaluation_pipeline(samples_path): ] # assert the evaluation result has all the metrics, inputs and questions - df = evaluation_result_a.to_pandas() - assert list(df.columns) == [ + detailed_report_json = evaluation_result_a.detailed_report() + assert list(detailed_report_json.keys()) == [ "question", "contexts", "answer", @@ -250,7 +252,6 @@ def test_evaluation_pipeline(samples_path): "Document Recall Multi Hit", "Contextual Relevance", ] - assert len(df) == 3 # running the RAG pipeline B rag_pipeline_b = rag_pipeline(doc_store, top_k=4) @@ -266,11 +267,10 @@ def test_evaluation_pipeline(samples_path): } results_b = built_input_for_results_eval(results_rag_b) evaluation_result_b = EvaluationRunResult(run_name="rag_pipeline_b", results=results_b, inputs=inputs_b) - df_comparative = evaluation_result_a.comparative_individual_scores_report(evaluation_result_b) + comparative_json = evaluation_result_a.comparative_detailed_report(evaluation_result_b) # assert the comparative score report has all the metrics, inputs and questions - assert len(df_comparative) == 3 - assert list(df_comparative.columns) == [ + assert list(comparative_json.keys()) == [ "question", "contexts", "answer",