Coverage for haystack/components/evaluators/context_relevance.py: 100%
50 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-12 09:42 +0000
1# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
2#
3# SPDX-License-Identifier: Apache-2.0
5import math
6from statistics import mean
7from typing import Any
9from haystack import component, default_from_dict, default_to_dict, logging
10from haystack.components.evaluators.llm_evaluator import LLMEvaluator
11from haystack.components.generators.chat.types import ChatGenerator
12from haystack.core.serialization import component_to_dict
13from haystack.utils import deserialize_chatgenerator_inplace
15logger = logging.getLogger(__name__)
17# Private global variable for default examples to include in the prompt if the user does not provide any examples
18_DEFAULT_EXAMPLES = [
19 {
20 "inputs": {
21 "questions": "What is the capital of Germany?",
22 "contexts": ["Berlin is the capital of Germany. Berlin and was founded in 1244."],
23 },
24 "outputs": {"relevant_statements": ["Berlin is the capital of Germany."]},
25 },
26 {
27 "inputs": {
28 "questions": "What is the capital of France?",
29 "contexts": [
30 "Berlin is the capital of Germany and was founded in 1244.",
31 "Europe is a continent with 44 countries.",
32 "Madrid is the capital of Spain.",
33 ],
34 },
35 "outputs": {"relevant_statements": []},
36 },
37 {
38 "inputs": {"questions": "What is the capital of Italy?", "contexts": ["Rome is the capital of Italy."]},
39 "outputs": {"relevant_statements": ["Rome is the capital of Italy."]},
40 },
41]
44@component
45class ContextRelevanceEvaluator(LLMEvaluator):
46 """
47 Evaluator that checks if a provided context is relevant to the question.
49 An LLM breaks up a context into multiple statements and checks whether each statement
50 is relevant for answering a question.
51 The score for each context is either binary score of 1 or 0, where 1 indicates that the context is relevant
52 to the question and 0 indicates that the context is not relevant.
53 The evaluator also provides the relevant statements from the context and an average score over all the provided
54 input questions contexts pairs.
56 Usage example:
57 ```python
58 from haystack.components.evaluators import ContextRelevanceEvaluator
60 questions = ["Who created the Python language?", "Why does Java needs a JVM?", "Is C++ better than Python?"]
61 contexts = [
62 [(
63 "Python, created by Guido van Rossum in the late 1980s, is a high-level general-purpose programming "
64 "language. Its design philosophy emphasizes code readability, and its language constructs aim to help "
65 "programmers write clear, logical code for both small and large-scale software projects."
66 )],
67 [(
68 "Java is a high-level, class-based, object-oriented programming language that is designed to have as few "
69 "implementation dependencies as possible. The JVM has two primary functions: to allow Java programs to run"
70 "on any device or operating system (known as the 'write once, run anywhere' principle), and to manage and"
71 "optimize program memory."
72 )],
73 [(
74 "C++ is a general-purpose programming language created by Bjarne Stroustrup as an extension of the C "
75 "programming language."
76 )],
77 ]
79 evaluator = ContextRelevanceEvaluator()
80 result = evaluator.run(questions=questions, contexts=contexts)
81 print(result["score"])
82 # 0.67
83 print(result["individual_scores"])
84 # [1,1,0]
85 print(result["results"])
86 # [{
87 # 'relevant_statements': ['Python, created by Guido van Rossum in the late 1980s.'],
88 # 'score': 1.0,
89 # 'status': 'evaluated'
90 # },
91 # {
92 # 'relevant_statements': ['The JVM has two primary functions: to allow Java programs to run on any device or
93 # operating system (known as the "write once, run anywhere" principle), and to manage and
94 # optimize program memory'],
95 # 'score': 1.0,
96 # 'status': 'evaluated'
97 # },
98 # {
99 # 'relevant_statements': [],
100 # 'score': 0.0,
101 # 'status': 'evaluated'
102 # }]
103 ```
104 """
106 def __init__(
107 self,
108 examples: list[dict[str, Any]] | None = None,
109 progress_bar: bool = True,
110 raise_on_failure: bool = True,
111 chat_generator: ChatGenerator | None = None,
112 ) -> None:
113 """
114 Creates an instance of ContextRelevanceEvaluator.
116 If no LLM is specified using the `chat_generator` parameter, the component will use OpenAI in JSON mode.
118 :param examples:
119 Optional few-shot examples conforming to the expected input and output format of ContextRelevanceEvaluator.
120 Default examples will be used if none are provided.
121 Each example must be a dictionary with keys "inputs" and "outputs".
122 "inputs" must be a dictionary with keys "questions" and "contexts".
123 "outputs" must be a dictionary with "relevant_statements".
124 Expected format:
125 ```python
126 [{
127 "inputs": {
128 "questions": "What is the capital of Italy?", "contexts": ["Rome is the capital of Italy."],
129 },
130 "outputs": {
131 "relevant_statements": ["Rome is the capital of Italy."],
132 },
133 }]
134 ```
135 :param progress_bar:
136 Whether to show a progress bar during the evaluation.
137 :param raise_on_failure:
138 If True, the component raises an exception when the chat generator call fails or its response isn't valid
139 JSON with the expected keys. If False, it logs a warning and sets the result for that input to `None`.
140 :param chat_generator:
141 a ChatGenerator instance which represents the LLM.
142 In order for the component to work, the LLM should be configured to return a JSON object. For example,
143 when using the OpenAIChatGenerator, you should pass `{"response_format": {"type": "json_object"}}` in the
144 `generation_kwargs`.
145 """
147 self.instructions = (
148 "Please extract only sentences from the provided context which are absolutely relevant and "
149 "required to answer the following question. If no relevant sentences are found, or if you "
150 "believe the question cannot be answered from the given context, return an empty list, example: []"
151 )
152 self.inputs = [("questions", list[str]), ("contexts", list[list[str]])]
153 self.outputs = ["relevant_statements"]
154 self.examples = examples or _DEFAULT_EXAMPLES
156 super(ContextRelevanceEvaluator, self).__init__( # noqa: UP008
157 instructions=self.instructions,
158 inputs=self.inputs,
159 outputs=self.outputs,
160 examples=self.examples,
161 chat_generator=chat_generator,
162 raise_on_failure=raise_on_failure,
163 progress_bar=progress_bar,
164 )
166 @component.output_types(
167 score=float, individual_scores=list[float], results=list[dict[str, Any]], meta=list[dict[str, Any]] | None
168 )
169 def run(self, **inputs: Any) -> dict[str, Any]:
170 """
171 Run the LLM evaluator.
173 :param questions:
174 A list of questions.
175 :param contexts:
176 A list of lists of contexts. Each list of contexts corresponds to one question.
177 :returns:
178 A dictionary with the following outputs:
179 - `score`: Mean context relevance score over all the provided input questions.
180 - `individual_scores`: A list of context relevance scores for each input question.
181 - `results`: A list of dictionaries with `relevant_statements`, `score`, and `status` for each input
182 context. `status` is `evaluated` for valid results and `error` for failed evaluations.
183 """
184 result = super(ContextRelevanceEvaluator, self).run(**inputs) # noqa: UP008
185 # Post-process the raw results to calculate relevance metrics and scores
186 return self._postprocess_results(result)
188 @component.output_types(
189 score=float, individual_scores=list[float], results=list[dict[str, Any]], meta=list[dict[str, Any]] | None
190 )
191 async def run_async(self, **inputs: Any) -> dict[str, Any]:
192 """
193 Run the LLM evaluator asynchronously.
195 :param questions:
196 A list of questions.
197 :param contexts:
198 A list of lists of contexts. Each list of contexts corresponds to one question.
199 :returns:
200 A dictionary with the following outputs:
201 - `score`: Mean context relevance score over all the provided input questions.
202 - `individual_scores`: A list of context relevance scores for each input question.
203 - `results`: A list of dictionaries with `relevant_statements`, `score`, and `status` for each input
204 context. `status` is `evaluated` for valid results and `error` for failed evaluations.
205 """
206 result = await super(ContextRelevanceEvaluator, self).run_async(**inputs) # noqa: UP008
207 # Post-process the raw results to calculate relevance metrics and scores
208 return self._postprocess_results(result)
210 def _postprocess_results(self, result: dict[str, Any]) -> dict[str, Any]:
211 """
212 Post-processes raw LLM evaluator outputs to compute context relevance scores.
214 Calculates binary scores based on whether relevant statements were found,
215 averages the scores across all successful queries, and updates the result payload.
217 :param result:
218 The raw evaluation dictionary from the base LLM evaluator.
219 :returns:
220 The updated dictionary containing final scores and tracking metrics.
221 """
222 for idx, res in enumerate(result["results"]):
223 if res is None:
224 result["results"][idx] = {"relevant_statements": [], "score": float("nan"), "status": "error"}
225 continue
226 res["status"] = "evaluated"
227 if len(res["relevant_statements"]) > 0:
228 res["score"] = 1
229 else:
230 res["score"] = 0
232 # calculate average context relevance score over all queries
233 scores = [res["score"] for res in result["results"]]
234 valid_scores = [s for s in scores if not math.isnan(s)]
235 skipped = len(scores) - len(valid_scores)
236 if skipped:
237 logger.warning("{skipped} query(s) failed and were excluded from the score.", skipped=skipped)
238 result["score"] = mean(valid_scores) if valid_scores else float("nan")
239 result["individual_scores"] = scores # useful for the EvaluationRunResult
241 return result
243 def to_dict(self) -> dict[str, Any]:
244 """
245 Serialize this component to a dictionary.
247 :returns:
248 A dictionary with serialized data.
249 """
250 return default_to_dict(
251 self,
252 chat_generator=component_to_dict(obj=self._chat_generator, name="chat_generator"),
253 examples=self.examples,
254 progress_bar=self.progress_bar,
255 raise_on_failure=self.raise_on_failure,
256 )
258 @classmethod
259 def from_dict(cls, data: dict[str, Any]) -> "ContextRelevanceEvaluator":
260 """
261 Deserialize this component from a dictionary.
263 :param data:
264 The dictionary representation of this component.
265 :returns:
266 The deserialized component instance.
267 """
268 if data["init_parameters"].get("chat_generator"):
269 deserialize_chatgenerator_inplace(data["init_parameters"], key="chat_generator")
270 return default_from_dict(cls, data)