Coverage for haystack/components/evaluators/llm_evaluator.py: 99%
168 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-12 09:42 +0000
1# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
2#
3# SPDX-License-Identifier: Apache-2.0
5import asyncio
6import json
7from typing import Any
9from tqdm import tqdm
10from tqdm.asyncio import tqdm as async_tqdm
12from haystack import component, default_from_dict, default_to_dict, logging
13from haystack.components.builders import PromptBuilder
14from haystack.components.generators.chat.openai import OpenAIChatGenerator
15from haystack.components.generators.chat.types import ChatGenerator
16from haystack.components.generators.utils import _trace_chat_generator_run
17from haystack.core.serialization import component_to_dict
18from haystack.dataclasses.chat_message import ChatMessage
19from haystack.utils import deserialize_chatgenerator_inplace, deserialize_type, serialize_type
20from haystack.utils.misc import _parse_dict_from_json
22logger = logging.getLogger(__name__)
25@component
26class LLMEvaluator:
27 """
28 Uses an LLM to evaluate inputs based on a prompt containing instructions and examples.
30 The default API requires an OpenAI API key to be provided as an environment variable "OPENAI_API_KEY".
31 The inputs are lists that are user-defined depending on the desired metric.
32 The output is a dictionary with a key `results` containing a list of evaluation results.
33 Each result is a dictionary with user-defined keys and values of either 0 for FALSE or 1 for TRUE respectively.
35 Usage example:
36 ```python
37 from haystack.components.evaluators import LLMEvaluator
38 evaluator = LLMEvaluator(
39 instructions="Is this answer problematic for children?",
40 inputs=[("predicted_answers", list[str])],
41 outputs=["score"],
42 examples=[
43 {"inputs": {"predicted_answers": "Damn, this is straight outta hell!!!"}, "outputs": {"score": 1}},
44 {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}},
45 ],
46 )
47 predicted_answers = [
48 "Football is the most popular sport with around 4 billion followers worldwide",
49 "Python language was created by Guido van Rossum.",
50 ]
51 results = evaluator.run(predicted_answers=predicted_answers)
52 print(results)
53 # {'results': [{'score': 0}, {'score': 0}]}
54 ```
55 """
57 def __init__(
58 self,
59 instructions: str,
60 inputs: list[tuple[str, type[list]]],
61 outputs: list[str],
62 examples: list[dict[str, Any]],
63 progress_bar: bool = True,
64 *,
65 raise_on_failure: bool = True,
66 chat_generator: ChatGenerator | None = None,
67 ) -> None:
68 """
69 Creates an instance of LLMEvaluator.
71 If no LLM is specified using the `chat_generator` parameter, the component will use OpenAI in JSON mode.
73 :param instructions:
74 The prompt instructions to use for evaluation.
75 Should be a question about the inputs that can be answered with yes or no.
76 :param inputs:
77 The inputs that the component expects as incoming connections and that it evaluates.
78 Each input is a tuple of an input name and input type. Input types must be lists.
79 :param outputs:
80 Output names of the evaluation results. They correspond to keys in the output dictionary.
81 :param examples:
82 Few-shot examples conforming to the expected input and output format as defined in the `inputs` and
83 `outputs` parameters.
84 Each example is a dictionary with keys "inputs" and "outputs"
85 They contain the input and output as dictionaries respectively.
86 :param raise_on_failure:
87 If True, the component raises an exception when the chat generator call fails or its response isn't valid
88 JSON with the expected keys. If False, it logs a warning and sets the result for that input to `None`.
89 :param progress_bar:
90 Whether to show a progress bar during the evaluation.
91 :param chat_generator:
92 a ChatGenerator instance which represents the LLM.
93 In order for the component to work, the LLM should be configured to return a JSON object. For example,
94 when using the OpenAIChatGenerator, you should pass `{"response_format": {"type": "json_object"}}` in the
95 `generation_kwargs`.
96 """
97 self.validate_init_parameters(inputs, outputs, examples)
98 component.set_input_types(self, **dict(inputs))
100 self.raise_on_failure = raise_on_failure
101 self.instructions = instructions
102 self.inputs = inputs
103 self.outputs = outputs
104 self.examples = examples
105 self.progress_bar = progress_bar
107 template = self.prepare_template()
108 self.builder = PromptBuilder(template=template)
110 if chat_generator is not None:
111 self._chat_generator = chat_generator
112 else:
113 generation_kwargs = {"response_format": {"type": "json_object"}, "seed": 42}
114 self._chat_generator = OpenAIChatGenerator(generation_kwargs=generation_kwargs)
116 def warm_up(self) -> None:
117 """
118 Warm up the underlying chat generator.
119 """
120 if hasattr(self._chat_generator, "warm_up"):
121 self._chat_generator.warm_up()
123 async def warm_up_async(self) -> None:
124 """
125 Warm up the underlying chat generator on the serving event loop.
126 """
127 if hasattr(self._chat_generator, "warm_up_async"):
128 await self._chat_generator.warm_up_async()
129 elif hasattr(self._chat_generator, "warm_up"):
130 self._chat_generator.warm_up()
132 def close(self) -> None:
133 """
134 Release the underlying chat generator's resources.
135 """
136 if hasattr(self._chat_generator, "close"):
137 self._chat_generator.close()
139 async def close_async(self) -> None:
140 """
141 Release the underlying chat generator's async resources.
142 """
143 if hasattr(self._chat_generator, "close_async"):
144 await self._chat_generator.close_async()
145 elif hasattr(self._chat_generator, "close"):
146 self._chat_generator.close()
148 @staticmethod
149 def validate_init_parameters(
150 inputs: list[tuple[str, type[list]]], outputs: list[str], examples: list[dict[str, Any]]
151 ) -> None:
152 """
153 Validate the init parameters.
155 :param inputs:
156 The inputs to validate.
157 :param outputs:
158 The outputs to validate.
159 :param examples:
160 The examples to validate.
162 :raises ValueError:
163 If the inputs are not a list of tuples with a string and a type of list.
164 If the outputs are not a list of strings.
165 If the examples are not a list of dictionaries.
166 If any example does not have keys "inputs" and "outputs" with values that are dictionaries with string keys.
167 """
168 # Validate inputs
169 if (
170 not isinstance(inputs, list)
171 or not all(isinstance(_input, tuple) for _input in inputs)
172 or not all(isinstance(_input[0], str) and _input[1] is not list and len(_input) == 2 for _input in inputs)
173 ):
174 msg = (
175 f"LLM evaluator expects inputs to be a list of tuples. Each tuple must contain an input name and "
176 f"type of list but received {inputs}."
177 )
178 raise ValueError(msg)
180 # Validate outputs
181 if not isinstance(outputs, list) or not all(isinstance(output, str) for output in outputs):
182 msg = f"LLM evaluator expects outputs to be a list of str but received {outputs}."
183 raise ValueError(msg)
185 # Validate examples are lists of dicts
186 if not isinstance(examples, list) or not all(isinstance(example, dict) for example in examples):
187 msg = f"LLM evaluator expects examples to be a list of dictionaries but received {examples}."
188 raise ValueError(msg)
190 # Validate each example
191 for example in examples:
192 if (
193 {"inputs", "outputs"} != example.keys()
194 or not all(isinstance(example[param], dict) for param in ["inputs", "outputs"])
195 or not all(isinstance(key, str) for param in ["inputs", "outputs"] for key in example[param])
196 ):
197 msg = (
198 f"LLM evaluator expects each example to have keys `inputs` and `outputs` with values that are "
199 f"dictionaries with str keys but received {example}."
200 )
201 raise ValueError(msg)
203 def _parse_reply(self, reply_text: str) -> dict[str, Any] | None:
204 """
205 Parse the chat generator's reply into a dictionary with the expected output keys.
207 :param reply_text: The text of the chat generator's reply.
208 :returns: The parsed dictionary, or `None` if parsing fails and `raise_on_failure` is False.
209 :raises ValueError: If parsing fails and `raise_on_failure` is True.
210 """
211 try:
212 return _parse_dict_from_json(reply_text, expected_keys=self.outputs, raise_on_failure=True)
213 except ValueError as e:
214 if self.raise_on_failure:
215 raise
216 logger.warning(
217 "{component} failed to parse the ChatGenerator response: {response}. Setting the result for this "
218 "input to None. Error: {error}",
219 component=type(self).__name__,
220 response=reply_text,
221 error=e,
222 )
223 return None
225 @component.output_types(results=list[dict[str, Any]], meta=list[dict[str, Any]] | None)
226 def run(self, **inputs: Any) -> dict[str, Any]:
227 """
228 Run the LLM evaluator.
230 :param inputs:
231 The input values to evaluate. The keys are the input names and the values are lists of input values.
232 :returns:
233 A dictionary with a `results` entry that contains a list of results.
234 Each result is a dictionary containing the keys as defined in the `outputs` parameter of the LLMEvaluator
235 and the evaluation results as the values. If an exception occurs for a particular input value, the result
236 will be `None` for that entry.
237 If the API is "openai" and the response contains a "meta" key, the metadata from OpenAI will be included
238 in the output dictionary, under the key "meta".
239 :raises ValueError:
240 Only in the case that `raise_on_failure` is set to True and the received inputs are not lists or have
241 different lengths, or if the output is not a valid JSON or doesn't contain the expected keys.
242 """
243 self.warm_up()
245 self.validate_input_parameters(dict(self.inputs), inputs)
247 # inputs is a dictionary with keys being input names and values being a list of input values
248 # We need to iterate through the lists in parallel for all keys of the dictionary
249 input_names, values = inputs.keys(), list(zip(*inputs.values(), strict=True))
250 list_of_input_names_to_values = [dict(zip(input_names, v, strict=True)) for v in values]
252 results: list[dict[str, Any] | None] = []
253 metadata = []
254 errors = 0
255 for input_names_to_values in tqdm(list_of_input_names_to_values, disable=not self.progress_bar):
256 prompt = self.builder.run(**input_names_to_values)
257 messages = [ChatMessage.from_user(prompt["prompt"])]
258 try:
259 with _trace_chat_generator_run(self._chat_generator, {"messages": messages}) as span:
260 result = self._chat_generator.run(messages=messages)
261 span.set_content_tag("haystack.component.output", result)
262 except Exception as e:
263 if self.raise_on_failure:
264 raise ValueError(f"Error while generating response for prompt: {prompt}. Error: {e}") from e
265 logger.warning(
266 "{component} failed during chat generation for prompt: {prompt}. Setting the result for this "
267 "input to None. Error: {e}",
268 component=type(self).__name__,
269 prompt=prompt,
270 e=e,
271 )
272 results.append(None)
273 errors += 1
274 continue
276 parsed_result = self._parse_reply(result["replies"][0].text)
277 if parsed_result is None:
278 results.append(None)
279 errors += 1
280 else:
281 results.append(parsed_result)
283 if result["replies"][0].meta:
284 metadata.append(result["replies"][0].meta)
286 if errors > 0:
287 logger.warning(
288 "{component} failed for {errors} out of {total} inputs. The results for these inputs are None.",
289 component=type(self).__name__,
290 errors=errors,
291 total=len(list_of_input_names_to_values),
292 )
294 return {"results": results, "meta": metadata or None}
296 @component.output_types(results=list[dict[str, Any]], meta=list[dict[str, Any]] | None)
297 async def run_async(self, **inputs: Any) -> dict[str, Any]:
298 """
299 Run the LLM evaluator asynchronously
301 :param inputs:
302 The input values to evaluate. The keys are the input names and the values are lists of input values.
303 :returns:
304 A dictionary with a `results` entry that contains a list of results.
305 Each result is a dictionary containing the keys as defined in the `outputs` parameter of the LLMEvaluator
306 and the evaluation results as the values. If an exception occurs for a particular input value, the result
307 will be `None` for that entry.
308 If the API is "openai" and the response contains a "meta" key, the metadata from OpenAI will be included
309 in the output dictionary, under the key "meta".
310 :raises TypeError:
311 If the chat generator does not support async execution.
312 :raises ValueError:
313 Only in the case that `raise_on_failure` is set to True and the received inputs are not lists or have
314 different lengths, or if the output is not a valid JSON or doesn't contain the expected keys.
315 """
317 await self.warm_up_async()
319 self.validate_input_parameters(dict(self.inputs), inputs)
321 # inputs is a dictionary with keys being input names and values being a list of input values
322 # We need to iterate through the lists in parallel for all keys of the dictionary
323 input_names, values = inputs.keys(), list(zip(*inputs.values(), strict=True))
324 list_of_input_names_to_values = [dict(zip(input_names, v, strict=True)) for v in values]
326 results: list[dict[str, Any] | None] = []
327 metadata = []
328 errors = 0
330 generator_has_async = hasattr(self._chat_generator, "run_async")
331 for input_names_to_values in async_tqdm(list_of_input_names_to_values, disable=not self.progress_bar):
332 prompt = self.builder.run(**input_names_to_values)
333 messages = [ChatMessage.from_user(prompt["prompt"])]
334 try:
335 with _trace_chat_generator_run(self._chat_generator, {"messages": messages}) as span:
336 if generator_has_async:
337 result = await self._chat_generator.run_async(messages=messages) # type: ignore[attr-defined]
338 else:
339 logger.debug(
340 "{generator_type} does not implement 'run_async'."
341 " Running the synchronous 'run' method in a thread to avoid blocking the event loop.",
342 generator_type=type(self._chat_generator).__name__,
343 )
344 result = await asyncio.to_thread(self._chat_generator.run, messages=messages)
345 span.set_content_tag("haystack.component.output", result)
346 except Exception as e:
347 if self.raise_on_failure:
348 raise ValueError(f"Error while generating response for prompt: {prompt}. Error: {e}") from e
349 logger.warning(
350 "{component} failed during chat generation for prompt: {prompt}. Setting the result for this "
351 "input to None. Error: {e}",
352 component=type(self).__name__,
353 prompt=prompt,
354 e=e,
355 )
356 results.append(None)
357 errors += 1
358 continue
360 parsed_result = self._parse_reply(result["replies"][0].text)
361 if parsed_result is None:
362 results.append(None)
363 errors += 1
364 else:
365 results.append(parsed_result)
367 if result["replies"][0].meta:
368 metadata.append(result["replies"][0].meta)
370 if errors > 0:
371 logger.warning(
372 "{component} failed for {errors} out of {total} inputs. The results for these inputs are None.",
373 component=type(self).__name__,
374 errors=errors,
375 total=len(list_of_input_names_to_values),
376 )
378 return {"results": results, "meta": metadata or None}
380 def prepare_template(self) -> str:
381 """
382 Prepare the prompt template.
384 Combine instructions, inputs, outputs, and examples into one prompt template with the following format:
385 Instructions:
386 `<instructions>`
388 Generate the response in JSON format with the following keys:
389 `<list of output keys>`
390 Consider the instructions and the examples below to determine those values.
392 Examples:
393 `<examples>`
395 Inputs:
396 `<inputs>`
397 Outputs:
399 :returns:
400 The prompt template.
401 """
402 inputs_section = (
403 "{" + ", ".join([f'"{input_socket[0]}": {{{{ {input_socket[0]} }}}}' for input_socket in self.inputs]) + "}"
404 )
406 examples_section = "\n".join(
407 [
408 "Inputs:\n" + json.dumps(example["inputs"]) + "\nOutputs:\n" + json.dumps(example["outputs"])
409 for example in self.examples
410 ]
411 )
412 return (
413 f"Instructions:\n"
414 f"{self.instructions}\n\n"
415 f"Generate the response in JSON format with the following keys:\n"
416 f"{json.dumps(self.outputs)}\n"
417 f"Consider the instructions and the examples below to determine those values.\n\n"
418 f"Examples:\n"
419 f"{examples_section}\n\n"
420 f"Inputs:\n"
421 f"{inputs_section}\n"
422 f"Outputs:\n"
423 )
425 def to_dict(self) -> dict[str, Any]:
426 """
427 Serialize this component to a dictionary.
429 :returns:
430 The serialized component as a dictionary.
431 """
432 # Since we cannot currently serialize tuples, convert the inputs to a list.
433 inputs = [[name, serialize_type(type_)] for name, type_ in self.inputs]
434 return default_to_dict(
435 self,
436 instructions=self.instructions,
437 inputs=inputs,
438 outputs=self.outputs,
439 examples=self.examples,
440 chat_generator=component_to_dict(obj=self._chat_generator, name="chat_generator"),
441 progress_bar=self.progress_bar,
442 raise_on_failure=self.raise_on_failure,
443 )
445 @classmethod
446 def from_dict(cls, data: dict[str, Any]) -> "LLMEvaluator":
447 """
448 Deserialize this component from a dictionary.
450 :param data:
451 The dictionary representation of this component.
452 :returns:
453 The deserialized component instance.
454 """
455 data["init_parameters"]["inputs"] = [
456 (name, deserialize_type(type_)) for name, type_ in data["init_parameters"]["inputs"]
457 ]
459 if data["init_parameters"].get("chat_generator"):
460 deserialize_chatgenerator_inplace(data["init_parameters"], key="chat_generator")
462 return default_from_dict(cls, data)
464 @staticmethod
465 def validate_input_parameters(expected: dict[str, Any], received: dict[str, Any]) -> None:
466 """
467 Validate the input parameters.
469 :param expected:
470 The expected input parameters.
471 :param received:
472 The received input parameters.
474 :raises ValueError:
475 If not all expected inputs are present in the received inputs
476 If the received inputs are not lists or have different lengths
477 """
478 # Validate that all expected inputs are present in the received inputs
479 for param in expected:
480 if param not in received:
481 msg = f"LLM evaluator expected input parameter '{param}' but received only {received.keys()}."
482 raise ValueError(msg)
484 # Validate that all received inputs are lists
485 if not all(isinstance(_input, list) for _input in received.values()):
486 msg = (
487 "LLM evaluator expects all input values to be lists but received "
488 f"{[type(_input) for _input in received.values()]}."
489 )
490 raise ValueError(msg)
492 # Validate that all received inputs are of the same length
493 inputs = received.values()
494 length = len(next(iter(inputs)))
495 if not all(len(_input) == length for _input in inputs):
496 msg = (
497 f"LLM evaluator expects all input lists to have the same length but received {inputs} with lengths "
498 f"{[len(_input) for _input in inputs]}."
499 )
500 raise ValueError(msg)