mirror of
https://github.com/Mintplex-Labs/langchain-python.git
synced 2026-07-19 21:33:31 -04:00
Zander Chase
cc60fed3be
Notebook shows preference scoring between two chains and reports wilson score interval + p value I think I'll add the option to insert ground truth labels but doesn't have to be in this PR
114 lines
3.8 KiB
Python
114 lines
3.8 KiB
Python
"""Interfaces to be implemented by general evaluators."""
|
|
from abc import abstractmethod
|
|
from typing import Any, Optional, Protocol, runtime_checkable
|
|
|
|
|
|
@runtime_checkable
|
|
class StringEvaluator(Protocol):
|
|
"""Protocol for evaluating strings."""
|
|
|
|
@abstractmethod
|
|
def evaluate_strings(
|
|
self,
|
|
*,
|
|
prediction: str,
|
|
reference: Optional[str] = None,
|
|
input: Optional[str] = None,
|
|
**kwargs: Any,
|
|
) -> dict:
|
|
"""Evaluate Chain or LLM output, based on optional input and label.
|
|
|
|
Args:
|
|
prediction (str): the LLM or chain prediction to evaluate.
|
|
reference (Optional[str], optional): the reference label
|
|
to evaluate against.
|
|
input (Optional[str], optional): the input to consider during evaluation
|
|
**kwargs: additional keyword arguments, including callbacks, tags, etc.
|
|
Returns:
|
|
dict: The evaluation results containing the score or value.
|
|
"""
|
|
|
|
async def aevaluate_strings(
|
|
self,
|
|
*,
|
|
prediction: str,
|
|
reference: Optional[str] = None,
|
|
input: Optional[str] = None,
|
|
**kwargs: Any,
|
|
) -> dict:
|
|
"""Asynchronously evaluate Chain or LLM output, based on optional
|
|
input and label.
|
|
|
|
Args:
|
|
prediction (str): the LLM or chain prediction to evaluate.
|
|
reference (Optional[str], optional): the reference label
|
|
to evaluate against.
|
|
input (Optional[str], optional): the input to consider during evaluation
|
|
**kwargs: additional keyword arguments, including callbacks, tags, etc.
|
|
Returns:
|
|
dict: The evaluation results containing the score or value.
|
|
"""
|
|
raise NotImplementedError(
|
|
f"{self.__class__.__name__} hasn't implemented an "
|
|
"async aevaluate_strings method."
|
|
)
|
|
|
|
|
|
@runtime_checkable
|
|
class PairwiseStringEvaluator(Protocol):
|
|
"""A protocol for comparing the output of two models."""
|
|
|
|
@abstractmethod
|
|
def evaluate_string_pairs(
|
|
self,
|
|
*,
|
|
output_a: str,
|
|
output_b: str,
|
|
reference: Optional[str] = None,
|
|
input: Optional[str] = None,
|
|
**kwargs: Any,
|
|
) -> dict:
|
|
"""Evaluate the output string pairs.
|
|
|
|
Args:
|
|
output_a (str): The output string from the first model.
|
|
output_b (str): The output string from the second model.
|
|
reference (str, optional): The expected output / reference
|
|
string. Defaults to None.
|
|
input (str, optional): The input string. Defaults to None.
|
|
**kwargs (Any): Additional keyword arguments, such
|
|
as callbacks and optional reference strings.
|
|
|
|
Returns:
|
|
dict: A dictionary containing the preference, scores, and/or
|
|
other information.
|
|
"""
|
|
|
|
async def aevaluate_string_pairs(
|
|
self,
|
|
output_a: str,
|
|
output_b: str,
|
|
reference: Optional[str] = None,
|
|
input: Optional[str] = None,
|
|
**kwargs: Any,
|
|
) -> dict:
|
|
"""Evaluate the output string pairs.
|
|
|
|
Args:
|
|
output_a (str): The output string from the first model.
|
|
output_b (str): The output string from the second model.
|
|
reference (str, optional): The expected output / reference
|
|
string. Defaults to None.
|
|
input (str, optional): The input string. Defaults to None.
|
|
**kwargs (Any): Additional keyword arguments, such
|
|
as callbacks and optional reference strings.
|
|
|
|
Returns:
|
|
dict: A dictionary containing the preference, scores, and/or
|
|
other information.
|
|
"""
|
|
raise NotImplementedError(
|
|
f"{self.__class__.__name__} hasn't implemented an async "
|
|
"aevaluate_string_pairs method."
|
|
)
|