mirror of
https://github.com/vibrantlabsai/ragas.git
synced 2026-10-12 02:24:10 +00:00
Retire ragas.experimental namespace (#2223)
## Summary - Move experimental tests to appropriate locations in main test structure - Update build infrastructure to remove experimental references - Remove obsolete experimental directory and scripts
This commit is contained in:
Nirant
@@ -123,10 +123,10 @@ jobs:
|
||||
# Run different test suites based on test type
|
||||
if [ "${{ matrix.test-type }}" = "full" ]; then
|
||||
# Full test suite with notebook tests
|
||||
uv run pytest --nbmake tests/unit tests/experimental "${OPTS[@]}"
|
||||
uv run pytest --nbmake tests/unit "${OPTS[@]}"
|
||||
else
|
||||
# Essential tests only (faster for cross-platform validation)
|
||||
uv run pytest tests/unit tests/experimental -k "not slow" "${OPTS[@]}"
|
||||
uv run pytest tests/unit -k "not slow" "${OPTS[@]}"
|
||||
fi
|
||||
env:
|
||||
__RAGAS_DEBUG_TRACKING: true
|
||||
|
||||
@@ -88,7 +88,7 @@ make test-e2e
|
||||
|
||||
# Direct pytest commands for more control
|
||||
uv run pytest tests/unit -k "test_name"
|
||||
uv run pytest tests/experimental -v
|
||||
uv run pytest tests/unit -v
|
||||
```
|
||||
|
||||
### Documentation
|
||||
@@ -161,7 +161,7 @@ The experimental features are now integrated into the main ragas package:
|
||||
To use experimental features:
|
||||
|
||||
```python
|
||||
from ragas.experimental import Dataset
|
||||
from ragas import Dataset
|
||||
from ragas import experiment
|
||||
from ragas.backends import get_registry
|
||||
```
|
||||
|
||||
+2
-2
@@ -194,7 +194,7 @@ make run-ci # Run full CI pipeline
|
||||
|
||||
# Or use direct commands for specific tasks
|
||||
uv run pytest tests/unit # Run core unit tests
|
||||
uv run pytest tests/experimental # Run experimental tests
|
||||
uv run pytest tests/unit # Run unit tests
|
||||
uv run pyright src # Type check source code
|
||||
```
|
||||
|
||||
@@ -278,7 +278,7 @@ make install
|
||||
uv run pytest tests/unit/test_specific.py -v
|
||||
|
||||
# Check experimental test dependencies
|
||||
uv run pytest tests/experimental --collect-only
|
||||
uv run pytest tests/unit --collect-only
|
||||
```
|
||||
|
||||
#### Formatting Issues
|
||||
|
||||
@@ -48,7 +48,7 @@ install: ## Install full dependencies with uv sync (backward compatible - modern
|
||||
# CODE QUALITY
|
||||
# =============================================================================
|
||||
|
||||
.PHONY: help setup-venv install-minimal install format type check clean test test-e2e benchmarks benchmarks-docker run-ci run-ci-fast run-ci-format-check run-ci-type run-ci-tests build-docs serve-docs process-experimental-notebooks
|
||||
.PHONY: help setup-venv install-minimal install format type check clean test test-e2e benchmarks benchmarks-docker run-ci run-ci-fast run-ci-format-check run-ci-type run-ci-tests build-docs serve-docs
|
||||
format: ## Format and lint all code
|
||||
@echo "Formatting and linting all code..."
|
||||
@echo "(ruff format) Formatting ragas..."
|
||||
@@ -84,7 +84,7 @@ benchmarks-docker: ## Run benchmarks in docker
|
||||
|
||||
benchmarks-test: ## Run benchmarks for ragas unit tests
|
||||
@echo "Running ragas unit tests with timing benchmarks..."
|
||||
$(Q)uv run --active pytest --nbmake tests/unit tests/experimental --durations=0 -v $(shell if [ -n "$(k)" ]; then echo "-k $(k)"; fi)
|
||||
$(Q)uv run --active pytest --nbmake tests/unit --durations=0 -v $(shell if [ -n "$(k)" ]; then echo "-k $(k)"; fi)
|
||||
|
||||
# =============================================================================
|
||||
# CI/BUILD
|
||||
@@ -98,7 +98,7 @@ run-ci: ## Run complete CI pipeline (mirrors GitHub CI exactly)
|
||||
@echo "Type check..."
|
||||
$(Q)$(MAKE) type
|
||||
@echo "Unit tests..."
|
||||
$(Q)__RAGAS_DEBUG_TRACKING=true RAGAS_DO_NOT_TRACK=true uv run --active pytest --nbmake tests/unit tests/experimental --dist loadfile -n auto
|
||||
$(Q)__RAGAS_DEBUG_TRACKING=true RAGAS_DO_NOT_TRACK=true uv run --active pytest --nbmake tests/unit --dist loadfile -n auto
|
||||
@echo "All CI checks passed!"
|
||||
|
||||
run-ci-format-check: ## Run format check in dry-run mode (like GitHub CI)
|
||||
@@ -113,7 +113,7 @@ run-ci-type: ## Run type checking (matches GitHub CI)
|
||||
|
||||
run-ci-tests: ## Run all tests with CI options
|
||||
@echo "Running all tests with CI options..."
|
||||
$(Q)__RAGAS_DEBUG_TRACKING=true RAGAS_DO_NOT_TRACK=true pytest --nbmake tests/unit tests/experimental --dist loadfile -n auto
|
||||
$(Q)__RAGAS_DEBUG_TRACKING=true RAGAS_DO_NOT_TRACK=true pytest --nbmake tests/unit --dist loadfile -n auto
|
||||
|
||||
run-ci-fast: ## Fast CI check for quick local validation (2-3 minutes)
|
||||
@echo "Running fast CI check for quick feedback..."
|
||||
@@ -121,15 +121,15 @@ run-ci-fast: ## Fast CI check for quick local validation (2-3 minutes)
|
||||
$(Q)uv run --active ruff format --check src tests docs --config pyproject.toml
|
||||
$(Q)uv run --active ruff check src docs tests --config pyproject.toml
|
||||
@echo "Core unit tests (no nbmake for speed)..."
|
||||
$(Q)uv run --active pytest tests/unit tests/experimental --dist loadfile -n auto -x
|
||||
$(Q)uv run --active pytest tests/unit --dist loadfile -n auto -x
|
||||
@echo "Fast CI check completed!"
|
||||
|
||||
clean: ## Clean all generated files
|
||||
@echo "Cleaning all generated files..."
|
||||
$(Q)find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete
|
||||
$(Q)rm -rf site/ docs/site/ .mypy_cache .pytest_cache .ruff_cache
|
||||
$(Q)rm -rf dist/ build/ *.egg-info/ src/*.egg-info/ experimental/*.egg-info/
|
||||
$(Q)rm -rf .coverage htmlcov/ .tox/ .venv/ experimental/.venv/
|
||||
$(Q)rm -rf dist/ build/ *.egg-info/ src/*.egg-info/
|
||||
$(Q)rm -rf .coverage htmlcov/ .tox/ .venv/
|
||||
$(Q)find . -name '*.log' -delete
|
||||
$(Q)find . -name '.DS_Store' -delete
|
||||
$(Q)find . -name 'temp*' -type d -exec rm -rf {} + 2>/dev/null || true
|
||||
@@ -142,11 +142,11 @@ clean: ## Clean all generated files
|
||||
|
||||
test: ## Run all unit tests
|
||||
@echo "Running all unit tests..."
|
||||
$(Q)uv run --active pytest tests/unit tests/experimental $(shell if [ -n "$(k)" ]; then echo "-k $(k)"; fi)
|
||||
$(Q)uv run --active pytest tests/unit $(shell if [ -n "$(k)" ]; then echo "-k $(k)"; fi)
|
||||
|
||||
test-all: ## Run all unit tests (including notebooks)
|
||||
@echo "Running all unit tests (including notebooks)..."
|
||||
$(Q)uv run --active pytest --nbmake tests/unit tests/experimental $(shell if [ -n "$(k)" ]; then echo "-k $(k)"; fi)
|
||||
$(Q)uv run --active pytest --nbmake tests/unit $(shell if [ -n "$(k)" ]; then echo "-k $(k)"; fi)
|
||||
|
||||
test-e2e: ## Run all end-to-end tests
|
||||
@echo "Running all end-to-end tests..."
|
||||
@@ -156,11 +156,7 @@ test-e2e: ## Run all end-to-end tests
|
||||
# DOCUMENTATION
|
||||
# =============================================================================
|
||||
|
||||
process-experimental-notebooks: ## Process experimental notebooks to markdown for MkDocs
|
||||
@echo "Processing experimental notebooks..."
|
||||
$(Q)python $(GIT_ROOT)/scripts/process_experimental_notebooks.py
|
||||
|
||||
build-docs: process-experimental-notebooks ## Build all documentation
|
||||
build-docs: ## Build all documentation
|
||||
@echo "Building all documentation..."
|
||||
@echo "Converting ipynb notebooks to md files..."
|
||||
$(Q)python $(GIT_ROOT)/docs/ipynb_to_md.py
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from agent import get_default_agent
|
||||
|
||||
from ragas import experiment
|
||||
from ragas.experimental import Dataset
|
||||
from ragas import Dataset
|
||||
from ragas.metrics.numeric import numeric_metric
|
||||
from ragas.metrics.result import MetricResult
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from prompt import run_prompt
|
||||
|
||||
from ragas import experiment
|
||||
from ragas.experimental import Dataset
|
||||
from ragas import Dataset
|
||||
from ragas.metrics.discrete import discrete_metric
|
||||
from ragas.metrics.result import MetricResult
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ from openai import OpenAI
|
||||
from rag import default_rag_client
|
||||
|
||||
from ragas import experiment
|
||||
from ragas.experimental import Dataset
|
||||
from ragas import Dataset
|
||||
from ragas.llms import instructor_llm_factory
|
||||
from ragas.metrics import DiscreteMetric
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ from openai import OpenAI
|
||||
from workflow import default_workflow_client
|
||||
|
||||
from ragas import experiment
|
||||
from ragas.experimental import Dataset
|
||||
from ragas import Dataset
|
||||
from ragas.llms import instructor_llm_factory
|
||||
from ragas.metrics import DiscreteMetric
|
||||
|
||||
|
||||
+8
-19
@@ -12,25 +12,19 @@ dependencies = [
|
||||
"nest-asyncio",
|
||||
"appdirs",
|
||||
"diskcache>=5.6.3",
|
||||
|
||||
"typer",
|
||||
"rich",
|
||||
"openai>=1.0.0",
|
||||
"tqdm",
|
||||
"instructor",
|
||||
"gitpython",
|
||||
"pillow>=10.4.0",
|
||||
|
||||
# LangChain ecosystem
|
||||
"langchain",
|
||||
"langchain-core",
|
||||
"langchain-community",
|
||||
"langchain_openai",
|
||||
|
||||
# CLI dependencies
|
||||
"typer",
|
||||
"rich",
|
||||
|
||||
# LLM providers
|
||||
"openai>=1.0.0",
|
||||
|
||||
# Experimental features
|
||||
"tqdm",
|
||||
"instructor",
|
||||
"gitpython",
|
||||
"pillow>=10.4.0",
|
||||
]
|
||||
dynamic = ["version", "readme"]
|
||||
|
||||
@@ -117,7 +111,6 @@ preview = false
|
||||
|
||||
[tool.pyright]
|
||||
include = ["src/ragas"]
|
||||
exclude = ["src/ragas/experimental"]
|
||||
excludeTypeshedPaths = ["@types/*"]
|
||||
pythonVersion = "3.9"
|
||||
pythonPlatform = "All"
|
||||
@@ -140,10 +133,6 @@ version_file = "src/ragas/_version.py"
|
||||
addopts = "-n 0"
|
||||
asyncio_default_fixture_loop_scope = "function"
|
||||
testpaths = ["tests"]
|
||||
markers = [
|
||||
"experimental_ci: Set of tests that will be run as part of Experimental CI",
|
||||
"e2e: End-to-End tests for Experimental",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
# Full dev dependencies with all features (used by make install)
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
Script to process experimental notebooks with nbdev and convert to markdown for MkDocs documentation.
|
||||
This script should be executed from the project root directory.
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Path configurations
|
||||
RAGAS_ROOT = Path(__file__).parent.parent
|
||||
EXPERIMENTAL_DIR = RAGAS_ROOT / "experimental"
|
||||
PROC_DIR = EXPERIMENTAL_DIR / "_proc"
|
||||
DOCS_DIR = RAGAS_ROOT / "docs" / "experimental"
|
||||
|
||||
|
||||
def run_command(cmd, cwd=None):
|
||||
"""Run a shell command and print output"""
|
||||
print(f"Running: {' '.join(cmd)}")
|
||||
result = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
print(f"Error executing command: {' '.join(cmd)}")
|
||||
print(f"STDERR: {result.stderr}")
|
||||
sys.exit(1)
|
||||
return result.stdout
|
||||
|
||||
|
||||
def process_notebooks():
|
||||
"""Process notebooks with nbdev_proc_nbs"""
|
||||
print("Processing notebooks with nbdev...")
|
||||
run_command(["nbdev_proc_nbs"], cwd=EXPERIMENTAL_DIR)
|
||||
|
||||
if not PROC_DIR.exists():
|
||||
print(
|
||||
f"Error: Expected processed notebooks at {PROC_DIR}, but directory does not exist."
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Notebooks processed successfully to {PROC_DIR}")
|
||||
|
||||
|
||||
def render_with_quarto():
|
||||
"""Render processed notebooks to markdown using Quarto"""
|
||||
print("Rendering notebooks to markdown with Quarto...")
|
||||
|
||||
# Ensure the output directory exists
|
||||
os.makedirs(DOCS_DIR, exist_ok=True)
|
||||
|
||||
# Run Quarto to convert notebooks to markdown
|
||||
try:
|
||||
run_command(
|
||||
[
|
||||
"quarto",
|
||||
"render",
|
||||
"**/*.ipynb",
|
||||
"--to",
|
||||
"gfm",
|
||||
"--output-dir",
|
||||
str(DOCS_DIR),
|
||||
],
|
||||
cwd=PROC_DIR,
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Error rendering notebooks with Quarto: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Notebooks rendered successfully to {DOCS_DIR}")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function to process notebooks and render to markdown"""
|
||||
# Ensure we're in the project root
|
||||
if (
|
||||
not (RAGAS_ROOT / "ragas").exists()
|
||||
or not (RAGAS_ROOT / "experimental").exists()
|
||||
):
|
||||
print("Error: This script must be run from the ragas project root directory.")
|
||||
sys.exit(1)
|
||||
|
||||
# Process notebooks with nbdev
|
||||
process_notebooks()
|
||||
|
||||
# Render notebooks to markdown with Quarto
|
||||
render_with_quarto()
|
||||
|
||||
print("Notebook processing and rendering completed successfully!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+1
-1
@@ -547,7 +547,7 @@ def hello_world(
|
||||
evals_content = '''import typing as t
|
||||
|
||||
import numpy as np
|
||||
from ragas.experimental import BaseModel
|
||||
from pydantic import BaseModel
|
||||
# from ragas.experimental.project.backends import LocalCSVProjectBackend # TODO: Not implemented yet
|
||||
from ragas.metrics.result import MetricResult
|
||||
from ragas.metrics.numeric import numeric_metric
|
||||
|
||||
@@ -5,6 +5,7 @@ import typing as t
|
||||
import numpy as np
|
||||
import pytest
|
||||
from langchain_core.outputs import Generation, LLMResult
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ragas.embeddings.base import BaseRagasEmbeddings
|
||||
from ragas.llms.base import BaseRagasLLM
|
||||
@@ -70,3 +71,65 @@ def fake_llm():
|
||||
@pytest.fixture
|
||||
def fake_embedding():
|
||||
return EchoEmbedding()
|
||||
|
||||
|
||||
# ====================
|
||||
# Mock fixtures from experimental tests
|
||||
# ====================
|
||||
|
||||
|
||||
class MockLLM:
|
||||
"""Mock LLM for testing purposes"""
|
||||
|
||||
def __init__(self):
|
||||
self.provider = "mock"
|
||||
self.model = "mock-model"
|
||||
self.is_async = True
|
||||
|
||||
def generate(self, prompt: str, response_model: t.Type[BaseModel]) -> BaseModel:
|
||||
# Return a mock instance of the response model
|
||||
return response_model()
|
||||
|
||||
async def agenerate(
|
||||
self, prompt: str, response_model: t.Type[BaseModel]
|
||||
) -> BaseModel:
|
||||
# Return a mock instance of the response model
|
||||
return response_model()
|
||||
|
||||
|
||||
class MockEmbedding(BaseRagasEmbeddings):
|
||||
"""Mock Embedding for testing purposes"""
|
||||
|
||||
def embed_text(self, text: str, **kwargs: t.Any) -> t.List[float]:
|
||||
np.random.seed(42) # Set seed for deterministic tests
|
||||
return np.random.rand(768).tolist()
|
||||
|
||||
async def aembed_text(self, text: str, **kwargs: t.Any) -> t.List[float]:
|
||||
np.random.seed(42) # Set seed for deterministic tests
|
||||
return np.random.rand(768).tolist()
|
||||
|
||||
def embed_document(
|
||||
self,
|
||||
text: str,
|
||||
metadata: t.Optional[t.Dict[str, t.Any]] = None,
|
||||
**kwargs: t.Any,
|
||||
) -> t.List[float]:
|
||||
return self.embed_text(text, **kwargs)
|
||||
|
||||
async def aembed_document(
|
||||
self,
|
||||
text: str,
|
||||
metadata: t.Optional[t.Dict[str, t.Any]] = None,
|
||||
**kwargs: t.Any,
|
||||
) -> t.List[float]:
|
||||
return await self.aembed_text(text, **kwargs)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_llm():
|
||||
return MockLLM()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_embedding():
|
||||
return MockEmbedding()
|
||||
|
||||
@@ -1,83 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import typing as t
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ragas.embeddings.base import BaseRagasEmbedding as BaseEmbedding
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
"""
|
||||
configure pytest for experimental tests
|
||||
"""
|
||||
# Extra Pytest Markers
|
||||
# add `experimental_ci`
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
"experimental_ci: Set of tests that will be run as part of Experimental CI",
|
||||
)
|
||||
# add `e2e`
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
"e2e: End-to-End tests for Experimental",
|
||||
)
|
||||
|
||||
|
||||
class MockLLM:
|
||||
"""Mock LLM for testing purposes"""
|
||||
|
||||
def __init__(self):
|
||||
self.provider = "mock"
|
||||
self.model = "mock-model"
|
||||
self.is_async = True
|
||||
|
||||
def generate(self, prompt: str, response_model: t.Type[BaseModel]) -> BaseModel:
|
||||
# Return a mock instance of the response model
|
||||
return response_model()
|
||||
|
||||
async def agenerate(
|
||||
self, prompt: str, response_model: t.Type[BaseModel]
|
||||
) -> BaseModel:
|
||||
# Return a mock instance of the response model
|
||||
return response_model()
|
||||
|
||||
|
||||
class MockEmbedding(BaseEmbedding):
|
||||
"""Mock Embedding for testing purposes"""
|
||||
|
||||
def embed_text(self, text: str, **kwargs: t.Any) -> t.List[float]:
|
||||
np.random.seed(42) # Set seed for deterministic tests
|
||||
return np.random.rand(768).tolist()
|
||||
|
||||
async def aembed_text(self, text: str, **kwargs: t.Any) -> t.List[float]:
|
||||
np.random.seed(42) # Set seed for deterministic tests
|
||||
return np.random.rand(768).tolist()
|
||||
|
||||
def embed_document(
|
||||
self,
|
||||
text: str,
|
||||
metadata: t.Optional[t.Dict[str, t.Any]] = None,
|
||||
**kwargs: t.Any,
|
||||
) -> t.List[float]:
|
||||
return self.embed_text(text, **kwargs)
|
||||
|
||||
async def aembed_document(
|
||||
self,
|
||||
text: str,
|
||||
metadata: t.Optional[t.Dict[str, t.Any]] = None,
|
||||
**kwargs: t.Any,
|
||||
) -> t.List[float]:
|
||||
return await self.aembed_text(text, **kwargs)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_llm():
|
||||
return MockLLM()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_embedding():
|
||||
return MockEmbedding()
|
||||
@@ -1,103 +0,0 @@
|
||||
import typing as t
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ragas.metrics import MetricResult
|
||||
from ragas.metrics.llm_based import LLMMetric
|
||||
|
||||
|
||||
class MetricResponseModel(BaseModel):
|
||||
value: int
|
||||
reason: t.Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class CustomMetric(LLMMetric):
|
||||
"""Custom metric implementation for testing."""
|
||||
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
self._response_model = MetricResponseModel
|
||||
|
||||
def get_correlation(
|
||||
self, gold_labels: t.List[str], predictions: t.List[str]
|
||||
) -> float:
|
||||
return 0.0 # Placeholder for correlation logic
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_llm(mock_llm):
|
||||
"""Use the mock LLM from conftest."""
|
||||
return mock_llm
|
||||
|
||||
|
||||
def test_metric_creation():
|
||||
"""Test creating a custom metric."""
|
||||
metric = CustomMetric(name="test_metric", prompt="What is the result of {input}?")
|
||||
|
||||
assert metric.name == "test_metric"
|
||||
assert isinstance(metric.prompt, str) or hasattr(metric.prompt, "format")
|
||||
|
||||
|
||||
def test_metric_get_variables():
|
||||
"""Test extracting variables from prompt template."""
|
||||
metric = CustomMetric(
|
||||
name="test_metric",
|
||||
prompt="Evaluate the {question} given the {context} and {answer}",
|
||||
)
|
||||
|
||||
variables = metric.get_variables()
|
||||
expected_vars = ["question", "context", "answer"]
|
||||
|
||||
assert set(variables) == set(expected_vars)
|
||||
|
||||
|
||||
def test_metric_score_single(mock_llm):
|
||||
"""Test scoring with a single input."""
|
||||
metric = CustomMetric(name="test_metric", prompt="What is the result of {input}?")
|
||||
|
||||
# Mock the LLM to return a valid response
|
||||
def mock_generate(prompt, response_model):
|
||||
return response_model(value=1, reason="test reason")
|
||||
|
||||
mock_llm.generate = mock_generate
|
||||
|
||||
result = metric.score(llm=mock_llm, input="test")
|
||||
|
||||
assert isinstance(result, MetricResult)
|
||||
assert result.traces is not None
|
||||
assert "input" in result.traces
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metric_async_score(mock_llm):
|
||||
"""Test async scoring functionality."""
|
||||
metric = CustomMetric(name="test_metric", prompt="What is the result of {input}?")
|
||||
|
||||
# Mock the async LLM method
|
||||
async def mock_agenerate(prompt, response_model):
|
||||
return response_model(value=1, reason="test reason")
|
||||
|
||||
mock_llm.agenerate = mock_agenerate
|
||||
|
||||
result = await metric.ascore(llm=mock_llm, input="test")
|
||||
|
||||
assert isinstance(result, MetricResult)
|
||||
assert result.traces is not None
|
||||
|
||||
|
||||
def test_metric_response_model():
|
||||
"""Test that metric has correct response model."""
|
||||
metric = CustomMetric(name="test_metric", prompt="What is the result of {input}?")
|
||||
|
||||
assert metric._response_model == MetricResponseModel
|
||||
|
||||
|
||||
def test_metric_prompt_conversion():
|
||||
"""Test that string prompts are converted to Prompt objects."""
|
||||
metric = CustomMetric(name="test_metric", prompt="What is the result of {input}?")
|
||||
|
||||
# After __post_init__, prompt should be converted to Prompt object
|
||||
assert hasattr(metric.prompt, "format")
|
||||
+103
-1
@@ -2,10 +2,12 @@ import typing as t
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ragas.dataset_schema import SingleTurnSample
|
||||
from ragas.metrics import AspectCritic, SimpleCriteriaScore
|
||||
from ragas.metrics import AspectCritic, MetricResult, SimpleCriteriaScore
|
||||
from ragas.metrics.base import MetricType
|
||||
from ragas.metrics.llm_based import LLMMetric
|
||||
|
||||
|
||||
def test_single_turn_metric():
|
||||
@@ -109,3 +111,103 @@ def test_metrics_with_definition(metric):
|
||||
m.definition = "this is a new definition"
|
||||
assert m.definition == "this is a new definition"
|
||||
assert "this is a new definition" in m.single_turn_prompt.instruction
|
||||
|
||||
|
||||
# ====================
|
||||
# Metric Base Tests (formerly test_metric_base.py)
|
||||
# ====================
|
||||
|
||||
|
||||
class MetricResponseModel(BaseModel):
|
||||
value: int
|
||||
reason: t.Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class CustomMetric(LLMMetric):
|
||||
"""Custom metric implementation for testing."""
|
||||
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
self._response_model = MetricResponseModel
|
||||
|
||||
def get_correlation(
|
||||
self, gold_labels: t.List[str], predictions: t.List[str]
|
||||
) -> float:
|
||||
return 0.0 # Placeholder for correlation logic
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_llm(mock_llm):
|
||||
"""Use the mock LLM from conftest."""
|
||||
return mock_llm
|
||||
|
||||
|
||||
def test_metric_creation():
|
||||
"""Test creating a custom metric."""
|
||||
metric = CustomMetric(name="test_metric", prompt="What is the result of {input}?")
|
||||
|
||||
assert metric.name == "test_metric"
|
||||
assert isinstance(metric.prompt, str) or hasattr(metric.prompt, "format")
|
||||
|
||||
|
||||
def test_metric_get_variables():
|
||||
"""Test extracting variables from prompt template."""
|
||||
metric = CustomMetric(
|
||||
name="test_metric",
|
||||
prompt="Evaluate the {question} given the {context} and {answer}",
|
||||
)
|
||||
|
||||
variables = metric.get_variables()
|
||||
expected_vars = ["question", "context", "answer"]
|
||||
|
||||
assert set(variables) == set(expected_vars)
|
||||
|
||||
|
||||
def test_metric_score_single(mock_llm):
|
||||
"""Test scoring with a single input."""
|
||||
metric = CustomMetric(name="test_metric", prompt="What is the result of {input}?")
|
||||
|
||||
# Mock the LLM to return a valid response
|
||||
def mock_generate(prompt, response_model):
|
||||
return response_model(value=1, reason="test reason")
|
||||
|
||||
mock_llm.generate = mock_generate
|
||||
|
||||
result = metric.score(llm=mock_llm, input="test")
|
||||
|
||||
assert isinstance(result, MetricResult)
|
||||
assert result.traces is not None
|
||||
assert "input" in result.traces
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metric_async_score(mock_llm):
|
||||
"""Test async scoring functionality."""
|
||||
metric = CustomMetric(name="test_metric", prompt="What is the result of {input}?")
|
||||
|
||||
# Mock the async LLM method
|
||||
async def mock_agenerate(prompt, response_model):
|
||||
return response_model(value=1, reason="test reason")
|
||||
|
||||
mock_llm.agenerate = mock_agenerate
|
||||
|
||||
result = await metric.ascore(llm=mock_llm, input="test")
|
||||
|
||||
assert isinstance(result, MetricResult)
|
||||
assert result.traces is not None
|
||||
|
||||
|
||||
def test_metric_response_model():
|
||||
"""Test that metric has correct response model."""
|
||||
metric = CustomMetric(name="test_metric", prompt="What is the result of {input}?")
|
||||
|
||||
assert metric._response_model == MetricResponseModel
|
||||
|
||||
|
||||
def test_metric_prompt_conversion():
|
||||
"""Test that string prompts are converted to Prompt objects."""
|
||||
metric = CustomMetric(name="test_metric", prompt="What is the result of {input}?")
|
||||
|
||||
# After __post_init__, prompt should be converted to Prompt object
|
||||
assert hasattr(metric.prompt, "format")
|
||||
|
||||
Reference in New Issue
Block a user