handle extract error field (#870)

parallelise e2e tests

.github/workflows/test_e2e_py.yml

@@ -31,7 +31,7 @@ jobs:
 
       - name: Run Tests
         working-directory: py
-        run: uv run pytest unit_tests/ tests/ -v
+        run: make e2e
 
       - name: Remove virtual environment
         working-directory: py

py/Makefile

@@ -4,11 +4,15 @@ help:	## Show all Makefile targets.
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[33m%-30s\033[0m %s\n", $$1, $$2}'
 
 format:	## Run code autoformatters (black).
-	pre-commit install
-	git ls-files | xargs pre-commit run black --files
+	uv run pre-commit install
+	git ls-files | xargs uv run pre-commit run black --files
 
 lint:	## Run linters: pre-commit (black, ruff, codespell) and mypy
-	pre-commit install && git ls-files | xargs pre-commit run --show-diff-on-failure --files
+	uv run pre-commit install && git ls-files | xargs uv run pre-commit run --show-diff-on-failure --files
 
-test:	## Run tests via pytest
-	pytest tests
+test:	## Run unit tests via pytest
+	uv run pytest -v unit_tests/
+
+.PHONY: e2e
+e2e:	## Run all tests. Run with high parallelism using xdist since tests are bottlenecked bound by the slow backend parsing
+	uv run pytest -v -n 32 tests/

py/llama_cloud_services/beta/agent_data/schema.py

@@ -223,10 +223,12 @@ def parse_extracted_field_metadata(
         k: _parse_extracted_field_metadata_recursive(v)
         for k, v in field_metadata.items()
         if k not in _METADATA_FIELDS_SIBLING_TO_LEAF
+        and k not in _ADDITIONAL_ROOT_METADATA_FIELDS
     }
 
 
 _METADATA_FIELDS_SIBLING_TO_LEAF = {"reasoning"}
+_ADDITIONAL_ROOT_METADATA_FIELDS = {"error"}
 
 
 def _parse_extracted_field_metadata_recursive(
@@ -417,11 +419,14 @@ class ExtractedData(BaseModel, Generic[ExtractedT]):
         """
         file_id = file_id or result.file.id
         file_name = file_name or result.file.name
+        job_id = result.job_id
+        job_field_metadata = result.extraction_metadata.get("field_metadata", {})
+        errors = job_field_metadata.get("error", None)
+        if not isinstance(errors, str):
+            errors = None
 
         try:
-            field_metadata = parse_extracted_field_metadata(
-                result.extraction_metadata.get("field_metadata", {})
-            )
+            field_metadata = parse_extracted_field_metadata(job_field_metadata)
         except ValidationError:
             field_metadata = {}
 
@@ -430,11 +435,15 @@ class ExtractedData(BaseModel, Generic[ExtractedT]):
             return cls.create(
                 data=data,
                 status=status,
-                field_metadata=field_metadata,
+                field_metadata=job_field_metadata,
                 file_id=file_id,
                 file_name=file_name,
                 file_hash=file_hash,
-                metadata=metadata or {},
+                metadata={
+                    **({"field_errors": errors} if errors else {}),
+                    "job_id": job_id,
+                    **(metadata or {}),
+                },
             )
         except ValidationError as e:
             invalid_item = ExtractedData[Dict[str, Any]].create(

py/llama_parse/pyproject.toml

@@ -11,13 +11,13 @@ dev = [
 
 [project]
 name = "llama-parse"
-version = "0.6.58"
+version = "0.6.59"
 description = "Parse files into RAG-Optimized formats."
 authors = [{name = "Logan Markewich", email = "logan@llamaindex.ai"}]
 requires-python = ">=3.9,<4.0"
 readme = "README.md"
 license = "MIT"
-dependencies = ["llama-cloud-services>=0.6.58"]
+dependencies = ["llama-cloud-services>=0.6.59"]
 
 [project.scripts]
 llama-parse = "llama_parse.cli.main:parse"

py/pyproject.toml

@@ -5,6 +5,7 @@ build-backend = "hatchling.build"
 [dependency-groups]
 dev = [
   "pytest>=8.0.0,<9",
+  "pytest-xdist>=3.6.1,<4",
   "pytest-asyncio",
   "ipykernel>=6.29.0,<7",
   "pre-commit==3.2.0",
@@ -18,7 +19,7 @@ dev = [
 
 [project]
 name = "llama-cloud-services"
-version = "0.6.58"
+version = "0.6.59"
 description = "Tailored SDK clients for LlamaCloud services."
 authors = [{name = "Logan Markewich", email = "logan@runllama.ai"}]
 requires-python = ">=3.9,<4.0"

py/unit_tests/beta/agent/test_agent_data_schema.py

@@ -24,8 +24,8 @@ from llama_cloud_services.beta.agent_data.schema import (
 # Test data models
 class Person(BaseModel):
     name: str
-    age: int
-    email: str
+    age: Optional[int] = None
+    email: Optional[str] = None
 
 
 class Company(BaseModel):
@@ -405,6 +405,7 @@ def create_file(
 
 def create_extract_run(
     id: str = "extract-123",
+    job_id: str = "job-123",
     data: Dict[str, Any] = {"name": "John Doe", "age": 30, "email": "john@example.com"},
     extraction_metadata: Dict[str, Any] = {
         "name": {
@@ -423,6 +424,7 @@ def create_extract_run(
     return ExtractRun.parse_obj(
         {
             "id": id,
+            "job_id": job_id,
             "data": data,
             "extraction_metadata": {
                 "field_metadata": extraction_metadata,
@@ -501,10 +503,9 @@ def test_extracted_data_from_extraction_result_invalid_data():
     # Create ExtractRun with data that doesn't match Person schema
     extract_run = create_extract_run(
         data={
-            "name": "Valid Name",
+            "missing_name": "Valid Name",
             "age": "not_a_number",
-            "missing_email": True,
-        },  # Invalid age, missing email
+        },  # Invalid age, missing name
         extraction_metadata={
             "name": {"confidence": 0.9},
         },
@@ -523,9 +524,8 @@ def test_extracted_data_from_extraction_result_invalid_data():
     assert isinstance(invalid_data, ExtractedData)
     assert invalid_data.status == "error"
     assert invalid_data.data == {
-        "name": "Valid Name",
+        "missing_name": "Valid Name",
         "age": "not_a_number",
-        "missing_email": True,
     }
     assert invalid_data.file_id == "error-file"
     assert invalid_data.file_name == "bad_data.pdf"
@@ -590,3 +590,26 @@ def test_full_parse_nested_dimensions():
     assert result.field_metadata == expected
     parsed = ExtractedData.model_validate_json(result.model_dump_json())
     assert parsed.field_metadata == expected
+
+
+def test_parses_field_metadata_with_error_field():
+    extract_run = create_extract_run(
+        extraction_metadata={
+            "name": {
+                "confidence": 0.95,
+                "citation": [{"page": 1, "matching_text": "John Smith"}],
+            },
+            "error": "This is an error",
+        },
+    )
+
+    parsed = ExtractedData.from_extraction_result(extract_run, Person)
+
+    assert parsed.field_metadata == {
+        "name": ExtractedFieldMetadata(
+            confidence=0.95,
+            citation=[FieldCitation(page=1, matching_text="John Smith")],
+        ),
+    }
+    assert parsed.metadata.get("field_errors") == "This is an error"
+    assert parsed.metadata.get("job_id") == "job-123"

py/uv.lock

@@ -741,6 +741,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" },
 ]
 
+[[package]]
+name = "execnet"
+version = "2.1.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/bb/ff/b4c0dc78fbe20c3e59c0c7334de0c27eb4001a2b2017999af398bf730817/execnet-2.1.1.tar.gz", hash = "sha256:5189b52c6121c24feae288166ab41b32549c7e2348652736540b9e6e7d4e72e3", size = 166524, upload-time = "2024-04-08T09:04:19.245Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/43/09/2aea36ff60d16dd8879bdb2f5b3ee0ba8d08cbbdcdfe870e695ce3784385/execnet-2.1.1-py3-none-any.whl", hash = "sha256:26dee51f1b80cebd6d0ca8e74dd8745419761d3bef34163928cbebbdc4749fdc", size = 40612, upload-time = "2024-04-08T09:04:17.414Z" },
+]
+
 [[package]]
 name = "executing"
 version = "2.2.0"
@@ -1587,7 +1596,7 @@ wheels = [
 
 [[package]]
 name = "llama-cloud-services"
-version = "0.6.58"
+version = "0.6.59"
 source = { editable = "." }
 dependencies = [
     { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
@@ -1615,6 +1624,7 @@ dev = [
     { name = "pydantic-settings" },
     { name = "pytest" },
     { name = "pytest-asyncio" },
+    { name = "pytest-xdist" },
 ]
 
 [package.metadata]
@@ -1642,6 +1652,7 @@ dev = [
     { name = "pydantic-settings", specifier = ">=2.10.1" },
     { name = "pytest", specifier = ">=8.0.0,<9" },
     { name = "pytest-asyncio" },
+    { name = "pytest-xdist", specifier = ">=3.6.1,<4" },
 ]
 
 [[package]]
@@ -2912,6 +2923,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c7/9d/bf86eddabf8c6c9cb1ea9a869d6873b46f105a5d292d3a6f7071f5b07935/pytest_asyncio-1.1.0-py3-none-any.whl", hash = "sha256:5fe2d69607b0bd75c656d1211f969cadba035030156745ee09e7d71740e58ecf", size = 15157, upload-time = "2025-07-16T04:29:24.929Z" },
 ]
 
+[[package]]
+name = "pytest-xdist"
+version = "3.8.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "execnet" },
+    { name = "pytest" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" },
+]
+
 [[package]]
 name = "python-dateutil"
 version = "2.9.0.post0"

ts/llama_cloud_services/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llama-cloud-services",
-  "version": "0.3.2",
+  "version": "0.3.3",
   "type": "module",
   "license": "MIT",
   "scripts": {

ts/llama_cloud_services/src/beta/agent/index.ts

@@ -4,6 +4,8 @@ export type {
   AggregateAgentDataOptions,
   ComparisonOperator,
   ExtractedData,
+  ExtractedFieldMetadata,
+  ExtractedFieldMetadataDict,
   FilterOperation,
   SearchAgentDataOptions,
   StatusType,