fix parse model

* Run hourly extract tests to catch SDK schema drifts

* fix url

* fix prod/staging env

.github/workflows/hourly_extract_e2e.yml

@@ -0,0 +1,162 @@
+name: Hourly Extract E2E Tests
+
+on:
+  schedule:
+    - cron: "18 * * * *"
+  workflow_dispatch:
+    # Allows manual triggering
+    inputs:
+      environment:
+        description: "Environment to run the tests in"
+        required: false
+        default: staging
+        type: choice
+        options:
+          - staging
+          - production
+      notify_slack:
+        description: "Notify Slack"
+        required: false
+        default: false
+        type: boolean
+  workflow_call:
+
+env:
+  UV_VERSION: "0.7.20"
+  PYTHON_VERSION: "3.12"
+  SLACK_CHANNEL_ID: C078PHNTF44 # Extract channel ID
+  API_E2E_LOG_PATH: ${{ github.workspace }}/extract-e2e.log
+
+jobs:
+  extract-e2e:
+    name: "Hourly Extract E2E Tests (${{ matrix.environment }})"
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.environment }}
+      cancel-in-progress: true
+    strategy:
+      fail-fast: false
+      matrix:
+        environment: ${{ github.event_name == 'schedule' && fromJson('["staging", "production"]') || fromJson(format('["{0}"]', github.event.inputs.environment || 'staging')) }}
+    steps:
+      - name: Set runtime inputs
+        id: runtime
+        run: |
+          environment=${{ matrix.environment }}
+          notify_slack=${{ github.event.inputs.notify_slack || github.event_name == 'schedule' }}
+          echo "environment=${environment}" >> $GITHUB_OUTPUT
+          echo "notify_slack=${notify_slack}" >> $GITHUB_OUTPUT
+
+          if [ "${environment}" = "production" ]; then
+            echo "LLAMA_CLOUD_BASE_URL=https://api.cloud.llamaindex.ai" >> $GITHUB_ENV
+            api_key_secret="${{ secrets.LLAMA_CLOUD_API_KEY }}"
+            project_id_secret="${{ secrets.LLAMA_CLOUD_PROJECT_ID }}"
+          else
+            echo "LLAMA_CLOUD_BASE_URL=https://api.staging.llamaindex.ai" >> $GITHUB_ENV
+            api_key_secret="${{ secrets.LLAMA_CLOUD_API_KEY_STAGING }}"
+            project_id_secret="${{ secrets.LLAMA_CLOUD_PROJECT_ID_STAGING }}"
+          fi
+
+          if [ -n "$api_key_secret" ]; then
+            echo "LLAMA_CLOUD_API_KEY=$api_key_secret" >> $GITHUB_ENV
+          fi
+
+          if [ -n "$project_id_secret" ]; then
+            echo "LLAMA_CLOUD_PROJECT_ID=$project_id_secret" >> $GITHUB_ENV
+          fi
+
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          version: ${{ env.UV_VERSION }}
+
+      - name: Set up Python
+        run: uv python install ${{ env.PYTHON_VERSION }} && uv python pin ${{ env.PYTHON_VERSION }}
+
+      - name: Run Extract E2E tests
+        id: extract-tests
+        continue-on-error: true
+        working-directory: py
+        run: |
+          set -o pipefail
+          rm -f "$API_E2E_LOG_PATH"
+          uv run pytest -v -n 8 --timeout=300 --session-timeout=1740 tests/extract/ 2>&1 | tee "$API_E2E_LOG_PATH"
+
+      - name: Extract pytest failure summary
+        id: failed-tests
+        if: steps.extract-tests.outcome == 'failure' || cancelled()
+        run: |
+          summary="$(python3 - <<'PY'
+          import os
+          import re
+          from pathlib import Path
+
+          log_path = Path(os.environ["API_E2E_LOG_PATH"])
+          if not log_path.exists():
+              print("Test log not found.")
+              raise SystemExit(0)
+
+          lines = log_path.read_text(errors="ignore").splitlines()
+
+          # Find the "short test summary info" section
+          start = None
+          for i, line in enumerate(lines):
+              if line.startswith("=") and "short test summary info" in line:
+                  start = i + 1
+                  break
+
+          if start is None:
+              print("No test summary found.")
+              raise SystemExit(0)
+
+          # Extract just the FAILED/ERROR lines (test name + short reason)
+          failed_tests = []
+          for line in lines[start:]:
+              if line.startswith("="):
+                  break  # End of section
+              if line.startswith("FAILED ") or line.startswith("ERROR "):
+                  # Extract test name and truncate the error message
+                  match = re.match(r"(FAILED|ERROR) ([\w/:.\[\]_-]+)", line)
+                  if match:
+                      failed_tests.append(f"{match.group(1)}: {match.group(2)}")
+
+          if failed_tests:
+              print("\n".join(failed_tests[:20]))  # Limit to 20 tests max
+          else:
+              print("No failed tests found in summary.")
+          PY
+          )"
+          if [ -z "$summary" ]; then
+            summary="Failed test summary not available. Review the full run logs."
+          fi
+          {
+            printf 'summary<<EOF\n%s\nEOF\n' "$summary"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Check test results
+        if: always()
+        run: |
+          if [ "${{ steps.extract-tests.outcome }}" == "failure" ]; then
+            echo "Extract E2E tests failed"
+            exit 1
+          fi
+
+      - name: Post to Extract Slack channel
+        id: slack
+        if: (failure() || cancelled()) && steps.runtime.outputs.notify_slack == 'true'
+        uses: slackapi/slack-github-action@v1.27.0
+        with:
+          channel-id: ${{ env.SLACK_CHANNEL_ID }}
+          slack-message: |
+            :red_circle: *Extract E2E Failed* (${{ steps.runtime.outputs.environment }})
+            ```
+            ${{ steps.failed-tests.outputs.summary }}
+            ```
+            <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

py/CHANGELOG.md

@@ -1,5 +1,12 @@
 # llama-cloud-services-py
 
+## 0.6.91
+
+### Patch Changes
+
+- 07ec282: Bump up patch versions for python packages
+- 3040951: Use error description in ExtractedData invalid extraction error
+
 ## 0.6.90
 
 ### Patch Changes

py/llama_cloud_services/beta/agent_data/schema.py

@@ -475,26 +475,49 @@ class ExtractedData(BaseModel, Generic[ExtractedT]):
                 },
             )
         except ValidationError as e:
+            # Capture the job-level error from the extraction run if available
+            job_error = result.error
+
             invalid_item = ExtractedData[Dict[str, Any]].create(
                 data=result.data or {},
                 status="error",
                 field_metadata=field_metadata,
-                metadata={"extraction_error": str(e), **(metadata or {})},
+                metadata={
+                    "extraction_error": str(e),
+                    **({"job_error": job_error} if job_error else {}),
+                    **(metadata or {}),
+                },
                 file_id=file_id,
                 file_name=file_name,
                 file_hash=file_hash,
             )
-            raise InvalidExtractionData(invalid_item) from e
+            raise InvalidExtractionData(invalid_item, extraction_error=job_error) from e
 
 
 class InvalidExtractionData(Exception):
     """
     Exception raised when the extracted data does not conform to the schema.
+
+    Attributes:
+        invalid_item: The ExtractedData instance containing the invalid data and metadata
+        extraction_error: The error message from the extraction job, if available
     """
 
-    def __init__(self, invalid_item: ExtractedData[Dict[str, Any]]):
+    def __init__(
+        self,
+        invalid_item: ExtractedData[Dict[str, Any]],
+        extraction_error: Optional[str] = None,
+    ):
         self.invalid_item = invalid_item
-        super().__init__("Not able to parse the extracted data, parsed invalid format")
+        self.extraction_error = extraction_error
+
+        # Build an informative error message
+        if extraction_error:
+            message = f"Extraction error: {extraction_error}"
+        else:
+            message = "Not able to parse the extracted data, parsed invalid format"
+
+        super().__init__(message)
 
 
 def calculate_overall_confidence(

py/llama_cloud_services/extract/extract.py

@@ -4,10 +4,11 @@ import os
 import time
 from io import BufferedIOBase, TextIOWrapper
 from pathlib import Path
-from typing import List, Optional, Type, Union, Coroutine, Any, TypeVar
+from typing import Callable, List, Optional, Type, Union, Coroutine, Any, TypeVar
 import warnings
 import httpx
 from pydantic import BaseModel
+from functools import wraps
 from tenacity import (
     retry_if_exception,
     stop_after_attempt,
@@ -54,7 +55,7 @@ DEFAULT_EXTRACT_CONFIG = ExtractConfig(
 def _is_retryable_error(exception: BaseException) -> bool:
     """Check if an exception is retryable."""
     if isinstance(exception, ApiError):
-        return exception.status_code in (502, 503, 504, 425, 408)
+        return exception.status_code in (429, 500, 502, 503, 504, 425, 408)
     elif isinstance(
         exception, (httpx.HTTPStatusError, httpx.RequestError, httpx.TimeoutException)
     ):
@@ -62,6 +63,33 @@ def _is_retryable_error(exception: BaseException) -> bool:
     return False
 
 
+def _async_retry(
+    max_attempts: int = 5,
+    initial_wait: float = 1,
+    max_wait: float = 30,
+    jitter: float = 3,
+) -> Callable:
+    """Decorator for async functions with retry logic for rate limiting and transient errors."""
+
+    def decorator(func: Callable) -> Callable:
+        @wraps(func)
+        async def wrapper(*args: Any, **kwargs: Any) -> Any:
+            async for attempt in AsyncRetrying(
+                retry=retry_if_exception(_is_retryable_error),
+                stop=stop_after_attempt(max_attempts),
+                wait=wait_exponential_jitter(
+                    initial=initial_wait, max=max_wait, jitter=jitter
+                ),
+                reraise=True,
+            ):
+                with attempt:
+                    return await func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
 async def _validate_schema(
     client: AsyncLlamaCloud, data_schema: SchemaInput
 ) -> JSONObjectType:
@@ -82,50 +110,6 @@ async def _validate_schema(
     return validated_schema.data_schema
 
 
-async def _get_job_with_retry(
-    client: AsyncLlamaCloud,
-    job_id: str,
-    max_attempts: int = 5,
-    initial_wait: float = 1,
-    max_wait: float = 60,
-    jitter: float = 5,
-) -> ExtractJob:
-    """Get extraction job with retry logic."""
-    async for attempt in AsyncRetrying(
-        retry=retry_if_exception(_is_retryable_error),
-        stop=stop_after_attempt(max_attempts),
-        wait=wait_exponential_jitter(initial=initial_wait, max=max_wait, jitter=jitter),
-        reraise=True,
-    ):
-        with attempt:
-            return await client.llama_extract.get_job(job_id=job_id)
-
-
-async def _get_run_with_retry(
-    client: AsyncLlamaCloud,
-    job_id: str,
-    project_id: Optional[str] = None,
-    organization_id: Optional[str] = None,
-    max_attempts: int = 3,
-    initial_wait: float = 1,
-    max_wait: float = 20,
-    jitter: float = 3,
-) -> ExtractRun:
-    """Get extraction run with retry logic."""
-    async for attempt in AsyncRetrying(
-        retry=retry_if_exception(_is_retryable_error),
-        stop=stop_after_attempt(max_attempts),
-        wait=wait_exponential_jitter(initial=initial_wait, max=max_wait, jitter=jitter),
-        reraise=True,
-    ):
-        with attempt:
-            return await client.llama_extract.get_run_by_job_id(
-                job_id=job_id,
-                project_id=project_id,
-                organization_id=organization_id,
-            )
-
-
 async def _wait_for_job_result(
     client: AsyncLlamaCloud,
     job_id: str,
@@ -142,30 +126,33 @@ async def _wait_for_job_result(
     run_jitter: float = 3,
 ) -> Optional[ExtractRun]:
     """Wait for and return the results of an extraction job."""
+
+    @_async_retry(
+        max_attempts=job_retry_attempts, max_wait=job_max_wait, jitter=job_jitter
+    )
+    async def _get_job() -> ExtractJob:
+        return await client.llama_extract.get_job(job_id=job_id)
+
+    @_async_retry(
+        max_attempts=run_retry_attempts, max_wait=run_max_wait, jitter=run_jitter
+    )
+    async def _get_run() -> ExtractRun:
+        return await client.llama_extract.get_run_by_job_id(
+            job_id=job_id,
+            project_id=project_id,
+            organization_id=organization_id,
+        )
+
     start = time.perf_counter()
     poll_count = 0
 
     while True:
         await asyncio.sleep(check_interval)
         poll_count += 1
-        job = await _get_job_with_retry(
-            client,
-            job_id,
-            max_attempts=job_retry_attempts,
-            max_wait=job_max_wait,
-            jitter=job_jitter,
-        )
+        job = await _get_job()
 
         if job.status == StatusEnum.SUCCESS:
-            return await _get_run_with_retry(
-                client,
-                job_id,
-                project_id,
-                organization_id,
-                max_attempts=run_retry_attempts,
-                max_wait=run_max_wait,
-                jitter=run_jitter,
-            )
+            return await _get_run()
         elif job.status == StatusEnum.PENDING:
             end = time.perf_counter()
             if end - start > max_timeout:
@@ -177,15 +164,7 @@ async def _wait_for_job_result(
             warnings.warn(
                 f"Failure in job: {job_id}, status: {job.status}, error: {job.error}"
             )
-            return await _get_run_with_retry(
-                client,
-                job_id,
-                project_id,
-                organization_id,
-                max_attempts=run_retry_attempts,
-                max_wait=run_max_wait,
-                jitter=run_jitter,
-            )
+            return await _get_run()
 
 
 def run_in_thread(
@@ -498,9 +477,12 @@ class ExtractionAgent:
         Args:
             run_id (str): The ID of the extraction run to delete
         """
-        self._run_in_thread(
-            self._client.llama_extract.delete_extraction_run(run_id=run_id)
-        )
+
+        @_async_retry()
+        async def _delete() -> None:
+            return await self._client.llama_extract.delete_extraction_run(run_id=run_id)
+
+        self._run_in_thread(_delete())
 
     def list_extraction_runs(
         self, page: int = 0, limit: int = 100
@@ -510,13 +492,16 @@ class ExtractionAgent:
         Returns:
             PaginatedExtractRunsResponse: Paginated list of extraction runs
         """
-        return self._run_in_thread(
-            self._client.llama_extract.list_extract_runs(
+
+        @_async_retry()
+        async def _list() -> PaginatedExtractRunsResponse:
+            return await self._client.llama_extract.list_extract_runs(
                 extraction_agent_id=self.id,
                 skip=page * limit,
                 limit=limit,
             )
-        )
+
+        return self._run_in_thread(_list())
 
     def __repr__(self) -> str:
         return f"ExtractionAgent(id={self.id}, name={self.name})"
@@ -658,15 +643,17 @@ class LlamaExtract(BaseComponent):
                 "data_schema must be either a dictionary or a Pydantic model"
             )
 
-        agent = self._run_in_thread(
-            self._async_client.llama_extract.create_extraction_agent(
+        @_async_retry()
+        async def _create() -> CloudExtractAgent:
+            return await self._async_client.llama_extract.create_extraction_agent(
                 project_id=self._project_id,
                 organization_id=self._organization_id,
                 name=name,
                 data_schema=data_schema,
                 config=config,
             )
-        )
+
+        agent = self._run_in_thread(_create())
 
         return ExtractionAgent(
             client=self._async_client,
@@ -702,19 +689,27 @@ class LlamaExtract(BaseComponent):
             )
 
         if id:
-            agent = self._run_in_thread(
-                self._async_client.llama_extract.get_extraction_agent(
+
+            @_async_retry()
+            async def _get_by_id() -> CloudExtractAgent:
+                return await self._async_client.llama_extract.get_extraction_agent(
                     extraction_agent_id=id,
                 )
-            )
+
+            agent = self._run_in_thread(_get_by_id())
 
         elif name:
-            agent = self._run_in_thread(
-                self._async_client.llama_extract.get_extraction_agent_by_name(
-                    name=name,
-                    project_id=self._project_id,
+
+            @_async_retry()
+            async def _get_by_name() -> CloudExtractAgent:
+                return (
+                    await self._async_client.llama_extract.get_extraction_agent_by_name(
+                        name=name,
+                        project_id=self._project_id,
+                    )
                 )
-            )
+
+            agent = self._run_in_thread(_get_by_name())
         else:
             raise ValueError("Either name or extraction_agent_id must be provided.")
 
@@ -734,11 +729,14 @@ class LlamaExtract(BaseComponent):
 
     def list_agents(self) -> List[ExtractionAgent]:
         """List all available extraction agents."""
-        agents = self._run_in_thread(
-            self._async_client.llama_extract.list_extraction_agents(
+
+        @_async_retry()
+        async def _list() -> List[CloudExtractAgent]:
+            return await self._async_client.llama_extract.list_extraction_agents(
                 project_id=self._project_id,
             )
-        )
+
+        agents = self._run_in_thread(_list())
 
         return [
             ExtractionAgent(
@@ -763,11 +761,14 @@ class LlamaExtract(BaseComponent):
         Args:
             agent_id (str): ID of the extraction agent to delete
         """
-        self._run_in_thread(
-            self._async_client.llama_extract.delete_extraction_agent(
-                extraction_agent_id=agent_id
+
+        @_async_retry()
+        async def _delete() -> None:
+            return await self._async_client.llama_extract.delete_extraction_agent(
+                extraction_agent_id=agent_id,
             )
-        )
+
+        self._run_in_thread(_delete())
 
     async def _wait_for_job_result(self, job_id: str) -> Optional[ExtractRun]:
         """Wait for and return the results of an extraction job."""

py/llama_cloud_services/files/client.py

@@ -3,7 +3,7 @@ from typing import BinaryIO
 import os
 from pathlib import Path
 from llama_cloud.client import AsyncLlamaCloud
-from llama_cloud.types import File, FileCreate
+from llama_cloud.types import File
 from typing import Optional
 from llama_cloud_services.utils import SourceText, FileInput
 
@@ -73,11 +73,9 @@ class FileClient:
             presigned_url = await self.client.files.generate_presigned_url(
                 project_id=self.project_id,
                 organization_id=self.organization_id,
-                request=FileCreate(
-                    name=name,
-                    external_file_id=external_file_id,
-                    file_size=file_size,
-                ),
+                name=name,
+                external_file_id=external_file_id,
+                file_size=file_size,
             )
             httpx_client = self.client._client_wrapper.httpx_client
             upload_response = await httpx_client.put(

py/llama_cloud_services/index/base.py

@@ -21,7 +21,6 @@ from llama_cloud import (
     PipelineCreateTransformConfig,
     PipelineFileCreateCustomMetadataValue,
     PipelineType,
-    ProjectCreate,
     ManagedIngestionStatus,
     CloudDocumentCreate,
     CloudDocument,
@@ -507,14 +506,19 @@ class LlamaCloudIndex(BaseManagedIndex):
         client = get_client(api_key, base_url, app_url, timeout)
 
         if project_id is None:
-            # create project if it doesn't exist
-            project = client.projects.upsert_project(
+            # get project by name
+            projects = client.projects.list_projects(
                 organization_id=organization_id,
-                request=ProjectCreate(name=project_name),
+                project_name=project_name,
             )
+            if not projects:
+                raise ValueError(
+                    f"Project '{project_name}' not found. Please create it first in the LlamaCloud UI."
+                )
+            project = projects[0]
             project_id = project.id
             if verbose:
-                print(f"Created project {project_id} with name {project_name}")
+                print(f"Found project {project_id} with name {project_name}")
 
         # create pipeline
         pipeline_create = PipelineCreate(
@@ -563,15 +567,20 @@ class LlamaCloudIndex(BaseManagedIndex):
         app_url = app_url or os.environ.get("LLAMA_CLOUD_APP_URL", DEFAULT_APP_URL)
         aclient = get_aclient(api_key, base_url, app_url, timeout)
 
-        # create project if it doesn't exist
-        project = await aclient.projects.upsert_project(
-            organization_id=organization_id, request=ProjectCreate(name=project_name)
+        # get project by name
+        projects = await aclient.projects.list_projects(
+            organization_id=organization_id, project_name=project_name
         )
+        if not projects:
+            raise ValueError(
+                f"Project '{project_name}' not found. Please create it first in the LlamaCloud UI."
+            )
+        project = projects[0]
         if project.id is None:
-            raise ValueError(f"Failed to create/get project {project_name}")
+            raise ValueError(f"Failed to get project {project_name}")
 
         if verbose:
-            print(f"Created project {project.id} with name {project.name}")
+            print(f"Found project {project.id} with name {project.name}")
 
         # create pipeline
         pipeline_create = PipelineCreate(

py/llama_parse/CHANGELOG.md

@@ -1,5 +1,14 @@
 # llama_parse
 
+## 0.6.91
+
+### Patch Changes
+
+- 07ec282: Bump up patch versions for python packages
+- Updated dependencies [07ec282]
+- Updated dependencies [3040951]
+  - llama-cloud-services-py@0.6.91
+
 ## 0.6.90
 
 ### Patch Changes

py/llama_parse/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llama_parse",
-  "version": "0.6.90",
+  "version": "0.6.91",
   "description": "",
   "main": "index.js",
   "private": false,

py/llama_parse/pyproject.toml

@@ -11,13 +11,13 @@ dev = [
 
 [project]
 name = "llama-parse"
-version = "0.6.90"
+version = "0.6.91"
 description = "Parse files into RAG-Optimized formats."
 authors = [{name = "Logan Markewich", email = "logan@llamaindex.ai"}]
 requires-python = ">=3.9,<4.0"
 readme = "README.md"
 license = "MIT"
-dependencies = ["llama-cloud-services>=0.6.90"]
+dependencies = ["llama-cloud-services>=0.6.91"]
 
 [project.scripts]
 llama-parse = "llama_parse.cli.main:parse"

py/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llama-cloud-services-py",
-  "version": "0.6.90",
+  "version": "0.6.91",
   "private": false,
   "license": "MIT",
   "scripts": {},

py/pyproject.toml

@@ -23,7 +23,7 @@ dev = [
 
 [project]
 name = "llama-cloud-services"
-version = "0.6.90"
+version = "0.6.91"
 description = "Tailored SDK clients for LlamaCloud services."
 authors = [{name = "Logan Markewich", email = "logan@runllama.ai"}]
 requires-python = ">=3.9,<4.0"
@@ -31,7 +31,7 @@ readme = "README.md"
 license = "MIT"
 dependencies = [
   "llama-index-core>=0.12.0",
-  "llama-cloud==0.1.45",
+  "llama-cloud==0.1.46",
   "pydantic>=2.8,!=2.10",
   "click>=8.1.7,<9",
   "python-dotenv>=1.0.1,<2",

py/tests/extract/conftest.py

@@ -1,11 +1,43 @@
 import os
-from typing import List
-from llama_cloud_services.extract import LlamaExtract
+from typing import Any, Dict, List, Optional, Union
+
+from llama_cloud.core.api_error import ApiError
+from llama_cloud.types import ExtractConfig
+from pydantic import BaseModel
+from tenacity import (
+    retry,
+    retry_if_exception,
+    stop_after_attempt,
+    wait_exponential,
+)
+
+from llama_cloud_services.extract import ExtractionAgent, LlamaExtract
 
 # Global storage for agents to cleanup
 _TEST_AGENTS_TO_CLEANUP: List[str] = []
 
 
+def _is_rate_limit_error(exception: BaseException) -> bool:
+    """Check if the exception is a rate limit error (429)."""
+    return isinstance(exception, ApiError) and exception.status_code == 429
+
+
+@retry(
+    retry=retry_if_exception(_is_rate_limit_error),
+    wait=wait_exponential(multiplier=1, min=1, max=30),
+    stop=stop_after_attempt(5),
+    reraise=True,
+)
+def create_agent_with_retry(
+    extractor: LlamaExtract,
+    name: str,
+    data_schema: Union[Dict[str, Any], type[BaseModel]],
+    config: Optional[ExtractConfig] = None,
+) -> ExtractionAgent:
+    """Create an extraction agent with retry logic for rate limiting."""
+    return extractor.create_agent(name=name, data_schema=data_schema, config=config)
+
+
 def pytest_configure(config):
     """Register custom markers for extract tests."""
     config.addinivalue_line("markers", "agent_name: custom agent name for test")

py/tests/extract/test_extract_api.py

@@ -6,7 +6,7 @@ from pydantic import BaseModel
 from llama_cloud_services.extract import LlamaExtract, ExtractionAgent, SourceText
 from llama_cloud.types import ExtractConfig, ExtractMode, ExtractRun
 from tests.extract.util import load_test_dotenv
-from .conftest import register_agent_for_cleanup
+from .conftest import register_agent_for_cleanup, create_agent_with_retry
 
 load_test_dotenv()
 
@@ -87,7 +87,11 @@ def test_agent(llama_extract, test_agent_name, test_schema_dict, request):
     except Exception as e:
         print(f"Warning: Failed to cleanup existing agent: {e}")
 
-    agent = llama_extract.create_agent(name=name, data_schema=schema)
+    # Use config with cache invalidation to ensure fresh results in tests
+    config = ExtractConfig(invalidate_cache=True)
+    agent = create_agent_with_retry(
+        llama_extract, name=name, data_schema=schema, config=config
+    )
 
     # Add agent to cleanup list via conftest helper
     register_agent_for_cleanup(agent.id)
@@ -237,7 +241,7 @@ class TestStatelessExtraction:
 
     @pytest.fixture
     def test_config(self):
-        return ExtractConfig(extraction_mode=ExtractMode.FAST)
+        return ExtractConfig(extraction_mode=ExtractMode.FAST, invalidate_cache=True)
 
     @pytest.fixture
     def test_schema_dict(self):

py/tests/extract/test_extract_e2e.py

@@ -8,7 +8,7 @@ import uuid
 from llama_cloud.types import ExtractConfig, ExtractMode
 from deepdiff import DeepDiff
 from tests.extract.util import json_subset_match_score, load_test_dotenv
-from .conftest import register_agent_for_cleanup
+from .conftest import register_agent_for_cleanup, create_agent_with_retry
 
 load_test_dotenv()
 
@@ -56,10 +56,16 @@ def get_test_cases():
             input_files.append(file_path)
 
         settings = [
-            ExtractConfig(extraction_mode=ExtractMode.FAST),
-            ExtractConfig(extraction_mode=ExtractMode.BALANCED),
-            ExtractConfig(extraction_mode=ExtractMode.MULTIMODAL),
-            ExtractConfig(extraction_mode=ExtractMode.PREMIUM),
+            ExtractConfig(extraction_mode=ExtractMode.FAST, invalidate_cache=True),
+            ExtractConfig(extraction_mode=ExtractMode.BALANCED, invalidate_cache=True),
+            ExtractConfig(
+                extraction_mode=ExtractMode.MULTIMODAL, invalidate_cache=True
+            ),
+            ExtractConfig(
+                extraction_mode=ExtractMode.PREMIUM,
+                invalidate_cache=True,
+                parse_model="anthropic-sonnet-4.5",
+            ),
         ]
 
         for input_file in sorted(input_files):
@@ -117,8 +123,10 @@ def extraction_agent(test_case: ExtractionTestCase, extractor: LlamaExtract):
     except Exception as e:
         print(f"Warning: Failed to cleanup existing agent: {str(e)}")
 
-    # Create new agent
-    agent = extractor.create_agent(agent_name, schema, config=test_case.config)
+    # Create new agent with retry logic for rate limiting
+    agent = create_agent_with_retry(
+        extractor, name=agent_name, data_schema=schema, config=test_case.config
+    )
 
     # Register agent for cleanup at the end of the test session
     register_agent_for_cleanup(agent.id)

py/tests/index/test_index.py

@@ -8,7 +8,6 @@ from llama_cloud import (
     AutoTransformConfig,
     PipelineCreate,
     PipelineFileCreate,
-    ProjectCreate,
     CompositeRetrievalMode,
     LlamaParseParameters,
     ReRankConfig,
@@ -60,11 +59,15 @@ def local_figures_file() -> str:
 def _setup_index_with_file(
     client: LlamaCloud, index_name: str, remote_file: Tuple[str, str]
 ) -> LlamaCloudIndex:
-    # create project if it doesn't exist
-    project_create = ProjectCreate(name=project_name)
-    project = client.projects.upsert_project(
-        organization_id=organization_id, request=project_create
+    # get project by name
+    projects = client.projects.list_projects(
+        organization_id=organization_id, project_name=project_name
     )
+    if not projects:
+        raise ValueError(
+            f"Project '{project_name}' not found. Please create it first in the LlamaCloud UI."
+        )
+    project = projects[0]
 
     # create pipeline
     pipeline_create = PipelineCreate(

py/unit_tests/beta/agent/test_agent_data_schema.py

@@ -423,6 +423,7 @@ def create_extract_run(
     },
     data_schema: Dict[str, Any] = {},
     file: File = create_file(),
+    error: Optional[str] = None,
 ) -> ExtractRun:
     return ExtractRun.parse_obj(
         {
@@ -439,6 +440,7 @@ def create_extract_run(
             "status": "SUCCESS",
             "project_id": str(uuid.uuid4()),
             "from_ui": False,
+            "error": error,
         }
     )
 
@@ -544,6 +546,46 @@ def test_extracted_data_from_extraction_result_invalid_data():
     assert invalid_data.field_metadata["name"].confidence == 0.9
     assert invalid_data.overall_confidence == 0.9
 
+    # Verify default error message when no job error present
+    assert exc_info.value.extraction_error is None
+    assert "Not able to parse the extracted data" in str(exc_info.value)
+
+
+def test_extracted_data_from_extraction_result_with_job_error():
+    """Test ExtractedData.from_extraction_result with job-level error prominently displayed."""
+    job_error_message = "Failed to process document: unsupported file format"
+
+    # Create ExtractRun with both invalid data AND a job-level error
+    extract_run = create_extract_run(
+        data={
+            "missing_name": "Valid Name",
+            "age": "not_a_number",
+        },  # Invalid age, missing name
+        extraction_metadata={
+            "name": {"confidence": 0.9},
+        },
+        data_schema={},
+        file=create_file(id="error-file", name="bad_data.pdf"),
+        error=job_error_message,
+    )
+
+    # Should raise InvalidExtractionData with the job error prominently displayed
+    with pytest.raises(InvalidExtractionData) as exc_info:
+        ExtractedData.from_extraction_result(
+            extract_run, Person, metadata={"test": "metadata"}
+        )
+
+    # Verify the exception message prominently shows the job error
+    exception = exc_info.value
+    assert exception.extraction_error == job_error_message
+    assert f"Extraction error: {job_error_message}" == str(exception)
+
+    # Verify the invalid_item contains both errors in metadata
+    invalid_data = exception.invalid_item
+    assert invalid_data.metadata.get("job_error") == job_error_message
+    assert "extraction_error" in invalid_data.metadata  # Validation error still present
+    assert "test" in invalid_data.metadata  # Original metadata preserved
+
 
 class Dimensions(BaseModel):
     length: Optional[str] = Field(

py/unit_tests/index/test_unit_index.py

@@ -34,9 +34,10 @@ TEST_PIPELINE = Pipeline(
 def mock_client() -> MagicMock:
     """Mock client with sensible defaults."""
     client = MagicMock()
-    client.projects.upsert_project.return_value = Project(
+    default_project = Project(
         id="default-proj", name=DEFAULT_PROJECT_NAME, organization_id="default-org"
     )
+    client.projects.list_projects.return_value = [default_project]
     client.pipelines.upsert_pipeline.return_value = Pipeline(
         id="default-pipe",
         name="default",
@@ -100,8 +101,8 @@ def test_from_documents_uses_provided_project_id(mock_client: MagicMock) -> None
             project_id=provided_project_id,
         )
 
-    # Assert - project upsert not called; pipeline uses provided project_id
-    mock_client.projects.upsert_project.assert_not_called()
+    # Assert - project list not called (project_id provided); pipeline uses provided project_id
+    mock_client.projects.list_projects.assert_not_called()
     assert mock_client.pipelines.upsert_pipeline.call_count == 1
     assert (
         mock_client.pipelines.upsert_pipeline.call_args.kwargs["project_id"]
@@ -110,29 +111,29 @@ def test_from_documents_uses_provided_project_id(mock_client: MagicMock) -> None
     assert index.project.id == provided_project_id
 
 
-def test_from_documents_upserts_project_when_project_id_missing(
+def test_from_documents_lists_project_when_project_id_missing(
     mock_client: MagicMock,
 ) -> None:
     organization_id = "org-xyz"
     index_name = "my_new_index"
 
-    # Project is created when project_id is not provided
-    upserted_project = Project(
+    # Project is found by name when project_id is not provided
+    found_project = Project(
         id="proj-999", name=DEFAULT_PROJECT_NAME, organization_id=organization_id
     )
-    mock_client.projects.upsert_project.return_value = upserted_project
+    mock_client.projects.list_projects.return_value = [found_project]
 
     test_pipeline = Pipeline(
         id="pipe-xyz",
         name=index_name,
-        project_id=upserted_project.id,
+        project_id=found_project.id,
         embedding_config=EMBEDDING_CONFIG,
     )
 
     with patch.object(
         base,
         "resolve_project_and_pipeline",
-        return_value=(upserted_project, test_pipeline),
+        return_value=(found_project, test_pipeline),
     ):
         docs = [Document(text="world")]
         index = LlamaCloudIndex.from_documents(
@@ -141,15 +142,15 @@ def test_from_documents_upserts_project_when_project_id_missing(
             organization_id=organization_id,
         )
 
-    # Assert - project was upserted with org id and default project name
-    mock_client.projects.upsert_project.assert_called_once()
-    kwargs = mock_client.projects.upsert_project.call_args.kwargs
+    # Assert - project was listed with org id and default project name
+    mock_client.projects.list_projects.assert_called_once()
+    kwargs = mock_client.projects.list_projects.call_args.kwargs
     assert kwargs["organization_id"] == organization_id
-    assert kwargs["request"].name == DEFAULT_PROJECT_NAME
+    assert kwargs["project_name"] == DEFAULT_PROJECT_NAME
 
-    # Pipeline created under the upserted project id
+    # Pipeline created under the found project id
     assert (
         mock_client.pipelines.upsert_pipeline.call_args.kwargs["project_id"]
-        == upserted_project.id
+        == found_project.id
     )
-    assert index.project.id == upserted_project.id
+    assert index.project.id == found_project.id

py/uv.lock

@@ -1595,21 +1595,21 @@ wheels = [
 
 [[package]]
 name = "llama-cloud"
-version = "0.1.45"
+version = "0.1.46"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "certifi" },
     { name = "httpx" },
     { name = "pydantic" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/e0/b7/3a2a209f1c3fa516de172cb13e03f5a897adea5523f2ee0f544d035e3704/llama_cloud-0.1.45.tar.gz", hash = "sha256:140244008cc5710e31ae97c6043973a3a9969a51b0f38155fa33a8434078e8aa", size = 140968, upload-time = "2025-12-03T02:22:49.484Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/40/f3/f4d6520f8d546e6c5a02f6ebeed5c09774a074b8d2c24ad559ace97a56a6/llama_cloud-0.1.46.tar.gz", hash = "sha256:e86f8791c053590d70cc59e0fc13ce72f9b681a8e658bc61df86d0285288d8ee", size = 127752, upload-time = "2026-01-21T18:40:57.103Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/62/1d/466b0df69b81ce9410ad6ec7229a1e6601ff69640f02f246e06cfcc7428c/llama_cloud-0.1.45-py3-none-any.whl", hash = "sha256:500299a6d3f25f97bcf6755d6338523023564fa8f376955c2cf299bbc9561cc2", size = 397184, upload-time = "2025-12-03T02:22:48.335Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/3a/6caaea28c8c804add33c91d356ed7d5a5412d6c9598e1450af95a15e0bcd/llama_cloud-0.1.46-py3-none-any.whl", hash = "sha256:6c6546c09c04a038c86d84d42f00eae8fd3bff49991ad3aab844bd866ecdf352", size = 361989, upload-time = "2026-01-21T18:40:54.863Z" },
 ]
 
 [[package]]
 name = "llama-cloud-services"
-version = "0.6.88"
+version = "0.6.90"
 source = { editable = "." }
 dependencies = [
     { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
@@ -1649,7 +1649,7 @@ dev = [
 requires-dist = [
     { name = "click", specifier = ">=8.1.7,<9" },
     { name = "eval-type-backport", marker = "python_full_version < '3.10'", specifier = ">=0.2.0,<0.3" },
-    { name = "llama-cloud", specifier = "==0.1.45" },
+    { name = "llama-cloud", specifier = "==0.1.46" },
     { name = "llama-index-core", specifier = ">=0.12.0" },
     { name = "packaging", specifier = ">=23.0" },
     { name = "platformdirs", specifier = ">=4.3.7,<5" },