mirror of
https://github.com/run-llama/llama_cloud_services.git
synced 2026-07-21 03:55:22 -04:00
Compare commits
30 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 39a93d602b | |||
| 0c04955c10 | |||
| 38da9a52d7 | |||
| 1e7ec40ee7 | |||
| dd83c1a9d0 | |||
| 7cb83f5cd3 | |||
| b05266be6d | |||
| eab4798165 | |||
| b174fa8fab | |||
| b12ffef916 | |||
| 07ec282257 | |||
| 013b689812 | |||
| 3040951cb8 | |||
| 9239498945 | |||
| 19cbb25631 | |||
| 812e2f7d72 | |||
| d7864afe3f | |||
| ade8d027a5 | |||
| 997bcc8531 | |||
| 8be554c234 | |||
| f777cab0c5 | |||
| b9b83c953d | |||
| 3ec7024626 | |||
| d5b18a03fa | |||
| 18dd04b6de | |||
| 685a5e6ccc | |||
| 576c3d9076 | |||
| c8321d2bc5 | |||
| 131bbed7aa | |||
| 41c8ac2348 |
@@ -0,0 +1,162 @@
|
||||
name: Hourly Extract E2E Tests
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "18 * * * *"
|
||||
workflow_dispatch:
|
||||
# Allows manual triggering
|
||||
inputs:
|
||||
environment:
|
||||
description: "Environment to run the tests in"
|
||||
required: false
|
||||
default: staging
|
||||
type: choice
|
||||
options:
|
||||
- staging
|
||||
- production
|
||||
notify_slack:
|
||||
description: "Notify Slack"
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
workflow_call:
|
||||
|
||||
env:
|
||||
UV_VERSION: "0.7.20"
|
||||
PYTHON_VERSION: "3.12"
|
||||
SLACK_CHANNEL_ID: C078PHNTF44 # Extract channel ID
|
||||
API_E2E_LOG_PATH: ${{ github.workspace }}/extract-e2e.log
|
||||
|
||||
jobs:
|
||||
extract-e2e:
|
||||
name: "Hourly Extract E2E Tests (${{ matrix.environment }})"
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.environment }}
|
||||
cancel-in-progress: true
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
environment: ${{ github.event_name == 'schedule' && fromJson('["staging", "production"]') || fromJson(format('["{0}"]', github.event.inputs.environment || 'staging')) }}
|
||||
steps:
|
||||
- name: Set runtime inputs
|
||||
id: runtime
|
||||
run: |
|
||||
environment=${{ matrix.environment }}
|
||||
notify_slack=${{ github.event.inputs.notify_slack || github.event_name == 'schedule' }}
|
||||
echo "environment=${environment}" >> $GITHUB_OUTPUT
|
||||
echo "notify_slack=${notify_slack}" >> $GITHUB_OUTPUT
|
||||
|
||||
if [ "${environment}" = "production" ]; then
|
||||
echo "LLAMA_CLOUD_BASE_URL=https://api.cloud.llamaindex.ai" >> $GITHUB_ENV
|
||||
api_key_secret="${{ secrets.LLAMA_CLOUD_API_KEY }}"
|
||||
project_id_secret="${{ secrets.LLAMA_CLOUD_PROJECT_ID }}"
|
||||
else
|
||||
echo "LLAMA_CLOUD_BASE_URL=https://api.staging.llamaindex.ai" >> $GITHUB_ENV
|
||||
api_key_secret="${{ secrets.LLAMA_CLOUD_API_KEY_STAGING }}"
|
||||
project_id_secret="${{ secrets.LLAMA_CLOUD_PROJECT_ID_STAGING }}"
|
||||
fi
|
||||
|
||||
if [ -n "$api_key_secret" ]; then
|
||||
echo "LLAMA_CLOUD_API_KEY=$api_key_secret" >> $GITHUB_ENV
|
||||
fi
|
||||
|
||||
if [ -n "$project_id_secret" ]; then
|
||||
echo "LLAMA_CLOUD_PROJECT_ID=$project_id_secret" >> $GITHUB_ENV
|
||||
fi
|
||||
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
version: ${{ env.UV_VERSION }}
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install ${{ env.PYTHON_VERSION }} && uv python pin ${{ env.PYTHON_VERSION }}
|
||||
|
||||
- name: Run Extract E2E tests
|
||||
id: extract-tests
|
||||
continue-on-error: true
|
||||
working-directory: py
|
||||
run: |
|
||||
set -o pipefail
|
||||
rm -f "$API_E2E_LOG_PATH"
|
||||
uv run pytest -v -n 8 --timeout=300 --session-timeout=1740 tests/extract/ 2>&1 | tee "$API_E2E_LOG_PATH"
|
||||
|
||||
- name: Extract pytest failure summary
|
||||
id: failed-tests
|
||||
if: steps.extract-tests.outcome == 'failure' || cancelled()
|
||||
run: |
|
||||
summary="$(python3 - <<'PY'
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
log_path = Path(os.environ["API_E2E_LOG_PATH"])
|
||||
if not log_path.exists():
|
||||
print("Test log not found.")
|
||||
raise SystemExit(0)
|
||||
|
||||
lines = log_path.read_text(errors="ignore").splitlines()
|
||||
|
||||
# Find the "short test summary info" section
|
||||
start = None
|
||||
for i, line in enumerate(lines):
|
||||
if line.startswith("=") and "short test summary info" in line:
|
||||
start = i + 1
|
||||
break
|
||||
|
||||
if start is None:
|
||||
print("No test summary found.")
|
||||
raise SystemExit(0)
|
||||
|
||||
# Extract just the FAILED/ERROR lines (test name + short reason)
|
||||
failed_tests = []
|
||||
for line in lines[start:]:
|
||||
if line.startswith("="):
|
||||
break # End of section
|
||||
if line.startswith("FAILED ") or line.startswith("ERROR "):
|
||||
# Extract test name and truncate the error message
|
||||
match = re.match(r"(FAILED|ERROR) ([\w/:.\[\]_-]+)", line)
|
||||
if match:
|
||||
failed_tests.append(f"{match.group(1)}: {match.group(2)}")
|
||||
|
||||
if failed_tests:
|
||||
print("\n".join(failed_tests[:20])) # Limit to 20 tests max
|
||||
else:
|
||||
print("No failed tests found in summary.")
|
||||
PY
|
||||
)"
|
||||
if [ -z "$summary" ]; then
|
||||
summary="Failed test summary not available. Review the full run logs."
|
||||
fi
|
||||
{
|
||||
printf 'summary<<EOF\n%s\nEOF\n' "$summary"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Check test results
|
||||
if: always()
|
||||
run: |
|
||||
if [ "${{ steps.extract-tests.outcome }}" == "failure" ]; then
|
||||
echo "Extract E2E tests failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Post to Extract Slack channel
|
||||
id: slack
|
||||
if: (failure() || cancelled()) && steps.runtime.outputs.notify_slack == 'true'
|
||||
uses: slackapi/slack-github-action@v1.27.0
|
||||
with:
|
||||
channel-id: ${{ env.SLACK_CHANNEL_ID }}
|
||||
slack-message: |
|
||||
:red_circle: *Extract E2E Failed* (${{ steps.runtime.outputs.environment }})
|
||||
```
|
||||
${{ steps.failed-tests.outputs.summary }}
|
||||
```
|
||||
<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>
|
||||
env:
|
||||
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
@@ -0,0 +1,183 @@
|
||||
"""
|
||||
Example: Batch Processing a Folder of PDFs with LlamaParse
|
||||
|
||||
This script demonstrates how to process multiple PDFs from a folder
|
||||
using LlamaParse with controlled concurrency using asyncio and semaphores.
|
||||
|
||||
Usage:
|
||||
python batch_parse_folder.py --input-dir ./pdfs --max-concurrent 5
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Any
|
||||
from datetime import datetime
|
||||
from dotenv import load_dotenv
|
||||
import os
|
||||
|
||||
from llama_cloud_services import LlamaParse
|
||||
|
||||
# Load environment variables from .env file
|
||||
load_dotenv()
|
||||
|
||||
|
||||
async def parse_single_file(
|
||||
parser: LlamaParse,
|
||||
file_path: Path,
|
||||
semaphore: asyncio.Semaphore,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Parse a single PDF file with concurrency control.
|
||||
|
||||
Args:
|
||||
parser: LlamaParse instance
|
||||
file_path: Path to the PDF file
|
||||
semaphore: Semaphore to control concurrent requests
|
||||
|
||||
Returns:
|
||||
Dictionary with file info and parse result
|
||||
"""
|
||||
async with semaphore:
|
||||
try:
|
||||
print(f"Starting parse: {file_path.name}")
|
||||
|
||||
result = await parser.aparse(str(file_path))
|
||||
|
||||
print(f"✓ Completed: {file_path.name} ({len(result.pages)} pages)")
|
||||
|
||||
return {
|
||||
"file": file_path.name,
|
||||
"status": "success",
|
||||
"result": result,
|
||||
"pages": len(result.pages) if result.pages else 0,
|
||||
}
|
||||
except Exception as e:
|
||||
print(f"✗ Error parsing {file_path.name}: {str(e)}")
|
||||
return {
|
||||
"file": file_path.name,
|
||||
"status": "error",
|
||||
"error": str(e),
|
||||
}
|
||||
|
||||
|
||||
async def parse_folder(
|
||||
input_dir: Path,
|
||||
max_concurrent: int = 5,
|
||||
api_key: str = None,
|
||||
) -> List[Dict[str, any]]:
|
||||
"""
|
||||
Parse all PDFs in a folder with controlled concurrency.
|
||||
|
||||
Args:
|
||||
input_dir: Directory containing PDF files
|
||||
max_concurrent: Maximum number of concurrent parse operations
|
||||
api_key: LlamaCloud API key (loaded from .env file)
|
||||
|
||||
Returns:
|
||||
List of parse results for each file
|
||||
"""
|
||||
# Find all PDF files
|
||||
pdf_files = list(input_dir.glob("*.pdf"))
|
||||
|
||||
if not pdf_files:
|
||||
print(f"No PDF files found in {input_dir}")
|
||||
return []
|
||||
|
||||
print(f"Found {len(pdf_files)} PDF files to parse")
|
||||
|
||||
# Initialize parser
|
||||
parser = LlamaParse(
|
||||
api_key=api_key,
|
||||
num_workers=1, # We control concurrency with semaphore
|
||||
show_progress=False, # We'll show our own progress
|
||||
)
|
||||
|
||||
# Create semaphore to limit concurrent requests
|
||||
semaphore = asyncio.Semaphore(max_concurrent)
|
||||
|
||||
# Create tasks for all files
|
||||
tasks = [parse_single_file(parser, pdf_file, semaphore) for pdf_file in pdf_files]
|
||||
|
||||
# Run all tasks concurrently (but limited by semaphore)
|
||||
print(
|
||||
f"Processing {len(tasks)} files with max {max_concurrent} concurrent operations..."
|
||||
)
|
||||
start_time = datetime.now()
|
||||
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
end_time = datetime.now()
|
||||
duration = (end_time - start_time).total_seconds()
|
||||
|
||||
# Process results
|
||||
successful = [
|
||||
r for r in results if isinstance(r, dict) and r.get("status") == "success"
|
||||
]
|
||||
failed = [r for r in results if isinstance(r, dict) and r.get("status") == "error"]
|
||||
|
||||
# Print summary
|
||||
print("PARSE SUMMARY \n")
|
||||
print(f"Total files: {len(pdf_files)}")
|
||||
print(f"Successful: {len(successful)}")
|
||||
print(f"Failed: {len(failed)}")
|
||||
print(f"Total time: {duration:.2f} seconds")
|
||||
print(f"Average time per file: {duration / len(pdf_files):.2f} seconds")
|
||||
|
||||
if failed:
|
||||
print("\nFailed files:")
|
||||
for result in failed:
|
||||
print(f" - {result['file']}: {result.get('error', 'Unknown error')}")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point for the script."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Batch process PDFs in a folder with LlamaParse"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--input-dir",
|
||||
type=str,
|
||||
required=True,
|
||||
help="Directory containing PDF files to parse",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-concurrent",
|
||||
type=int,
|
||||
default=5,
|
||||
help="Maximum number of concurrent parse operations (default: 5)",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
input_dir = Path(args.input_dir)
|
||||
|
||||
# Validate input directory
|
||||
if not input_dir.exists():
|
||||
print(f"Error: Input directory does not exist: {input_dir}")
|
||||
return
|
||||
|
||||
if not input_dir.is_dir():
|
||||
print(f"Error: Input path is not a directory: {input_dir}")
|
||||
return
|
||||
|
||||
# Get API key from environment (loaded from .env file)
|
||||
api_key = os.getenv("LLAMA_CLOUD_API_KEY")
|
||||
if not api_key:
|
||||
print("Error: LLAMA_CLOUD_API_KEY not found. Please set it in your .env file")
|
||||
return
|
||||
|
||||
# Run async function
|
||||
asyncio.run(
|
||||
parse_folder(
|
||||
input_dir=input_dir,
|
||||
max_concurrent=args.max_concurrent,
|
||||
api_key=api_key,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Binary file not shown.
@@ -0,0 +1,540 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Document Splitting with LlamaCloud\n",
|
||||
"\n",
|
||||
"This notebook demonstrates how to use the LlamaCloud **Split** API to automatically segment a concatenated PDF into logical document sections based on content categories.\n",
|
||||
"\n",
|
||||
"## Use Case\n",
|
||||
"\n",
|
||||
"When dealing with large PDFs that contain multiple distinct documents or sections (e.g., a bundle of research papers, a collection of reports), you often need to split them into individual segments. The Split API uses AI to:\n",
|
||||
"\n",
|
||||
"1. Analyze each page's content\n",
|
||||
"2. Classify pages into user-defined categories\n",
|
||||
"3. Group consecutive pages of the same category into segments\n",
|
||||
"\n",
|
||||
"## Example Document\n",
|
||||
"\n",
|
||||
"We'll use a PDF containing three concatenated documents:\n",
|
||||
"- **Alan Turing's essay** \"Intelligent Machinery, A Heretical Theory\" (an essay)\n",
|
||||
"- **ImageNet paper** (a research paper)\n",
|
||||
"- **\"Attention is All You Need\"** paper (a research paper)\n",
|
||||
"\n",
|
||||
"We'll split this into segments categorized as either `essay` or `research_paper`.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Setup\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Requirement already satisfied: llama-cloud in /Users/javier/llama_cloud_services/.venv/lib/python3.11/site-packages (0.1.44)\n",
|
||||
"Requirement already satisfied: python-dotenv in /Users/javier/llama_cloud_services/.venv/lib/python3.11/site-packages (1.2.1)\n",
|
||||
"Requirement already satisfied: requests in /Users/javier/llama_cloud_services/.venv/lib/python3.11/site-packages (2.32.5)\n",
|
||||
"Requirement already satisfied: certifi>=2024.7.4 in /Users/javier/llama_cloud_services/.venv/lib/python3.11/site-packages (from llama-cloud) (2025.11.12)\n",
|
||||
"Requirement already satisfied: httpx>=0.20.0 in /Users/javier/llama_cloud_services/.venv/lib/python3.11/site-packages (from llama-cloud) (0.28.1)\n",
|
||||
"Requirement already satisfied: pydantic>=1.10 in /Users/javier/llama_cloud_services/.venv/lib/python3.11/site-packages (from llama-cloud) (2.12.5)\n",
|
||||
"Requirement already satisfied: charset_normalizer<4,>=2 in /Users/javier/llama_cloud_services/.venv/lib/python3.11/site-packages (from requests) (3.4.4)\n",
|
||||
"Requirement already satisfied: idna<4,>=2.5 in /Users/javier/llama_cloud_services/.venv/lib/python3.11/site-packages (from requests) (3.11)\n",
|
||||
"Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/javier/llama_cloud_services/.venv/lib/python3.11/site-packages (from requests) (2.5.0)\n",
|
||||
"Requirement already satisfied: anyio in /Users/javier/llama_cloud_services/.venv/lib/python3.11/site-packages (from httpx>=0.20.0->llama-cloud) (4.11.0)\n",
|
||||
"Requirement already satisfied: httpcore==1.* in /Users/javier/llama_cloud_services/.venv/lib/python3.11/site-packages (from httpx>=0.20.0->llama-cloud) (1.0.9)\n",
|
||||
"Requirement already satisfied: h11>=0.16 in /Users/javier/llama_cloud_services/.venv/lib/python3.11/site-packages (from httpcore==1.*->httpx>=0.20.0->llama-cloud) (0.16.0)\n",
|
||||
"Requirement already satisfied: annotated-types>=0.6.0 in /Users/javier/llama_cloud_services/.venv/lib/python3.11/site-packages (from pydantic>=1.10->llama-cloud) (0.7.0)\n",
|
||||
"Requirement already satisfied: pydantic-core==2.41.5 in /Users/javier/llama_cloud_services/.venv/lib/python3.11/site-packages (from pydantic>=1.10->llama-cloud) (2.41.5)\n",
|
||||
"Requirement already satisfied: typing-extensions>=4.14.1 in /Users/javier/llama_cloud_services/.venv/lib/python3.11/site-packages (from pydantic>=1.10->llama-cloud) (4.15.0)\n",
|
||||
"Requirement already satisfied: typing-inspection>=0.4.2 in /Users/javier/llama_cloud_services/.venv/lib/python3.11/site-packages (from pydantic>=1.10->llama-cloud) (0.4.2)\n",
|
||||
"Requirement already satisfied: sniffio>=1.1 in /Users/javier/llama_cloud_services/.venv/lib/python3.11/site-packages (from anyio->httpx>=0.20.0->llama-cloud) (1.3.1)\n",
|
||||
"\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m25.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.3\u001b[0m\n",
|
||||
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
|
||||
"Note: you may need to restart the kernel to use updated packages.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Install required packages\n",
|
||||
"%pip install llama-cloud python-dotenv requests"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"✅ API configured with base URL: https://api.cloud.llamaindex.ai\n",
|
||||
"✅ Project ID: using default project\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import time\n",
|
||||
"import requests\n",
|
||||
"from dotenv import load_dotenv\n",
|
||||
"\n",
|
||||
"# Load environment variables\n",
|
||||
"load_dotenv()\n",
|
||||
"\n",
|
||||
"# Configuration\n",
|
||||
"LLAMA_CLOUD_API_KEY = os.environ.get(\"LLAMA_CLOUD_API_KEY\", \"llx-...\")\n",
|
||||
"BASE_URL = os.environ.get(\"LLAMA_CLOUD_BASE_URL\", \"https://api.cloud.llamaindex.ai\")\n",
|
||||
"PROJECT_ID = os.environ.get(\"LLAMA_CLOUD_PROJECT_ID\", None)\n",
|
||||
"\n",
|
||||
"# Headers for API requests\n",
|
||||
"headers = {\n",
|
||||
" \"Authorization\": f\"Bearer {LLAMA_CLOUD_API_KEY}\",\n",
|
||||
" \"Content-Type\": \"application/json\",\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"print(f\"✅ API configured with base URL: {BASE_URL}\")\n",
|
||||
"print(f\"✅ Project ID: {PROJECT_ID or 'using default project'}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 1: Upload the PDF File\n",
|
||||
"\n",
|
||||
"First, we'll upload our concatenated PDF to LlamaCloud using the Files API. This can be done using the `llama-cloud` SDK.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"📤 Uploading ./data/turing+imagenet+attention.pdf...\n",
|
||||
"✅ File uploaded successfully!\n",
|
||||
" File name: turing+imagenet+attention.pdf\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from llama_cloud.client import LlamaCloud\n",
|
||||
"\n",
|
||||
"# Initialize the client\n",
|
||||
"client = LlamaCloud(token=LLAMA_CLOUD_API_KEY, base_url=BASE_URL)\n",
|
||||
"\n",
|
||||
"# Path to the PDF file\n",
|
||||
"pdf_path = \"./data/turing+imagenet+attention.pdf\"\n",
|
||||
"\n",
|
||||
"# Upload the file\n",
|
||||
"print(f\"📤 Uploading {pdf_path}...\")\n",
|
||||
"\n",
|
||||
"with open(pdf_path, \"rb\") as f:\n",
|
||||
" uploaded_file = client.files.upload_file(upload_file=f, project_id=PROJECT_ID)\n",
|
||||
"\n",
|
||||
"file_id = uploaded_file.id\n",
|
||||
"print(f\"✅ File uploaded successfully!\")\n",
|
||||
"print(f\" File name: {uploaded_file.name}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 2: Create a Split Job\n",
|
||||
"\n",
|
||||
"Now we'll create a split job using the Split API. Since the Split API is in beta and not yet available in the SDK, we'll use raw HTTP requests.\n",
|
||||
"\n",
|
||||
"We define two categories:\n",
|
||||
"- **essay**: For philosophical or reflective writing\n",
|
||||
"- **research_paper**: For formal academic documents with methodology and citations\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"🔄 Creating split job...\n",
|
||||
"✅ Split job created!\n",
|
||||
" Job ID: spl-zsssb632a742aikliu96pqkb56t5\n",
|
||||
" Status: pending\n",
|
||||
" Categories: ['essay', 'research_paper']\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Define the split job request\n",
|
||||
"split_request = {\n",
|
||||
" \"document_input\": {\n",
|
||||
" \"type\": \"file_id\", # only file_id is supported for now\n",
|
||||
" \"value\": file_id,\n",
|
||||
" },\n",
|
||||
" \"categories\": [\n",
|
||||
" {\n",
|
||||
" \"name\": \"essay\",\n",
|
||||
" \"description\": \"A philosophical or reflective piece of writing that presents personal viewpoints, arguments, or thoughts on a topic without strict formal structure\",\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"name\": \"research_paper\",\n",
|
||||
" \"description\": \"A formal academic document presenting original research, methodology, experiments, results, and conclusions with citations and references\",\n",
|
||||
" },\n",
|
||||
" ],\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"# Create the split job\n",
|
||||
"print(\"🔄 Creating split job...\")\n",
|
||||
"response = requests.post(\n",
|
||||
" f\"{BASE_URL}/api/v1/beta/split/jobs\",\n",
|
||||
" params={\"project_id\": PROJECT_ID},\n",
|
||||
" headers=headers,\n",
|
||||
" json=split_request,\n",
|
||||
")\n",
|
||||
"response.raise_for_status()\n",
|
||||
"\n",
|
||||
"split_job = response.json()\n",
|
||||
"job_id = split_job[\"id\"]\n",
|
||||
"\n",
|
||||
"print(f\"✅ Split job created!\")\n",
|
||||
"print(f\" Job ID: {job_id}\")\n",
|
||||
"print(f\" Status: {split_job['status']}\")\n",
|
||||
"print(f\" Categories: {[c['name'] for c in split_job['categories']]}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 3: Poll for Job Completion\n",
|
||||
"\n",
|
||||
"The split job runs asynchronously. We'll poll the job status until it completes.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"⏳ Waiting for split job to complete...\n",
|
||||
" Status: processing (elapsed: 0s)\n",
|
||||
" Status: processing (elapsed: 5s)\n",
|
||||
" Status: processing (elapsed: 11s)\n",
|
||||
" Status: completed (elapsed: 16s)\n",
|
||||
"\n",
|
||||
"✅ Split job completed successfully!\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"def poll_split_job(job_id: str, max_wait_seconds: int = 180, poll_interval: int = 5):\n",
|
||||
" \"\"\"\n",
|
||||
" Poll a split job until it reaches a terminal state.\n",
|
||||
"\n",
|
||||
" Args:\n",
|
||||
" job_id: The split job ID\n",
|
||||
" max_wait_seconds: Maximum time to wait for completion\n",
|
||||
" poll_interval: Seconds between poll attempts\n",
|
||||
"\n",
|
||||
" Returns:\n",
|
||||
" The completed job response\n",
|
||||
" \"\"\"\n",
|
||||
" start_time = time.time()\n",
|
||||
"\n",
|
||||
" while (time.time() - start_time) < max_wait_seconds:\n",
|
||||
" response = requests.get(\n",
|
||||
" f\"{BASE_URL}/api/v1/beta/split/jobs/{job_id}\",\n",
|
||||
" params={\"project_id\": PROJECT_ID},\n",
|
||||
" headers=headers,\n",
|
||||
" )\n",
|
||||
" response.raise_for_status()\n",
|
||||
" job = response.json()\n",
|
||||
"\n",
|
||||
" status = job[\"status\"]\n",
|
||||
" elapsed = int(time.time() - start_time)\n",
|
||||
" print(f\" Status: {status} (elapsed: {elapsed}s)\")\n",
|
||||
"\n",
|
||||
" if status in [\"completed\", \"failed\"]:\n",
|
||||
" return job\n",
|
||||
"\n",
|
||||
" time.sleep(poll_interval)\n",
|
||||
"\n",
|
||||
" raise TimeoutError(f\"Job did not complete within {max_wait_seconds} seconds\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"print(\"⏳ Waiting for split job to complete...\")\n",
|
||||
"completed_job = poll_split_job(job_id)\n",
|
||||
"\n",
|
||||
"if completed_job[\"status\"] == \"completed\":\n",
|
||||
" print(\"\\n✅ Split job completed successfully!\")\n",
|
||||
"else:\n",
|
||||
" print(\n",
|
||||
" f\"\\n❌ Split job failed: {completed_job.get('error_message', 'Unknown error')}\"\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Step 4: Analyze the Results\n",
|
||||
"\n",
|
||||
"Let's examine the split results to see how the document was segmented.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"📊 Split Results Summary\n",
|
||||
"==================================================\n",
|
||||
"Total segments found: 3\n",
|
||||
"\n",
|
||||
"Segments by category:\n",
|
||||
" • essay: 1 segment(s)\n",
|
||||
" • research_paper: 2 segment(s)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Get the segments from the result\n",
|
||||
"segments = completed_job.get(\"result\", {}).get(\"segments\", [])\n",
|
||||
"\n",
|
||||
"print(f\"📊 Split Results Summary\")\n",
|
||||
"print(f\"=\" * 50)\n",
|
||||
"print(f\"Total segments found: {len(segments)}\")\n",
|
||||
"print()\n",
|
||||
"\n",
|
||||
"# Count by category\n",
|
||||
"category_counts = {}\n",
|
||||
"for segment in segments:\n",
|
||||
" cat = segment[\"category\"]\n",
|
||||
" category_counts[cat] = category_counts.get(cat, 0) + 1\n",
|
||||
"\n",
|
||||
"print(\"Segments by category:\")\n",
|
||||
"for cat, count in category_counts.items():\n",
|
||||
" print(f\" • {cat}: {count} segment(s)\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"📄 Segment Details\n",
|
||||
"==================================================\n",
|
||||
"\n",
|
||||
"Segment 1:\n",
|
||||
" Category: essay\n",
|
||||
" Pages 1-4 (4 pages)\n",
|
||||
" Confidence: high\n",
|
||||
"\n",
|
||||
"Segment 2:\n",
|
||||
" Category: research_paper\n",
|
||||
" Pages 5-13 (9 pages)\n",
|
||||
" Confidence: high\n",
|
||||
"\n",
|
||||
"Segment 3:\n",
|
||||
" Category: research_paper\n",
|
||||
" Pages 14-24 (11 pages)\n",
|
||||
" Confidence: high\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Display detailed segment information\n",
|
||||
"print(f\"\\n📄 Segment Details\")\n",
|
||||
"print(f\"=\" * 50)\n",
|
||||
"\n",
|
||||
"for i, segment in enumerate(segments, 1):\n",
|
||||
" category = segment[\"category\"]\n",
|
||||
" pages = segment[\"pages\"]\n",
|
||||
" confidence = segment[\"confidence_category\"]\n",
|
||||
"\n",
|
||||
" # Format page range\n",
|
||||
" if len(pages) == 1:\n",
|
||||
" page_range = f\"Page {pages[0]}\"\n",
|
||||
" else:\n",
|
||||
" page_range = f\"Pages {min(pages)}-{max(pages)}\"\n",
|
||||
"\n",
|
||||
" print(f\"\\nSegment {i}:\")\n",
|
||||
" print(f\" Category: {category}\")\n",
|
||||
" print(f\" {page_range} ({len(pages)} page{'s' if len(pages) > 1 else ''})\")\n",
|
||||
" print(f\" Confidence: {confidence}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Expected Results\n",
|
||||
"\n",
|
||||
"Based on our test document, we expect:\n",
|
||||
"- **1 essay segment**: Alan Turing's \"Intelligent Machinery, A Heretical Theory\"\n",
|
||||
"- **2 research paper segments**: ImageNet paper and \"Attention is All You Need\" paper\n",
|
||||
"\n",
|
||||
"The pages should be grouped consecutively, with no overlap between segments.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\n",
|
||||
"✅ Validation\n",
|
||||
"==================================================\n",
|
||||
"Total pages assigned: 24\n",
|
||||
"Unique pages: 24\n",
|
||||
"✅ No page overlap detected - each page belongs to exactly one segment\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Verify no page overlap\n",
|
||||
"all_pages = []\n",
|
||||
"for segment in segments:\n",
|
||||
" all_pages.extend(segment[\"pages\"])\n",
|
||||
"\n",
|
||||
"unique_pages = set(all_pages)\n",
|
||||
"\n",
|
||||
"print(f\"\\n✅ Validation\")\n",
|
||||
"print(f\"=\" * 50)\n",
|
||||
"print(f\"Total pages assigned: {len(all_pages)}\")\n",
|
||||
"print(f\"Unique pages: {len(unique_pages)}\")\n",
|
||||
"\n",
|
||||
"if len(all_pages) == len(unique_pages):\n",
|
||||
" print(f\"✅ No page overlap detected - each page belongs to exactly one segment\")\n",
|
||||
"else:\n",
|
||||
" print(\n",
|
||||
" f\"⚠️ Page overlap detected - {len(all_pages) - len(unique_pages)} duplicate assignments\"\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Using `allow_uncategorized` Strategy\n",
|
||||
"\n",
|
||||
"You can also use the `allow_uncategorized` splitting strategy. This is useful when you want to capture pages that don't match any defined category.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"📝 With allow_uncategorized=True and only 'essay' category defined,\n",
|
||||
" pages that don't match 'essay' will be grouped as 'uncategorized'.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Example with allow_uncategorized strategy\n",
|
||||
"split_request_uncategorized = {\n",
|
||||
" \"document_input\": {\"type\": \"file_id\", \"value\": file_id},\n",
|
||||
" \"categories\": [\n",
|
||||
" {\n",
|
||||
" \"name\": \"essay\",\n",
|
||||
" \"description\": \"A philosophical or reflective piece of writing that presents personal viewpoints, arguments, or thoughts on a topic\",\n",
|
||||
" }\n",
|
||||
" # Note: We only define 'essay' category\n",
|
||||
" # Research papers will be classified as 'uncategorized'\n",
|
||||
" ],\n",
|
||||
" \"splitting_strategy\": {\"allow_uncategorized\": True},\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"print(\"📝 With allow_uncategorized=True and only 'essay' category defined,\")\n",
|
||||
"print(\" pages that don't match 'essay' will be grouped as 'uncategorized'.\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Conclusion\n",
|
||||
"\n",
|
||||
"The LlamaCloud Split API provides a powerful way to automatically segment concatenated documents based on content categories. This is useful for:\n",
|
||||
"\n",
|
||||
"- **Document processing pipelines**: Automatically separate bundled documents before further processing\n",
|
||||
"- **Content organization**: Categorize and organize mixed document collections\n",
|
||||
"- **Information extraction**: Identify different document types within a single file\n",
|
||||
"\n",
|
||||
"### Key Features\n",
|
||||
"\n",
|
||||
"- **AI-powered classification**: Uses LLMs to understand page content and assign categories\n",
|
||||
"- **Flexible categories**: Define any categories relevant to your use case\n",
|
||||
"- **Confidence scoring**: Each segment includes a confidence level\n",
|
||||
"- **Page-level granularity**: Results include exact page numbers for each segment\n",
|
||||
"\n",
|
||||
"### API Reference\n",
|
||||
"\n",
|
||||
"- **Create Split Job**: `POST /api/v1/beta/split/jobs`\n",
|
||||
"- **Get Split Job**: `GET /api/v1/beta/split/jobs/{job_id}`\n",
|
||||
"- **List Split Jobs**: `GET /api/v1/beta/split/jobs`\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": ".venv",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
+1
-1
@@ -398,6 +398,6 @@ Another option (orthogonal to the above) is to break the document into smaller s
|
||||
## Additional Resources
|
||||
|
||||
- [Extract Documentation](https://docs.cloud.llamaindex.ai/llamaextract/getting_started) - Details on Extract features, API and examples.
|
||||
- [Example Notebook](docs/examples-py/extract/resume_screening.ipynb) - Detailed walkthrough of resume parsing
|
||||
- [Example Notebook](examples/extract/resume_screening.ipynb) - Detailed walkthrough of resume parsing
|
||||
- [Example Application with TypeScript](./examples-ts/extract/) - End-to-end examples using LlamaExtract TypeScript client.
|
||||
- [Discord Community](https://discord.com/invite/eN6D2HQ4aX) - Get help and share feedback
|
||||
|
||||
@@ -97,7 +97,7 @@ for page in result.pages:
|
||||
print(page.structuredData)
|
||||
```
|
||||
|
||||
See more details about the result object in the [example notebook](./docs/examples-py/parse/demo_json_tour.ipynb).
|
||||
See more details about the result object in the [example notebook](./examples/parse/demo_json_tour.ipynb).
|
||||
|
||||
### Using with file object / bytes
|
||||
|
||||
@@ -153,10 +153,10 @@ Full documentation for `SimpleDirectoryReader` can be found on the [LlamaIndex D
|
||||
|
||||
Several end-to-end indexing examples can be found in the examples folder
|
||||
|
||||
- [Getting Started](docs/examples-py/parse/demo_basic.ipynb)
|
||||
- [Advanced RAG Example](docs/examples-py/parse/demo_advanced.ipynb)
|
||||
- [Raw API Usage](docs/examples-py/parse/demo_api.ipynb)
|
||||
- [Result Object Tour](docs/examples-py/parse/demo_json_tour.ipynb)
|
||||
- [Getting Started](examples/parse/demo_basic.ipynb)
|
||||
- [Advanced RAG Example](examples/parse/demo_advanced.ipynb)
|
||||
- [Raw API Usage](examples/parse/demo_api.ipynb)
|
||||
- [Result Object Tour](examples/parse/demo_json_tour.ipynb)
|
||||
|
||||
## Documentation
|
||||
|
||||
|
||||
Generated
+48
-3260
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,24 @@
|
||||
# llama-cloud-services-py
|
||||
|
||||
## 0.6.91
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 07ec282: Bump up patch versions for python packages
|
||||
- 3040951: Use error description in ExtractedData invalid extraction error
|
||||
|
||||
## 0.6.90
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 19cbb25: Remove extension filter
|
||||
|
||||
## 0.6.89
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- b9b83c9: Parse bounding boxes from extract jobs results in agent data
|
||||
|
||||
## 0.6.88
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -11,6 +11,9 @@ from .schema import (
|
||||
InvalidExtractionData,
|
||||
ExtractedFieldMetadata,
|
||||
ExtractedFieldMetaDataDict,
|
||||
FieldCitation,
|
||||
BoundingBox,
|
||||
PageDimensions,
|
||||
)
|
||||
from .client import AsyncAgentDataClient
|
||||
|
||||
@@ -28,4 +31,7 @@ __all__ = [
|
||||
"InvalidExtractionData",
|
||||
"ExtractedFieldMetadata",
|
||||
"ExtractedFieldMetaDataDict",
|
||||
"FieldCitation",
|
||||
"BoundingBox",
|
||||
"PageDimensions",
|
||||
]
|
||||
|
||||
@@ -174,6 +174,22 @@ class TypedAgentDataItems(BaseModel, Generic[AgentDataT]):
|
||||
)
|
||||
|
||||
|
||||
class BoundingBox(BaseModel):
|
||||
"""Bounding box coordinates for a citation location on a page."""
|
||||
|
||||
x: float = Field(description="X coordinate of the bounding box origin")
|
||||
y: float = Field(description="Y coordinate of the bounding box origin")
|
||||
w: float = Field(description="Width of the bounding box")
|
||||
h: float = Field(description="Height of the bounding box")
|
||||
|
||||
|
||||
class PageDimensions(BaseModel):
|
||||
"""Dimensions of a page in the source document."""
|
||||
|
||||
width: float = Field(description="Width of the page")
|
||||
height: float = Field(description="Height of the page")
|
||||
|
||||
|
||||
class FieldCitation(BaseModel):
|
||||
page: Optional[int] = Field(
|
||||
None, description="The page number that the field occurred on"
|
||||
@@ -182,6 +198,14 @@ class FieldCitation(BaseModel):
|
||||
None,
|
||||
description="The original text this field's value was derived from",
|
||||
)
|
||||
bounding_boxes: Optional[List[BoundingBox]] = Field(
|
||||
None,
|
||||
description="Bounding boxes indicating where the citation appears on the page",
|
||||
)
|
||||
page_dimensions: Optional[PageDimensions] = Field(
|
||||
None,
|
||||
description="Dimensions of the page containing the citation",
|
||||
)
|
||||
|
||||
|
||||
class ExtractedFieldMetadata(BaseModel):
|
||||
@@ -201,6 +225,10 @@ class ExtractedFieldMetadata(BaseModel):
|
||||
None,
|
||||
description="The confidence score for the field based on the extracted text only",
|
||||
)
|
||||
parsing_confidence: Optional[float] = Field(
|
||||
None,
|
||||
description="The confidence score for the field based on the parsing/OCR quality",
|
||||
)
|
||||
citation: Optional[List[FieldCitation]] = Field(
|
||||
None,
|
||||
description="The citation for the field, including page number and matching text",
|
||||
@@ -447,26 +475,49 @@ class ExtractedData(BaseModel, Generic[ExtractedT]):
|
||||
},
|
||||
)
|
||||
except ValidationError as e:
|
||||
# Capture the job-level error from the extraction run if available
|
||||
job_error = result.error
|
||||
|
||||
invalid_item = ExtractedData[Dict[str, Any]].create(
|
||||
data=result.data or {},
|
||||
status="error",
|
||||
field_metadata=field_metadata,
|
||||
metadata={"extraction_error": str(e), **(metadata or {})},
|
||||
metadata={
|
||||
"extraction_error": str(e),
|
||||
**({"job_error": job_error} if job_error else {}),
|
||||
**(metadata or {}),
|
||||
},
|
||||
file_id=file_id,
|
||||
file_name=file_name,
|
||||
file_hash=file_hash,
|
||||
)
|
||||
raise InvalidExtractionData(invalid_item) from e
|
||||
raise InvalidExtractionData(invalid_item, extraction_error=job_error) from e
|
||||
|
||||
|
||||
class InvalidExtractionData(Exception):
|
||||
"""
|
||||
Exception raised when the extracted data does not conform to the schema.
|
||||
|
||||
Attributes:
|
||||
invalid_item: The ExtractedData instance containing the invalid data and metadata
|
||||
extraction_error: The error message from the extraction job, if available
|
||||
"""
|
||||
|
||||
def __init__(self, invalid_item: ExtractedData[Dict[str, Any]]):
|
||||
def __init__(
|
||||
self,
|
||||
invalid_item: ExtractedData[Dict[str, Any]],
|
||||
extraction_error: Optional[str] = None,
|
||||
):
|
||||
self.invalid_item = invalid_item
|
||||
super().__init__("Not able to parse the extracted data, parsed invalid format")
|
||||
self.extraction_error = extraction_error
|
||||
|
||||
# Build an informative error message
|
||||
if extraction_error:
|
||||
message = f"Extraction error: {extraction_error}"
|
||||
else:
|
||||
message = "Not able to parse the extracted data, parsed invalid format"
|
||||
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
def calculate_overall_confidence(
|
||||
|
||||
@@ -4,10 +4,11 @@ import os
|
||||
import time
|
||||
from io import BufferedIOBase, TextIOWrapper
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Type, Union, Coroutine, Any, TypeVar
|
||||
from typing import Callable, List, Optional, Type, Union, Coroutine, Any, TypeVar
|
||||
import warnings
|
||||
import httpx
|
||||
from pydantic import BaseModel
|
||||
from functools import wraps
|
||||
from tenacity import (
|
||||
retry_if_exception,
|
||||
stop_after_attempt,
|
||||
@@ -54,7 +55,7 @@ DEFAULT_EXTRACT_CONFIG = ExtractConfig(
|
||||
def _is_retryable_error(exception: BaseException) -> bool:
|
||||
"""Check if an exception is retryable."""
|
||||
if isinstance(exception, ApiError):
|
||||
return exception.status_code in (502, 503, 504, 425, 408)
|
||||
return exception.status_code in (429, 500, 502, 503, 504, 425, 408)
|
||||
elif isinstance(
|
||||
exception, (httpx.HTTPStatusError, httpx.RequestError, httpx.TimeoutException)
|
||||
):
|
||||
@@ -62,6 +63,33 @@ def _is_retryable_error(exception: BaseException) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _async_retry(
|
||||
max_attempts: int = 5,
|
||||
initial_wait: float = 1,
|
||||
max_wait: float = 30,
|
||||
jitter: float = 3,
|
||||
) -> Callable:
|
||||
"""Decorator for async functions with retry logic for rate limiting and transient errors."""
|
||||
|
||||
def decorator(func: Callable) -> Callable:
|
||||
@wraps(func)
|
||||
async def wrapper(*args: Any, **kwargs: Any) -> Any:
|
||||
async for attempt in AsyncRetrying(
|
||||
retry=retry_if_exception(_is_retryable_error),
|
||||
stop=stop_after_attempt(max_attempts),
|
||||
wait=wait_exponential_jitter(
|
||||
initial=initial_wait, max=max_wait, jitter=jitter
|
||||
),
|
||||
reraise=True,
|
||||
):
|
||||
with attempt:
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
async def _validate_schema(
|
||||
client: AsyncLlamaCloud, data_schema: SchemaInput
|
||||
) -> JSONObjectType:
|
||||
@@ -82,50 +110,6 @@ async def _validate_schema(
|
||||
return validated_schema.data_schema
|
||||
|
||||
|
||||
async def _get_job_with_retry(
|
||||
client: AsyncLlamaCloud,
|
||||
job_id: str,
|
||||
max_attempts: int = 5,
|
||||
initial_wait: float = 1,
|
||||
max_wait: float = 60,
|
||||
jitter: float = 5,
|
||||
) -> ExtractJob:
|
||||
"""Get extraction job with retry logic."""
|
||||
async for attempt in AsyncRetrying(
|
||||
retry=retry_if_exception(_is_retryable_error),
|
||||
stop=stop_after_attempt(max_attempts),
|
||||
wait=wait_exponential_jitter(initial=initial_wait, max=max_wait, jitter=jitter),
|
||||
reraise=True,
|
||||
):
|
||||
with attempt:
|
||||
return await client.llama_extract.get_job(job_id=job_id)
|
||||
|
||||
|
||||
async def _get_run_with_retry(
|
||||
client: AsyncLlamaCloud,
|
||||
job_id: str,
|
||||
project_id: Optional[str] = None,
|
||||
organization_id: Optional[str] = None,
|
||||
max_attempts: int = 3,
|
||||
initial_wait: float = 1,
|
||||
max_wait: float = 20,
|
||||
jitter: float = 3,
|
||||
) -> ExtractRun:
|
||||
"""Get extraction run with retry logic."""
|
||||
async for attempt in AsyncRetrying(
|
||||
retry=retry_if_exception(_is_retryable_error),
|
||||
stop=stop_after_attempt(max_attempts),
|
||||
wait=wait_exponential_jitter(initial=initial_wait, max=max_wait, jitter=jitter),
|
||||
reraise=True,
|
||||
):
|
||||
with attempt:
|
||||
return await client.llama_extract.get_run_by_job_id(
|
||||
job_id=job_id,
|
||||
project_id=project_id,
|
||||
organization_id=organization_id,
|
||||
)
|
||||
|
||||
|
||||
async def _wait_for_job_result(
|
||||
client: AsyncLlamaCloud,
|
||||
job_id: str,
|
||||
@@ -142,30 +126,33 @@ async def _wait_for_job_result(
|
||||
run_jitter: float = 3,
|
||||
) -> Optional[ExtractRun]:
|
||||
"""Wait for and return the results of an extraction job."""
|
||||
|
||||
@_async_retry(
|
||||
max_attempts=job_retry_attempts, max_wait=job_max_wait, jitter=job_jitter
|
||||
)
|
||||
async def _get_job() -> ExtractJob:
|
||||
return await client.llama_extract.get_job(job_id=job_id)
|
||||
|
||||
@_async_retry(
|
||||
max_attempts=run_retry_attempts, max_wait=run_max_wait, jitter=run_jitter
|
||||
)
|
||||
async def _get_run() -> ExtractRun:
|
||||
return await client.llama_extract.get_run_by_job_id(
|
||||
job_id=job_id,
|
||||
project_id=project_id,
|
||||
organization_id=organization_id,
|
||||
)
|
||||
|
||||
start = time.perf_counter()
|
||||
poll_count = 0
|
||||
|
||||
while True:
|
||||
await asyncio.sleep(check_interval)
|
||||
poll_count += 1
|
||||
job = await _get_job_with_retry(
|
||||
client,
|
||||
job_id,
|
||||
max_attempts=job_retry_attempts,
|
||||
max_wait=job_max_wait,
|
||||
jitter=job_jitter,
|
||||
)
|
||||
job = await _get_job()
|
||||
|
||||
if job.status == StatusEnum.SUCCESS:
|
||||
return await _get_run_with_retry(
|
||||
client,
|
||||
job_id,
|
||||
project_id,
|
||||
organization_id,
|
||||
max_attempts=run_retry_attempts,
|
||||
max_wait=run_max_wait,
|
||||
jitter=run_jitter,
|
||||
)
|
||||
return await _get_run()
|
||||
elif job.status == StatusEnum.PENDING:
|
||||
end = time.perf_counter()
|
||||
if end - start > max_timeout:
|
||||
@@ -177,15 +164,7 @@ async def _wait_for_job_result(
|
||||
warnings.warn(
|
||||
f"Failure in job: {job_id}, status: {job.status}, error: {job.error}"
|
||||
)
|
||||
return await _get_run_with_retry(
|
||||
client,
|
||||
job_id,
|
||||
project_id,
|
||||
organization_id,
|
||||
max_attempts=run_retry_attempts,
|
||||
max_wait=run_max_wait,
|
||||
jitter=run_jitter,
|
||||
)
|
||||
return await _get_run()
|
||||
|
||||
|
||||
def run_in_thread(
|
||||
@@ -498,9 +477,12 @@ class ExtractionAgent:
|
||||
Args:
|
||||
run_id (str): The ID of the extraction run to delete
|
||||
"""
|
||||
self._run_in_thread(
|
||||
self._client.llama_extract.delete_extraction_run(run_id=run_id)
|
||||
)
|
||||
|
||||
@_async_retry()
|
||||
async def _delete() -> None:
|
||||
return await self._client.llama_extract.delete_extraction_run(run_id=run_id)
|
||||
|
||||
self._run_in_thread(_delete())
|
||||
|
||||
def list_extraction_runs(
|
||||
self, page: int = 0, limit: int = 100
|
||||
@@ -510,13 +492,16 @@ class ExtractionAgent:
|
||||
Returns:
|
||||
PaginatedExtractRunsResponse: Paginated list of extraction runs
|
||||
"""
|
||||
return self._run_in_thread(
|
||||
self._client.llama_extract.list_extract_runs(
|
||||
|
||||
@_async_retry()
|
||||
async def _list() -> PaginatedExtractRunsResponse:
|
||||
return await self._client.llama_extract.list_extract_runs(
|
||||
extraction_agent_id=self.id,
|
||||
skip=page * limit,
|
||||
limit=limit,
|
||||
)
|
||||
)
|
||||
|
||||
return self._run_in_thread(_list())
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"ExtractionAgent(id={self.id}, name={self.name})"
|
||||
@@ -658,15 +643,17 @@ class LlamaExtract(BaseComponent):
|
||||
"data_schema must be either a dictionary or a Pydantic model"
|
||||
)
|
||||
|
||||
agent = self._run_in_thread(
|
||||
self._async_client.llama_extract.create_extraction_agent(
|
||||
@_async_retry()
|
||||
async def _create() -> CloudExtractAgent:
|
||||
return await self._async_client.llama_extract.create_extraction_agent(
|
||||
project_id=self._project_id,
|
||||
organization_id=self._organization_id,
|
||||
name=name,
|
||||
data_schema=data_schema,
|
||||
config=config,
|
||||
)
|
||||
)
|
||||
|
||||
agent = self._run_in_thread(_create())
|
||||
|
||||
return ExtractionAgent(
|
||||
client=self._async_client,
|
||||
@@ -702,19 +689,27 @@ class LlamaExtract(BaseComponent):
|
||||
)
|
||||
|
||||
if id:
|
||||
agent = self._run_in_thread(
|
||||
self._async_client.llama_extract.get_extraction_agent(
|
||||
|
||||
@_async_retry()
|
||||
async def _get_by_id() -> CloudExtractAgent:
|
||||
return await self._async_client.llama_extract.get_extraction_agent(
|
||||
extraction_agent_id=id,
|
||||
)
|
||||
)
|
||||
|
||||
agent = self._run_in_thread(_get_by_id())
|
||||
|
||||
elif name:
|
||||
agent = self._run_in_thread(
|
||||
self._async_client.llama_extract.get_extraction_agent_by_name(
|
||||
name=name,
|
||||
project_id=self._project_id,
|
||||
|
||||
@_async_retry()
|
||||
async def _get_by_name() -> CloudExtractAgent:
|
||||
return (
|
||||
await self._async_client.llama_extract.get_extraction_agent_by_name(
|
||||
name=name,
|
||||
project_id=self._project_id,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
agent = self._run_in_thread(_get_by_name())
|
||||
else:
|
||||
raise ValueError("Either name or extraction_agent_id must be provided.")
|
||||
|
||||
@@ -734,11 +729,14 @@ class LlamaExtract(BaseComponent):
|
||||
|
||||
def list_agents(self) -> List[ExtractionAgent]:
|
||||
"""List all available extraction agents."""
|
||||
agents = self._run_in_thread(
|
||||
self._async_client.llama_extract.list_extraction_agents(
|
||||
|
||||
@_async_retry()
|
||||
async def _list() -> List[CloudExtractAgent]:
|
||||
return await self._async_client.llama_extract.list_extraction_agents(
|
||||
project_id=self._project_id,
|
||||
)
|
||||
)
|
||||
|
||||
agents = self._run_in_thread(_list())
|
||||
|
||||
return [
|
||||
ExtractionAgent(
|
||||
@@ -763,11 +761,14 @@ class LlamaExtract(BaseComponent):
|
||||
Args:
|
||||
agent_id (str): ID of the extraction agent to delete
|
||||
"""
|
||||
self._run_in_thread(
|
||||
self._async_client.llama_extract.delete_extraction_agent(
|
||||
extraction_agent_id=agent_id
|
||||
|
||||
@_async_retry()
|
||||
async def _delete() -> None:
|
||||
return await self._async_client.llama_extract.delete_extraction_agent(
|
||||
extraction_agent_id=agent_id,
|
||||
)
|
||||
)
|
||||
|
||||
self._run_in_thread(_delete())
|
||||
|
||||
async def _wait_for_job_result(self, job_id: str) -> Optional[ExtractRun]:
|
||||
"""Wait for and return the results of an extraction job."""
|
||||
|
||||
@@ -3,7 +3,7 @@ from typing import BinaryIO
|
||||
import os
|
||||
from pathlib import Path
|
||||
from llama_cloud.client import AsyncLlamaCloud
|
||||
from llama_cloud.types import File, FileCreate
|
||||
from llama_cloud.types import File
|
||||
from typing import Optional
|
||||
from llama_cloud_services.utils import SourceText, FileInput
|
||||
|
||||
@@ -73,11 +73,9 @@ class FileClient:
|
||||
presigned_url = await self.client.files.generate_presigned_url(
|
||||
project_id=self.project_id,
|
||||
organization_id=self.organization_id,
|
||||
request=FileCreate(
|
||||
name=name,
|
||||
external_file_id=external_file_id,
|
||||
file_size=file_size,
|
||||
),
|
||||
name=name,
|
||||
external_file_id=external_file_id,
|
||||
file_size=file_size,
|
||||
)
|
||||
httpx_client = self.client._client_wrapper.httpx_client
|
||||
upload_response = await httpx_client.put(
|
||||
|
||||
@@ -21,7 +21,6 @@ from llama_cloud import (
|
||||
PipelineCreateTransformConfig,
|
||||
PipelineFileCreateCustomMetadataValue,
|
||||
PipelineType,
|
||||
ProjectCreate,
|
||||
ManagedIngestionStatus,
|
||||
CloudDocumentCreate,
|
||||
CloudDocument,
|
||||
@@ -507,14 +506,19 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
client = get_client(api_key, base_url, app_url, timeout)
|
||||
|
||||
if project_id is None:
|
||||
# create project if it doesn't exist
|
||||
project = client.projects.upsert_project(
|
||||
# get project by name
|
||||
projects = client.projects.list_projects(
|
||||
organization_id=organization_id,
|
||||
request=ProjectCreate(name=project_name),
|
||||
project_name=project_name,
|
||||
)
|
||||
if not projects:
|
||||
raise ValueError(
|
||||
f"Project '{project_name}' not found. Please create it first in the LlamaCloud UI."
|
||||
)
|
||||
project = projects[0]
|
||||
project_id = project.id
|
||||
if verbose:
|
||||
print(f"Created project {project_id} with name {project_name}")
|
||||
print(f"Found project {project_id} with name {project_name}")
|
||||
|
||||
# create pipeline
|
||||
pipeline_create = PipelineCreate(
|
||||
@@ -563,15 +567,20 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
app_url = app_url or os.environ.get("LLAMA_CLOUD_APP_URL", DEFAULT_APP_URL)
|
||||
aclient = get_aclient(api_key, base_url, app_url, timeout)
|
||||
|
||||
# create project if it doesn't exist
|
||||
project = await aclient.projects.upsert_project(
|
||||
organization_id=organization_id, request=ProjectCreate(name=project_name)
|
||||
# get project by name
|
||||
projects = await aclient.projects.list_projects(
|
||||
organization_id=organization_id, project_name=project_name
|
||||
)
|
||||
if not projects:
|
||||
raise ValueError(
|
||||
f"Project '{project_name}' not found. Please create it first in the LlamaCloud UI."
|
||||
)
|
||||
project = projects[0]
|
||||
if project.id is None:
|
||||
raise ValueError(f"Failed to create/get project {project_name}")
|
||||
raise ValueError(f"Failed to get project {project_name}")
|
||||
|
||||
if verbose:
|
||||
print(f"Created project {project.id} with name {project.name}")
|
||||
print(f"Found project {project.id} with name {project.name}")
|
||||
|
||||
# create pipeline
|
||||
pipeline_create = PipelineCreate(
|
||||
|
||||
@@ -751,11 +751,9 @@ class LlamaParse(BasePydanticReader):
|
||||
file_path = str(file_input)
|
||||
file_ext = os.path.splitext(file_path)[1].lower()
|
||||
if file_ext not in SUPPORTED_FILE_TYPES:
|
||||
raise Exception(
|
||||
f"Currently, only the following file types are supported: {SUPPORTED_FILE_TYPES}\n"
|
||||
f"Current file type: {file_ext}"
|
||||
)
|
||||
mime_type = mimetypes.guess_type(file_path)[0]
|
||||
mime_type = "application/octet-stream"
|
||||
else:
|
||||
mime_type = mimetypes.guess_type(file_path)[0]
|
||||
# Open the file here for the duration of the async context
|
||||
# load data, set the mime type
|
||||
fs = fs or get_default_fs()
|
||||
|
||||
@@ -1,5 +1,29 @@
|
||||
# llama_parse
|
||||
|
||||
## 0.6.91
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 07ec282: Bump up patch versions for python packages
|
||||
- Updated dependencies [07ec282]
|
||||
- Updated dependencies [3040951]
|
||||
- llama-cloud-services-py@0.6.91
|
||||
|
||||
## 0.6.90
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 19cbb25: Remove extension filter
|
||||
- Updated dependencies [19cbb25]
|
||||
- llama-cloud-services-py@0.6.90
|
||||
|
||||
## 0.6.89
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [b9b83c9]
|
||||
- llama-cloud-services-py@0.6.89
|
||||
|
||||
## 0.6.88
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -146,9 +146,9 @@ Full documentation for `SimpleDirectoryReader` can be found on the [LlamaIndex D
|
||||
|
||||
Several end-to-end indexing examples can be found in the examples folder
|
||||
|
||||
- [Getting Started](/docs/examples-py/parse/demo_basic.ipynb)
|
||||
- [Advanced RAG Example](/docs/examples-py/parse/demo_advanced.ipynb)
|
||||
- [Raw API Usage](/docs/examples-py/parse/demo_api.ipynb)
|
||||
- [Getting Started](../../examples/parse/demo_basic.ipynb)
|
||||
- [Advanced RAG Example](../../examples/parse/demo_advanced.ipynb)
|
||||
- [Raw API Usage](../../examples/parse/demo_api.ipynb)
|
||||
|
||||
## Documentation
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "llama_parse",
|
||||
"version": "0.6.88",
|
||||
"version": "0.6.91",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"private": false,
|
||||
|
||||
@@ -11,13 +11,13 @@ dev = [
|
||||
|
||||
[project]
|
||||
name = "llama-parse"
|
||||
version = "0.6.88"
|
||||
version = "0.6.91"
|
||||
description = "Parse files into RAG-Optimized formats."
|
||||
authors = [{name = "Logan Markewich", email = "logan@llamaindex.ai"}]
|
||||
requires-python = ">=3.9,<4.0"
|
||||
readme = "README.md"
|
||||
license = "MIT"
|
||||
dependencies = ["llama-cloud-services>=0.6.88"]
|
||||
dependencies = ["llama-cloud-services>=0.6.91"]
|
||||
|
||||
[project.scripts]
|
||||
llama-parse = "llama_parse.cli.main:parse"
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "llama-cloud-services-py",
|
||||
"version": "0.6.88",
|
||||
"version": "0.6.91",
|
||||
"private": false,
|
||||
"license": "MIT",
|
||||
"scripts": {},
|
||||
|
||||
+2
-2
@@ -23,7 +23,7 @@ dev = [
|
||||
|
||||
[project]
|
||||
name = "llama-cloud-services"
|
||||
version = "0.6.88"
|
||||
version = "0.6.91"
|
||||
description = "Tailored SDK clients for LlamaCloud services."
|
||||
authors = [{name = "Logan Markewich", email = "logan@runllama.ai"}]
|
||||
requires-python = ">=3.9,<4.0"
|
||||
@@ -31,7 +31,7 @@ readme = "README.md"
|
||||
license = "MIT"
|
||||
dependencies = [
|
||||
"llama-index-core>=0.12.0",
|
||||
"llama-cloud==0.1.45",
|
||||
"llama-cloud==0.1.46",
|
||||
"pydantic>=2.8,!=2.10",
|
||||
"click>=8.1.7,<9",
|
||||
"python-dotenv>=1.0.1,<2",
|
||||
|
||||
@@ -1,11 +1,43 @@
|
||||
import os
|
||||
from typing import List
|
||||
from llama_cloud_services.extract import LlamaExtract
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from llama_cloud.core.api_error import ApiError
|
||||
from llama_cloud.types import ExtractConfig
|
||||
from pydantic import BaseModel
|
||||
from tenacity import (
|
||||
retry,
|
||||
retry_if_exception,
|
||||
stop_after_attempt,
|
||||
wait_exponential,
|
||||
)
|
||||
|
||||
from llama_cloud_services.extract import ExtractionAgent, LlamaExtract
|
||||
|
||||
# Global storage for agents to cleanup
|
||||
_TEST_AGENTS_TO_CLEANUP: List[str] = []
|
||||
|
||||
|
||||
def _is_rate_limit_error(exception: BaseException) -> bool:
|
||||
"""Check if the exception is a rate limit error (429)."""
|
||||
return isinstance(exception, ApiError) and exception.status_code == 429
|
||||
|
||||
|
||||
@retry(
|
||||
retry=retry_if_exception(_is_rate_limit_error),
|
||||
wait=wait_exponential(multiplier=1, min=1, max=30),
|
||||
stop=stop_after_attempt(5),
|
||||
reraise=True,
|
||||
)
|
||||
def create_agent_with_retry(
|
||||
extractor: LlamaExtract,
|
||||
name: str,
|
||||
data_schema: Union[Dict[str, Any], type[BaseModel]],
|
||||
config: Optional[ExtractConfig] = None,
|
||||
) -> ExtractionAgent:
|
||||
"""Create an extraction agent with retry logic for rate limiting."""
|
||||
return extractor.create_agent(name=name, data_schema=data_schema, config=config)
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
"""Register custom markers for extract tests."""
|
||||
config.addinivalue_line("markers", "agent_name: custom agent name for test")
|
||||
|
||||
@@ -6,7 +6,7 @@ from pydantic import BaseModel
|
||||
from llama_cloud_services.extract import LlamaExtract, ExtractionAgent, SourceText
|
||||
from llama_cloud.types import ExtractConfig, ExtractMode, ExtractRun
|
||||
from tests.extract.util import load_test_dotenv
|
||||
from .conftest import register_agent_for_cleanup
|
||||
from .conftest import register_agent_for_cleanup, create_agent_with_retry
|
||||
|
||||
load_test_dotenv()
|
||||
|
||||
@@ -87,7 +87,11 @@ def test_agent(llama_extract, test_agent_name, test_schema_dict, request):
|
||||
except Exception as e:
|
||||
print(f"Warning: Failed to cleanup existing agent: {e}")
|
||||
|
||||
agent = llama_extract.create_agent(name=name, data_schema=schema)
|
||||
# Use config with cache invalidation to ensure fresh results in tests
|
||||
config = ExtractConfig(invalidate_cache=True)
|
||||
agent = create_agent_with_retry(
|
||||
llama_extract, name=name, data_schema=schema, config=config
|
||||
)
|
||||
|
||||
# Add agent to cleanup list via conftest helper
|
||||
register_agent_for_cleanup(agent.id)
|
||||
@@ -237,7 +241,7 @@ class TestStatelessExtraction:
|
||||
|
||||
@pytest.fixture
|
||||
def test_config(self):
|
||||
return ExtractConfig(extraction_mode=ExtractMode.FAST)
|
||||
return ExtractConfig(extraction_mode=ExtractMode.FAST, invalidate_cache=True)
|
||||
|
||||
@pytest.fixture
|
||||
def test_schema_dict(self):
|
||||
|
||||
@@ -8,7 +8,7 @@ import uuid
|
||||
from llama_cloud.types import ExtractConfig, ExtractMode
|
||||
from deepdiff import DeepDiff
|
||||
from tests.extract.util import json_subset_match_score, load_test_dotenv
|
||||
from .conftest import register_agent_for_cleanup
|
||||
from .conftest import register_agent_for_cleanup, create_agent_with_retry
|
||||
|
||||
load_test_dotenv()
|
||||
|
||||
@@ -56,10 +56,16 @@ def get_test_cases():
|
||||
input_files.append(file_path)
|
||||
|
||||
settings = [
|
||||
ExtractConfig(extraction_mode=ExtractMode.FAST),
|
||||
ExtractConfig(extraction_mode=ExtractMode.BALANCED),
|
||||
ExtractConfig(extraction_mode=ExtractMode.MULTIMODAL),
|
||||
ExtractConfig(extraction_mode=ExtractMode.PREMIUM),
|
||||
ExtractConfig(extraction_mode=ExtractMode.FAST, invalidate_cache=True),
|
||||
ExtractConfig(extraction_mode=ExtractMode.BALANCED, invalidate_cache=True),
|
||||
ExtractConfig(
|
||||
extraction_mode=ExtractMode.MULTIMODAL, invalidate_cache=True
|
||||
),
|
||||
ExtractConfig(
|
||||
extraction_mode=ExtractMode.PREMIUM,
|
||||
invalidate_cache=True,
|
||||
parse_model="anthropic-sonnet-4.5",
|
||||
),
|
||||
]
|
||||
|
||||
for input_file in sorted(input_files):
|
||||
@@ -117,8 +123,10 @@ def extraction_agent(test_case: ExtractionTestCase, extractor: LlamaExtract):
|
||||
except Exception as e:
|
||||
print(f"Warning: Failed to cleanup existing agent: {str(e)}")
|
||||
|
||||
# Create new agent
|
||||
agent = extractor.create_agent(agent_name, schema, config=test_case.config)
|
||||
# Create new agent with retry logic for rate limiting
|
||||
agent = create_agent_with_retry(
|
||||
extractor, name=agent_name, data_schema=schema, config=test_case.config
|
||||
)
|
||||
|
||||
# Register agent for cleanup at the end of the test session
|
||||
register_agent_for_cleanup(agent.id)
|
||||
|
||||
@@ -8,7 +8,6 @@ from llama_cloud import (
|
||||
AutoTransformConfig,
|
||||
PipelineCreate,
|
||||
PipelineFileCreate,
|
||||
ProjectCreate,
|
||||
CompositeRetrievalMode,
|
||||
LlamaParseParameters,
|
||||
ReRankConfig,
|
||||
@@ -60,11 +59,15 @@ def local_figures_file() -> str:
|
||||
def _setup_index_with_file(
|
||||
client: LlamaCloud, index_name: str, remote_file: Tuple[str, str]
|
||||
) -> LlamaCloudIndex:
|
||||
# create project if it doesn't exist
|
||||
project_create = ProjectCreate(name=project_name)
|
||||
project = client.projects.upsert_project(
|
||||
organization_id=organization_id, request=project_create
|
||||
# get project by name
|
||||
projects = client.projects.list_projects(
|
||||
organization_id=organization_id, project_name=project_name
|
||||
)
|
||||
if not projects:
|
||||
raise ValueError(
|
||||
f"Project '{project_name}' not found. Please create it first in the LlamaCloud UI."
|
||||
)
|
||||
project = projects[0]
|
||||
|
||||
# create pipeline
|
||||
pipeline_create = PipelineCreate(
|
||||
|
||||
@@ -11,10 +11,12 @@ from llama_cloud.types.aggregate_group import AggregateGroup
|
||||
from pydantic import BaseModel, Field, ValidationError
|
||||
|
||||
from llama_cloud_services.beta.agent_data.schema import (
|
||||
BoundingBox,
|
||||
ExtractedData,
|
||||
ExtractedFieldMetadata,
|
||||
FieldCitation,
|
||||
InvalidExtractionData,
|
||||
PageDimensions,
|
||||
TypedAgentData,
|
||||
TypedAggregateGroup,
|
||||
calculate_overall_confidence,
|
||||
@@ -421,6 +423,7 @@ def create_extract_run(
|
||||
},
|
||||
data_schema: Dict[str, Any] = {},
|
||||
file: File = create_file(),
|
||||
error: Optional[str] = None,
|
||||
) -> ExtractRun:
|
||||
return ExtractRun.parse_obj(
|
||||
{
|
||||
@@ -437,6 +440,7 @@ def create_extract_run(
|
||||
"status": "SUCCESS",
|
||||
"project_id": str(uuid.uuid4()),
|
||||
"from_ui": False,
|
||||
"error": error,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -542,6 +546,46 @@ def test_extracted_data_from_extraction_result_invalid_data():
|
||||
assert invalid_data.field_metadata["name"].confidence == 0.9
|
||||
assert invalid_data.overall_confidence == 0.9
|
||||
|
||||
# Verify default error message when no job error present
|
||||
assert exc_info.value.extraction_error is None
|
||||
assert "Not able to parse the extracted data" in str(exc_info.value)
|
||||
|
||||
|
||||
def test_extracted_data_from_extraction_result_with_job_error():
|
||||
"""Test ExtractedData.from_extraction_result with job-level error prominently displayed."""
|
||||
job_error_message = "Failed to process document: unsupported file format"
|
||||
|
||||
# Create ExtractRun with both invalid data AND a job-level error
|
||||
extract_run = create_extract_run(
|
||||
data={
|
||||
"missing_name": "Valid Name",
|
||||
"age": "not_a_number",
|
||||
}, # Invalid age, missing name
|
||||
extraction_metadata={
|
||||
"name": {"confidence": 0.9},
|
||||
},
|
||||
data_schema={},
|
||||
file=create_file(id="error-file", name="bad_data.pdf"),
|
||||
error=job_error_message,
|
||||
)
|
||||
|
||||
# Should raise InvalidExtractionData with the job error prominently displayed
|
||||
with pytest.raises(InvalidExtractionData) as exc_info:
|
||||
ExtractedData.from_extraction_result(
|
||||
extract_run, Person, metadata={"test": "metadata"}
|
||||
)
|
||||
|
||||
# Verify the exception message prominently shows the job error
|
||||
exception = exc_info.value
|
||||
assert exception.extraction_error == job_error_message
|
||||
assert f"Extraction error: {job_error_message}" == str(exception)
|
||||
|
||||
# Verify the invalid_item contains both errors in metadata
|
||||
invalid_data = exception.invalid_item
|
||||
assert invalid_data.metadata.get("job_error") == job_error_message
|
||||
assert "extraction_error" in invalid_data.metadata # Validation error still present
|
||||
assert "test" in invalid_data.metadata # Original metadata preserved
|
||||
|
||||
|
||||
class Dimensions(BaseModel):
|
||||
length: Optional[str] = Field(
|
||||
@@ -663,3 +707,69 @@ def test_field_conflict_in_schema():
|
||||
assert isinstance(
|
||||
extracted["majority_opinion"]["reasoning"], ExtractedFieldMetadata
|
||||
)
|
||||
|
||||
|
||||
def test_parse_extracted_field_metadata_with_bounding_boxes():
|
||||
"""Test parse_extracted_field_metadata with bounding boxes and page dimensions."""
|
||||
raw_metadata = {
|
||||
"document_type": {
|
||||
"citation": [
|
||||
{
|
||||
"page": 1,
|
||||
"matching_text": "FACTURE ORIGINALE",
|
||||
"bounding_boxes": [{"x": 77.28, "y": 615.12, "w": 70.6, "h": 7.2}],
|
||||
"page_dimensions": {"width": 222.24, "height": 736.56},
|
||||
}
|
||||
],
|
||||
"parsing_confidence": 1.0,
|
||||
"extraction_confidence": 0.7252506422636493,
|
||||
"confidence": 0.7252506422636493,
|
||||
},
|
||||
"summary": {
|
||||
"citation": [
|
||||
{
|
||||
"page": 1,
|
||||
"matching_text": "FACTURE ORIGINALE",
|
||||
"bounding_boxes": [{"x": 77.28, "y": 615.12, "w": 70.6, "h": 7.2}],
|
||||
"page_dimensions": {"width": 222.24, "height": 736.56},
|
||||
},
|
||||
{
|
||||
"page": 1,
|
||||
"matching_text": "Café filtre assiette — $1.90",
|
||||
"bounding_boxes": [
|
||||
{"x": 10.56, "y": 172.83, "w": 171.85, "h": 497.01}
|
||||
],
|
||||
"page_dimensions": {"width": 222.24, "height": 736.56},
|
||||
},
|
||||
],
|
||||
"parsing_confidence": 1.0,
|
||||
"extraction_confidence": 0.5700013128334419,
|
||||
"confidence": 0.5700013128334419,
|
||||
},
|
||||
}
|
||||
|
||||
result = parse_extracted_field_metadata(raw_metadata)
|
||||
|
||||
# Verify document_type citation with bounding boxes
|
||||
assert isinstance(result["document_type"], ExtractedFieldMetadata)
|
||||
assert result["document_type"].parsing_confidence == 1.0
|
||||
assert result["document_type"].extraction_confidence == 0.7252506422636493
|
||||
assert result["document_type"].confidence == 0.7252506422636493
|
||||
assert len(result["document_type"].citation) == 1
|
||||
|
||||
citation = result["document_type"].citation[0]
|
||||
assert citation.page == 1
|
||||
assert citation.matching_text == "FACTURE ORIGINALE"
|
||||
assert len(citation.bounding_boxes) == 1
|
||||
assert citation.bounding_boxes[0] == BoundingBox(x=77.28, y=615.12, w=70.6, h=7.2)
|
||||
assert citation.page_dimensions == PageDimensions(width=222.24, height=736.56)
|
||||
|
||||
# Verify summary citation with multiple bounding boxes
|
||||
assert isinstance(result["summary"], ExtractedFieldMetadata)
|
||||
assert len(result["summary"].citation) == 2
|
||||
assert result["summary"].citation[0].bounding_boxes[0].x == 77.28
|
||||
assert result["summary"].citation[1].bounding_boxes[0].x == 10.56
|
||||
|
||||
# Verify round-trip serialization
|
||||
result2 = parse_extracted_field_metadata(result)
|
||||
assert result2 == result
|
||||
|
||||
@@ -34,9 +34,10 @@ TEST_PIPELINE = Pipeline(
|
||||
def mock_client() -> MagicMock:
|
||||
"""Mock client with sensible defaults."""
|
||||
client = MagicMock()
|
||||
client.projects.upsert_project.return_value = Project(
|
||||
default_project = Project(
|
||||
id="default-proj", name=DEFAULT_PROJECT_NAME, organization_id="default-org"
|
||||
)
|
||||
client.projects.list_projects.return_value = [default_project]
|
||||
client.pipelines.upsert_pipeline.return_value = Pipeline(
|
||||
id="default-pipe",
|
||||
name="default",
|
||||
@@ -100,8 +101,8 @@ def test_from_documents_uses_provided_project_id(mock_client: MagicMock) -> None
|
||||
project_id=provided_project_id,
|
||||
)
|
||||
|
||||
# Assert - project upsert not called; pipeline uses provided project_id
|
||||
mock_client.projects.upsert_project.assert_not_called()
|
||||
# Assert - project list not called (project_id provided); pipeline uses provided project_id
|
||||
mock_client.projects.list_projects.assert_not_called()
|
||||
assert mock_client.pipelines.upsert_pipeline.call_count == 1
|
||||
assert (
|
||||
mock_client.pipelines.upsert_pipeline.call_args.kwargs["project_id"]
|
||||
@@ -110,29 +111,29 @@ def test_from_documents_uses_provided_project_id(mock_client: MagicMock) -> None
|
||||
assert index.project.id == provided_project_id
|
||||
|
||||
|
||||
def test_from_documents_upserts_project_when_project_id_missing(
|
||||
def test_from_documents_lists_project_when_project_id_missing(
|
||||
mock_client: MagicMock,
|
||||
) -> None:
|
||||
organization_id = "org-xyz"
|
||||
index_name = "my_new_index"
|
||||
|
||||
# Project is created when project_id is not provided
|
||||
upserted_project = Project(
|
||||
# Project is found by name when project_id is not provided
|
||||
found_project = Project(
|
||||
id="proj-999", name=DEFAULT_PROJECT_NAME, organization_id=organization_id
|
||||
)
|
||||
mock_client.projects.upsert_project.return_value = upserted_project
|
||||
mock_client.projects.list_projects.return_value = [found_project]
|
||||
|
||||
test_pipeline = Pipeline(
|
||||
id="pipe-xyz",
|
||||
name=index_name,
|
||||
project_id=upserted_project.id,
|
||||
project_id=found_project.id,
|
||||
embedding_config=EMBEDDING_CONFIG,
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
base,
|
||||
"resolve_project_and_pipeline",
|
||||
return_value=(upserted_project, test_pipeline),
|
||||
return_value=(found_project, test_pipeline),
|
||||
):
|
||||
docs = [Document(text="world")]
|
||||
index = LlamaCloudIndex.from_documents(
|
||||
@@ -141,15 +142,15 @@ def test_from_documents_upserts_project_when_project_id_missing(
|
||||
organization_id=organization_id,
|
||||
)
|
||||
|
||||
# Assert - project was upserted with org id and default project name
|
||||
mock_client.projects.upsert_project.assert_called_once()
|
||||
kwargs = mock_client.projects.upsert_project.call_args.kwargs
|
||||
# Assert - project was listed with org id and default project name
|
||||
mock_client.projects.list_projects.assert_called_once()
|
||||
kwargs = mock_client.projects.list_projects.call_args.kwargs
|
||||
assert kwargs["organization_id"] == organization_id
|
||||
assert kwargs["request"].name == DEFAULT_PROJECT_NAME
|
||||
assert kwargs["project_name"] == DEFAULT_PROJECT_NAME
|
||||
|
||||
# Pipeline created under the upserted project id
|
||||
# Pipeline created under the found project id
|
||||
assert (
|
||||
mock_client.pipelines.upsert_pipeline.call_args.kwargs["project_id"]
|
||||
== upserted_project.id
|
||||
== found_project.id
|
||||
)
|
||||
assert index.project.id == upserted_project.id
|
||||
assert index.project.id == found_project.id
|
||||
|
||||
Generated
+6
-6
@@ -1,5 +1,5 @@
|
||||
version = 1
|
||||
revision = 2
|
||||
revision = 3
|
||||
requires-python = ">=3.9, <4.0"
|
||||
resolution-markers = [
|
||||
"python_full_version >= '3.14'",
|
||||
@@ -1595,21 +1595,21 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "llama-cloud"
|
||||
version = "0.1.45"
|
||||
version = "0.1.46"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "certifi" },
|
||||
{ name = "httpx" },
|
||||
{ name = "pydantic" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e0/b7/3a2a209f1c3fa516de172cb13e03f5a897adea5523f2ee0f544d035e3704/llama_cloud-0.1.45.tar.gz", hash = "sha256:140244008cc5710e31ae97c6043973a3a9969a51b0f38155fa33a8434078e8aa", size = 140968, upload-time = "2025-12-03T02:22:49.484Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/40/f3/f4d6520f8d546e6c5a02f6ebeed5c09774a074b8d2c24ad559ace97a56a6/llama_cloud-0.1.46.tar.gz", hash = "sha256:e86f8791c053590d70cc59e0fc13ce72f9b681a8e658bc61df86d0285288d8ee", size = 127752, upload-time = "2026-01-21T18:40:57.103Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/62/1d/466b0df69b81ce9410ad6ec7229a1e6601ff69640f02f246e06cfcc7428c/llama_cloud-0.1.45-py3-none-any.whl", hash = "sha256:500299a6d3f25f97bcf6755d6338523023564fa8f376955c2cf299bbc9561cc2", size = 397184, upload-time = "2025-12-03T02:22:48.335Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/3a/6caaea28c8c804add33c91d356ed7d5a5412d6c9598e1450af95a15e0bcd/llama_cloud-0.1.46-py3-none-any.whl", hash = "sha256:6c6546c09c04a038c86d84d42f00eae8fd3bff49991ad3aab844bd866ecdf352", size = 361989, upload-time = "2026-01-21T18:40:54.863Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "llama-cloud-services"
|
||||
version = "0.6.85"
|
||||
version = "0.6.90"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
|
||||
@@ -1649,7 +1649,7 @@ dev = [
|
||||
requires-dist = [
|
||||
{ name = "click", specifier = ">=8.1.7,<9" },
|
||||
{ name = "eval-type-backport", marker = "python_full_version < '3.10'", specifier = ">=0.2.0,<0.3" },
|
||||
{ name = "llama-cloud", specifier = "==0.1.45" },
|
||||
{ name = "llama-cloud", specifier = "==0.1.46" },
|
||||
{ name = "llama-index-core", specifier = ">=0.12.0" },
|
||||
{ name = "packaging", specifier = ">=23.0" },
|
||||
{ name = "platformdirs", specifier = ">=4.3.7,<5" },
|
||||
|
||||
@@ -1,5 +1,38 @@
|
||||
# llama-cloud-services
|
||||
|
||||
## 0.5.3
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- d7864af: bugfixes in retry logic for LlamaExtract and LlamaClassify
|
||||
|
||||
## 0.5.2
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 997bcc8: Add types for bounding boxes
|
||||
|
||||
## 0.5.1
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- d5b18a0: Fix publishing
|
||||
|
||||
## 0.5.0
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- 576c3d9: feat: support zod v4 & v3
|
||||
|
||||
Adds support for zod v4 while maintaining backward compatibility with v3.
|
||||
- Updated zod peer dependency to accept both v3 and v4: `^3.25.76 || ^4.0.0`
|
||||
- Migrated all import statements to use `zod/v4` import path for compatibility
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- c8321d2: Improve parse results polling
|
||||
- 576c3d9: Support zod v3 an v4
|
||||
|
||||
## 0.4.3
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "llama-cloud-services",
|
||||
"version": "0.4.3",
|
||||
"version": "0.5.3",
|
||||
"type": "module",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
"get-openapi": "node ./scripts/get-openapi.js",
|
||||
"generate": "./node_modules/.bin/openapi-ts",
|
||||
"build": "pnpm run generate && bunchee",
|
||||
"build": "bunchee",
|
||||
"dev": "bunchee --watch",
|
||||
"lint": "eslint src/ --ignore-pattern client/*.ts --no-warn-ignored",
|
||||
"format": "prettier --write ./src/ tests/",
|
||||
@@ -116,9 +116,9 @@
|
||||
"@eslint/js": "^9.32.0",
|
||||
"@hey-api/client-fetch": "^0.10.1",
|
||||
"@hey-api/openapi-ts": "^0.67.5",
|
||||
"@llamaindex/core": "^0.6.19",
|
||||
"@llamaindex/core": "^0.6.22",
|
||||
"@llamaindex/env": "^0.1.30",
|
||||
"@llamaindex/workflow-core": "^0.4.1",
|
||||
"@llamaindex/workflow-core": "^1.3.3",
|
||||
"@types/node": "^20.19.9",
|
||||
"@typescript-eslint/eslint-plugin": "^8.38.0",
|
||||
"@typescript-eslint/parser": "^8.38.0",
|
||||
@@ -131,18 +131,19 @@
|
||||
"turbo": "^2.5.5",
|
||||
"typescript": "^5.8.3",
|
||||
"typescript-eslint": "^8.38.0",
|
||||
"vitest": "^2.0.0"
|
||||
"vitest": "^2.0.0",
|
||||
"zod": "^4.1.13"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@llamaindex/core": "^0.6.19",
|
||||
"@llamaindex/env": "^0.1.30",
|
||||
"@llamaindex/workflow-core": "^0.4.1"
|
||||
"@llamaindex/workflow-core": "^1.3.3",
|
||||
"zod": "^3.25.0 || ^4.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"ajv": "^8.17.1",
|
||||
"file-type": "^21.0.0",
|
||||
"p-retry": "^6.2.1",
|
||||
"zod": "^3.25.76"
|
||||
"p-retry": "^6.2.1"
|
||||
},
|
||||
"packageManager": "pnpm@10.8.1"
|
||||
}
|
||||
|
||||
@@ -2,11 +2,14 @@ export { AgentClient, createAgentDataClient } from "./client";
|
||||
|
||||
export type {
|
||||
AggregateAgentDataOptions,
|
||||
BoundingBox,
|
||||
ComparisonOperator,
|
||||
ExtractedData,
|
||||
ExtractedFieldMetadata,
|
||||
ExtractedFieldMetadataDict,
|
||||
FieldCitation,
|
||||
FilterOperation,
|
||||
PageDimensions,
|
||||
SearchAgentDataOptions,
|
||||
StatusType,
|
||||
TypedAgentData,
|
||||
|
||||
@@ -28,6 +28,44 @@ export type ComparisonOperator =
|
||||
*/
|
||||
export type FilterOperation = RawFilterOperation;
|
||||
|
||||
/**
|
||||
* Bounding box coordinates for a citation location on a page
|
||||
*/
|
||||
export interface BoundingBox {
|
||||
/** X coordinate of the bounding box origin */
|
||||
x: number;
|
||||
/** Y coordinate of the bounding box origin */
|
||||
y: number;
|
||||
/** Width of the bounding box */
|
||||
w: number;
|
||||
/** Height of the bounding box */
|
||||
h: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Dimensions of a page in the source document
|
||||
*/
|
||||
export interface PageDimensions {
|
||||
/** Width of the page */
|
||||
width: number;
|
||||
/** Height of the page */
|
||||
height: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Citation information for an extracted field
|
||||
*/
|
||||
export interface FieldCitation {
|
||||
/** The page number that the field occurred on */
|
||||
page?: number;
|
||||
/** The original text this field's value was derived from */
|
||||
matching_text?: string;
|
||||
/** Bounding boxes indicating where the citation appears on the page */
|
||||
bounding_boxes?: BoundingBox[];
|
||||
/** Dimensions of the page containing the citation */
|
||||
page_dimensions?: PageDimensions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Metadata for an extracted field, including confidence and citation information
|
||||
*/
|
||||
@@ -38,16 +76,11 @@ export interface ExtractedFieldMetadata {
|
||||
confidence?: number;
|
||||
/** The confidence score for the field based on the extracted text only */
|
||||
extraction_confidence?: number;
|
||||
/** The confidence score for the field based on the parsing/OCR quality */
|
||||
parsing_confidence?: number;
|
||||
citation?: FieldCitation[];
|
||||
}
|
||||
|
||||
export interface FieldCitation {
|
||||
/** The page number that the field occurred on */
|
||||
page?: number;
|
||||
/** The original text this field's value was derived from */
|
||||
matching_text?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Dictionary mapping field names to their metadata
|
||||
* Values can be ExtractedFieldMetadata objects, nested dictionaries, or arrays
|
||||
|
||||
@@ -108,20 +108,19 @@ async function pollForJobCompletion({
|
||||
}
|
||||
const response =
|
||||
await getClassifyJobApiV1ClassifierJobsClassifyJobIdGet(jobOptions);
|
||||
if (!response.response.ok) {
|
||||
numIterations++;
|
||||
}
|
||||
if (typeof response.data != "undefined") {
|
||||
status = response.data.status as StatusEnum;
|
||||
if (status == StatusEnum.CANCELLED || status == StatusEnum.ERROR) {
|
||||
throw new Error("There was an error during the classification job.");
|
||||
} else if (status == StatusEnum.SUCCESS) {
|
||||
throw new Error("There was an error extracting data from your file.");
|
||||
} else if (
|
||||
status == StatusEnum.SUCCESS ||
|
||||
status == StatusEnum.PARTIAL_SUCCESS
|
||||
) {
|
||||
return true;
|
||||
} else {
|
||||
numIterations++;
|
||||
await sleep(interval * 1000);
|
||||
}
|
||||
}
|
||||
numIterations++;
|
||||
await sleep(interval * 1000);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,7 +168,7 @@ async function getJobResult({
|
||||
retries++;
|
||||
await sleep(retryInterval * 1000);
|
||||
}
|
||||
if (typeof response.data != "undefined") {
|
||||
if (response.response.ok && typeof response.data != "undefined") {
|
||||
return response.data as ClassifyJobResults;
|
||||
} else {
|
||||
throw new Error(
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// This file is auto-generated by @hey-api/openapi-ts
|
||||
|
||||
import { z } from "zod";
|
||||
import { z } from "zod/v4";
|
||||
|
||||
export const zNoneSegmentationConfig = z.object({
|
||||
mode: z.literal("none").optional().default("none"),
|
||||
@@ -770,7 +770,7 @@ export const zMetadataFilter = z.object({
|
||||
|
||||
export const zFilterCondition: z.ZodTypeAny = z.enum(["and", "or", "not"]);
|
||||
|
||||
export const zMetadataFilters: z.AnyZodObject = z.object({
|
||||
export const zMetadataFilters: z.ZodObject<z.ZodRawShape> = z.object({
|
||||
filters: z.array(z.unknown()),
|
||||
condition: z.union([zFilterCondition, z.null()]).optional(),
|
||||
});
|
||||
@@ -782,7 +782,7 @@ export const zRetrievalMode: z.ZodTypeAny = z.enum([
|
||||
"auto_routed",
|
||||
]);
|
||||
|
||||
export const zPresetRetrievalParams: z.AnyZodObject = z.object({
|
||||
export const zPresetRetrievalParams: z.ZodObject<z.ZodRawShape> = z.object({
|
||||
dense_similarity_top_k: z
|
||||
.union([z.number().int().gte(1).lte(100), z.null()])
|
||||
.optional(),
|
||||
@@ -825,7 +825,7 @@ export const zSupportedLlmModelNames: z.ZodTypeAny = z.enum([
|
||||
"VERTEX_AI_CLAUDE_3_5_SONNET_V2",
|
||||
]);
|
||||
|
||||
export const zLlmParameters: z.AnyZodObject = z.object({
|
||||
export const zLlmParameters: z.ZodObject<z.ZodRawShape> = z.object({
|
||||
model_name: zSupportedLlmModelNames.optional(),
|
||||
system_prompt: z.union([z.string().max(3000), z.null()]).optional(),
|
||||
temperature: z.union([z.number(), z.null()]).optional(),
|
||||
@@ -834,7 +834,7 @@ export const zLlmParameters: z.AnyZodObject = z.object({
|
||||
class_name: z.string().optional().default("base_component"),
|
||||
});
|
||||
|
||||
export const zChatData: z.AnyZodObject = z.object({
|
||||
export const zChatData: z.ZodObject<z.ZodRawShape> = z.object({
|
||||
retrieval_parameters: zPresetRetrievalParams.optional(),
|
||||
llm_parameters: z.union([zLlmParameters, z.null()]).optional(),
|
||||
class_name: z.string().optional().default("base_component"),
|
||||
@@ -2141,7 +2141,7 @@ export const zTextItem = z.object({
|
||||
value: z.string(),
|
||||
});
|
||||
|
||||
export const zListItem: z.AnyZodObject = z.object({
|
||||
export const zListItem: z.ZodObject<z.ZodRawShape> = z.object({
|
||||
type: z.literal("list").optional().default("list"),
|
||||
bBox: z.union([z.unknown(), z.null()]).optional(),
|
||||
items: z.array(z.unknown()),
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { workflowEvent } from "@llamaindex/workflow-core";
|
||||
import { zodEvent } from "@llamaindex/workflow-core/util/zod";
|
||||
import { z } from "zod";
|
||||
import { z } from "zod/v4";
|
||||
import { parseFormSchema } from "./schema";
|
||||
|
||||
export const uploadEvent = zodEvent(
|
||||
|
||||
@@ -296,20 +296,16 @@ async function pollForJobCompletion(
|
||||
return false;
|
||||
}
|
||||
const response = await getJobApiV1ExtractionJobsJobIdGet(jobOptions);
|
||||
if (!response.response.ok) {
|
||||
numIterations++;
|
||||
}
|
||||
if (typeof response.data != "undefined") {
|
||||
status = response.data.status as StatusEnum;
|
||||
if (status == StatusEnum.CANCELLED || status == StatusEnum.ERROR) {
|
||||
throw new Error("There was an error extracting data from your file.");
|
||||
} else if (status == StatusEnum.SUCCESS) {
|
||||
return true;
|
||||
} else {
|
||||
numIterations++;
|
||||
await sleep(interval * 1000);
|
||||
}
|
||||
}
|
||||
numIterations++;
|
||||
await sleep(interval * 1000);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -350,7 +346,7 @@ async function getJobResult(
|
||||
retries++;
|
||||
await sleep(retryInterval * 1000);
|
||||
}
|
||||
if (typeof response.data != "undefined") {
|
||||
if (response.response.ok && typeof response.data != "undefined") {
|
||||
return {
|
||||
data: response.data.data,
|
||||
extractionMetadata: response.data.extraction_metadata,
|
||||
|
||||
@@ -2,7 +2,7 @@ import type { JSONValue } from "@llamaindex/core/global";
|
||||
import type { ToolMetadata } from "@llamaindex/core/llms";
|
||||
import type { BaseQueryEngine } from "@llamaindex/core/query-engine";
|
||||
import { tool } from "@llamaindex/core/tools";
|
||||
import { z } from "zod";
|
||||
import { z } from "zod/v4";
|
||||
|
||||
const DEFAULT_NAME = "llama_cloud_index_tool";
|
||||
const DEFAULT_DESCRIPTION =
|
||||
@@ -21,9 +21,7 @@ export function createQueryEngineTool(
|
||||
name: metadata?.name ?? DEFAULT_NAME,
|
||||
description: metadata?.description ?? DEFAULT_DESCRIPTION,
|
||||
parameters: z.object({
|
||||
query: z.string({
|
||||
description: "The query to search for",
|
||||
}),
|
||||
query: z.string().describe("The query to search for"),
|
||||
}),
|
||||
execute: async ({ query }) => {
|
||||
const response = await queryEngine.query({ query });
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
import { type Client, createClient, createConfig } from "@hey-api/client-fetch";
|
||||
import { type FailedAttemptError } from "p-retry";
|
||||
import { Document, FileReader } from "@llamaindex/core/schema";
|
||||
import { fs, getEnv, path } from "@llamaindex/env";
|
||||
import {
|
||||
type BodyUploadFileApiParsingUploadPost,
|
||||
type BodyUploadFileApiV1ParsingUploadPost,
|
||||
type FailPageMode,
|
||||
type ParserLanguages,
|
||||
type ParsingMode,
|
||||
@@ -32,6 +33,33 @@ type WriteStream = {
|
||||
// eslint-disable-next-line no-var
|
||||
var process: any;
|
||||
|
||||
function handleFailedAttempt(
|
||||
error: FailedAttemptError,
|
||||
jobId: string,
|
||||
verbose: boolean,
|
||||
) {
|
||||
// Retry only on 5XX or socket errors.
|
||||
const status = (error.cause as any)?.response?.status;
|
||||
if (
|
||||
!(
|
||||
(status && status >= 500 && status < 600) ||
|
||||
((error.cause as any)?.code &&
|
||||
((error.cause as any).code === "ECONNRESET" ||
|
||||
(error.cause as any).code === "ETIMEDOUT" ||
|
||||
(error.cause as any).code === "ECONNREFUSED")) ||
|
||||
(status && status === 404)
|
||||
)
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (verbose) {
|
||||
console.warn(
|
||||
`Attempting to get job ${jobId} result (attempt ${error.attemptNumber}) failed. Retrying...`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a reader for parsing files using the LlamaParse API.
|
||||
* See https://github.com/run-llama/llama_parse
|
||||
@@ -188,6 +216,16 @@ export class LlamaParseReader extends FileReader {
|
||||
extract_printed_page_number?: boolean | undefined;
|
||||
tier?: string | undefined;
|
||||
version?: string | undefined;
|
||||
layout_aware?: boolean | undefined;
|
||||
line_level_bounding_box?: boolean | undefined;
|
||||
specialized_image_parsing?: boolean | undefined;
|
||||
aggressive_table_extraction?: boolean | undefined;
|
||||
preserve_very_small_text?: boolean | undefined;
|
||||
spreadsheet_force_formula_computation?: boolean | undefined;
|
||||
inline_images_in_markdown?: boolean | undefined;
|
||||
keep_page_separator_when_merging_tables?: boolean | undefined;
|
||||
remove_hidden_text?: boolean | undefined;
|
||||
presentation_out_of_bounds_content?: boolean | undefined;
|
||||
|
||||
constructor(
|
||||
params: Partial<Omit<LlamaParseReader, "language" | "apiKey">> & {
|
||||
@@ -387,11 +425,25 @@ export class LlamaParseReader extends FileReader {
|
||||
extract_printed_page_number: this.extract_printed_page_number,
|
||||
tier: this.tier,
|
||||
version: this.version,
|
||||
layout_aware: this.layout_aware,
|
||||
line_level_bounding_box: this.line_level_bounding_box,
|
||||
specialized_image_parsing: this.specialized_image_parsing,
|
||||
aggressive_table_extraction: this.aggressive_table_extraction,
|
||||
preserve_very_small_text: this.preserve_very_small_text,
|
||||
spreadsheet_force_formula_computation:
|
||||
this.spreadsheet_force_formula_computation,
|
||||
inline_images_in_markdown: this.inline_images_in_markdown,
|
||||
webhook_configurations: undefined,
|
||||
keep_page_separator_when_merging_tables:
|
||||
this.keep_page_separator_when_merging_tables,
|
||||
remove_hidden_text: this.remove_hidden_text,
|
||||
presentation_out_of_bounds_content:
|
||||
this.presentation_out_of_bounds_content,
|
||||
} satisfies {
|
||||
[Key in keyof BodyUploadFileApiParsingUploadPost]-?:
|
||||
| BodyUploadFileApiParsingUploadPost[Key]
|
||||
[Key in keyof BodyUploadFileApiV1ParsingUploadPost]-?:
|
||||
| BodyUploadFileApiV1ParsingUploadPost[Key]
|
||||
| undefined;
|
||||
} as unknown as BodyUploadFileApiParsingUploadPost;
|
||||
} as unknown as BodyUploadFileApiV1ParsingUploadPost;
|
||||
|
||||
const response = await uploadFileApiV1ParsingUploadPost({
|
||||
client: this.#client,
|
||||
@@ -443,26 +495,8 @@ export class LlamaParseReader extends FileReader {
|
||||
}),
|
||||
{
|
||||
retries: this.maxErrorCount,
|
||||
onFailedAttempt: (error) => {
|
||||
// Retry only on 5XX or socket errors.
|
||||
const status = (error.cause as any)?.response?.status;
|
||||
if (
|
||||
!(
|
||||
(status && status >= 500 && status < 600) ||
|
||||
((error.cause as any)?.code &&
|
||||
((error.cause as any).code === "ECONNRESET" ||
|
||||
(error.cause as any).code === "ETIMEDOUT" ||
|
||||
(error.cause as any).code === "ECONNREFUSED"))
|
||||
)
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
if (this.verbose) {
|
||||
console.warn(
|
||||
`Attempting to get job ${jobId} result (attempt ${error.attemptNumber}) failed. Retrying...`,
|
||||
);
|
||||
}
|
||||
},
|
||||
onFailedAttempt: (error) =>
|
||||
handleFailedAttempt(error, jobId, this.verbose),
|
||||
},
|
||||
);
|
||||
} catch (e: any) {
|
||||
@@ -475,49 +509,69 @@ export class LlamaParseReader extends FileReader {
|
||||
const status = (data as Record<string, unknown>)["status"];
|
||||
|
||||
if (status === "SUCCESS") {
|
||||
let resultData;
|
||||
switch (resultType) {
|
||||
case "json": {
|
||||
resultData =
|
||||
await getJobJsonResultApiV1ParsingJobJobIdResultJsonGet({
|
||||
client: this.#client,
|
||||
throwOnError: true,
|
||||
path: { job_id: jobId },
|
||||
query: {
|
||||
organization_id: this.organization_id ?? null,
|
||||
},
|
||||
signal: AbortSignal.timeout(this.maxTimeout * 1000),
|
||||
});
|
||||
break;
|
||||
const resultData = await pRetry(
|
||||
() =>
|
||||
getJobJsonResultApiV1ParsingJobJobIdResultJsonGet({
|
||||
client: this.#client,
|
||||
throwOnError: true,
|
||||
path: { job_id: jobId },
|
||||
query: {
|
||||
organization_id: this.organization_id ?? null,
|
||||
},
|
||||
signal: AbortSignal.timeout(this.maxTimeout * 1000),
|
||||
}),
|
||||
{
|
||||
retries: this.maxErrorCount,
|
||||
onFailedAttempt: (error) =>
|
||||
handleFailedAttempt(error, jobId, this.verbose),
|
||||
},
|
||||
);
|
||||
return resultData.data;
|
||||
}
|
||||
case "markdown": {
|
||||
resultData =
|
||||
await getJobResultApiV1ParsingJobJobIdResultMarkdownGet({
|
||||
client: this.#client,
|
||||
throwOnError: true,
|
||||
path: { job_id: jobId },
|
||||
query: {
|
||||
organization_id: this.organization_id ?? null,
|
||||
},
|
||||
signal: AbortSignal.timeout(this.maxTimeout * 1000),
|
||||
});
|
||||
break;
|
||||
const resultData = await pRetry(
|
||||
() =>
|
||||
getJobResultApiV1ParsingJobJobIdResultMarkdownGet({
|
||||
client: this.#client,
|
||||
throwOnError: true,
|
||||
path: { job_id: jobId },
|
||||
query: {
|
||||
organization_id: this.organization_id ?? null,
|
||||
},
|
||||
signal: AbortSignal.timeout(this.maxTimeout * 1000),
|
||||
}),
|
||||
{
|
||||
retries: this.maxErrorCount,
|
||||
onFailedAttempt: (error) =>
|
||||
handleFailedAttempt(error, jobId, this.verbose),
|
||||
},
|
||||
);
|
||||
return resultData.data;
|
||||
}
|
||||
case "text": {
|
||||
resultData =
|
||||
await getJobTextResultApiV1ParsingJobJobIdResultTextGet({
|
||||
client: this.#client,
|
||||
throwOnError: true,
|
||||
path: { job_id: jobId },
|
||||
query: {
|
||||
organization_id: this.organization_id ?? null,
|
||||
},
|
||||
signal: AbortSignal.timeout(this.maxTimeout * 1000),
|
||||
});
|
||||
break;
|
||||
const resultData = await pRetry(
|
||||
() =>
|
||||
getJobTextResultApiV1ParsingJobJobIdResultTextGet({
|
||||
client: this.#client,
|
||||
throwOnError: true,
|
||||
path: { job_id: jobId },
|
||||
query: {
|
||||
organization_id: this.organization_id ?? null,
|
||||
},
|
||||
signal: AbortSignal.timeout(this.maxTimeout * 1000),
|
||||
}),
|
||||
{
|
||||
retries: this.maxErrorCount,
|
||||
onFailedAttempt: (error) =>
|
||||
handleFailedAttempt(error, jobId, this.verbose),
|
||||
},
|
||||
);
|
||||
|
||||
return resultData.data;
|
||||
}
|
||||
}
|
||||
return resultData.data;
|
||||
} else if (status === "PENDING") {
|
||||
if (this.verbose && tries % 10 === 0) {
|
||||
this.stdout?.write(".");
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { FailPageMode, ParserLanguages, ParsingMode } from "./client";
|
||||
|
||||
import { z } from "zod";
|
||||
import { z } from "zod/v4";
|
||||
|
||||
type Language = ParserLanguages;
|
||||
const VALUES: [Language, ...Language[]] = [
|
||||
@@ -52,9 +52,10 @@ export const parseFormSchema = z.object({
|
||||
html_remove_navigation_elements: z.boolean().optional(),
|
||||
http_proxy: z
|
||||
.string()
|
||||
.url(
|
||||
'Set a valid URL for the HTTP proxy, e.g., "http://proxy.example.com:8080"',
|
||||
)
|
||||
.url({
|
||||
error:
|
||||
'Set a valid URL for the HTTP proxy, e.g., "http://proxy.example.com:8080"',
|
||||
})
|
||||
.refine(
|
||||
(url) => {
|
||||
try {
|
||||
@@ -67,7 +68,7 @@ export const parseFormSchema = z.object({
|
||||
}
|
||||
},
|
||||
{
|
||||
message: "Invalid HTTP proxy URL",
|
||||
error: "Invalid HTTP proxy URL",
|
||||
},
|
||||
)
|
||||
.optional(),
|
||||
@@ -100,7 +101,7 @@ export const parseFormSchema = z.object({
|
||||
vendor_multimodal_model_name: z.string().optional(),
|
||||
model: z.string().optional(),
|
||||
webhook_url: z.string().url().optional(),
|
||||
parse_mode: z.nativeEnum(ParsingMode).nullable().optional(),
|
||||
parse_mode: z.enum(ParsingMode).nullable().optional(),
|
||||
system_prompt: z.string().optional(),
|
||||
system_prompt_append: z.string().optional(),
|
||||
user_prompt: z.string().optional(),
|
||||
@@ -129,7 +130,7 @@ export const parseFormSchema = z.object({
|
||||
compact_markdown_table: z.boolean().optional(),
|
||||
markdown_table_multiline_header_separator: z.string().optional(),
|
||||
page_error_tolerance: z.number().min(0).max(1).optional(),
|
||||
replace_failed_page_mode: z.nativeEnum(FailPageMode).nullable().optional(),
|
||||
replace_failed_page_mode: z.enum(FailPageMode).nullable().optional(),
|
||||
replace_failed_page_with_error_message_prefix: z.string().optional(),
|
||||
replace_failed_page_with_error_message_suffix: z.string().optional(),
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user