mirror of
https://github.com/run-llama/llama_cloud_services.git
synced 2026-07-21 12:05:23 -04:00
Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| acfc90d0a8 | |||
| e80376c97e | |||
| 5acca1fc98 | |||
| 68cb4dacd3 | |||
| 0b16edf058 | |||
| 5bb9816f04 | |||
| d55f8cf153 | |||
| 09a911ee62 |
@@ -1,95 +0,0 @@
|
||||
name: Claude Code
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
pull_request_review_comment:
|
||||
types: [created]
|
||||
issues:
|
||||
types: [opened, assigned]
|
||||
pull_request_review:
|
||||
types: [submitted]
|
||||
|
||||
jobs:
|
||||
claude:
|
||||
if: |
|
||||
(github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
|
||||
(github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
issues: read
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Check repository access
|
||||
id: check-access
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
# Get the user who triggered the event
|
||||
case "${{ github.event_name }}" in
|
||||
"issue_comment")
|
||||
USER="${{ github.event.comment.user.login }}"
|
||||
;;
|
||||
"pull_request_review_comment")
|
||||
USER="${{ github.event.comment.user.login }}"
|
||||
;;
|
||||
"pull_request_review")
|
||||
USER="${{ github.event.review.user.login }}"
|
||||
;;
|
||||
"issues")
|
||||
USER="${{ github.event.issue.user.login }}"
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "Checking repository access for user: $USER"
|
||||
|
||||
# Check if user has write access to the repository
|
||||
REPO="${{ github.repository }}"
|
||||
if gh api repos/$REPO/collaborators/$USER/permission --jq '.permission' | grep -E "(admin|write)" > /dev/null 2>&1; then
|
||||
echo "User $USER has write access to the repository"
|
||||
echo "authorized=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "User $USER does not have write access to the repository"
|
||||
echo "authorized=false" >> $GITHUB_OUTPUT
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Checkout repository
|
||||
if: steps.check-access.outputs.authorized == 'true'
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Run Claude Code
|
||||
if: steps.check-access.outputs.authorized == 'true'
|
||||
id: claude
|
||||
uses: anthropics/claude-code-action@beta
|
||||
with:
|
||||
anthropic_api_key: ${{ secrets.ANTHROPIC_GITHUB_API_KEY }}
|
||||
|
||||
# Optional: Specify model (defaults to Claude Sonnet 4, uncomment for Claude Opus 4)
|
||||
# model: "claude-opus-4-20250514"
|
||||
|
||||
# Optional: Customize the trigger phrase (default: @claude)
|
||||
# trigger_phrase: "/claude"
|
||||
|
||||
# Optional: Trigger when specific user is assigned to an issue
|
||||
# assignee_trigger: "claude-bot"
|
||||
|
||||
# Optional: Allow Claude to run specific commands
|
||||
# Allow bash commands to be run, for things like running tests, linting, etc.
|
||||
allowed_tools: "Bash(rg:*),Bash(find:*),Bash(grep:*),Bash(pnpm:*),Bash(npm:*),Bash(uv:*),Bash(pip:*),Bash(pipx:*),Bash(make:*),Bash(cd:*),WebFetch"
|
||||
|
||||
# Optional: Add custom instructions for Claude to customize its behavior for your project
|
||||
# custom_instructions: |
|
||||
# Follow our coding standards
|
||||
# Ensure all new code has tests
|
||||
# Use TypeScript for new files
|
||||
|
||||
# Optional: Custom environment variables for Claude
|
||||
# claude_env: |
|
||||
# NODE_ENV: test
|
||||
@@ -28,6 +28,7 @@ jobs:
|
||||
with:
|
||||
node-version-file: "ts/llama_cloud_services/.nvmrc"
|
||||
- name: Install dependencies
|
||||
working-directory: ts/llama_cloud_services/
|
||||
run: pnpm install --no-frozen-lockfile
|
||||
- name: Run lint
|
||||
working-directory: ts/llama_cloud_services/
|
||||
|
||||
@@ -22,12 +22,9 @@ jobs:
|
||||
node-version-file: "ts/llama_cloud_services/.nvmrc"
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: ts/llama_cloud_services
|
||||
run: pnpm install --no-frozen-lockfile
|
||||
|
||||
- name: Run Build
|
||||
working-directory: ts/llama_cloud_services/
|
||||
run: pnpm build
|
||||
|
||||
- name: Build tarball
|
||||
run: |
|
||||
pnpm pack
|
||||
|
||||
@@ -30,13 +30,8 @@ jobs:
|
||||
with:
|
||||
node-version-file: "ts/llama_cloud_services/.nvmrc"
|
||||
- name: Install dependencies
|
||||
working-directory: ts/llama_cloud_services/
|
||||
run: pnpm install --no-frozen-lockfile
|
||||
- name: Run Build
|
||||
working-directory: ts/llama_cloud_services/
|
||||
run: pnpm build
|
||||
- name: Run Tests
|
||||
- name: Run tests
|
||||
working-directory: ts/llama_cloud_services/
|
||||
run: pnpm test
|
||||
- name: Run e2e tests
|
||||
working-directory: ts/e2e-tests/
|
||||
run: pnpm test
|
||||
|
||||
@@ -15,7 +15,6 @@ repos:
|
||||
- id: end-of-file-fixer
|
||||
- id: mixed-line-ending
|
||||
- id: trailing-whitespace
|
||||
exclude: ^ts/llama_cloud_services/src/client/
|
||||
- repo: https://github.com/charliermarsh/ruff-pre-commit
|
||||
rev: v0.1.5
|
||||
|
||||
@@ -64,13 +63,13 @@ repos:
|
||||
rev: v3.0.3
|
||||
hooks:
|
||||
- id: prettier
|
||||
exclude: ^(uv.lock|ts/llama_cloud_services/pnpm-lock.yaml|ts/e2e-tests)
|
||||
exclude: ^(uv.lock|ts/llama_cloud_services/pnpm-lock.yaml)
|
||||
- repo: https://github.com/codespell-project/codespell
|
||||
rev: v2.2.6
|
||||
hooks:
|
||||
- id: codespell
|
||||
additional_dependencies: [tomli]
|
||||
exclude: ^(uv.lock|docs|ts|examples|pnpm-lock.yaml)
|
||||
exclude: ^(uv.lock|docs|ts|examples)
|
||||
args:
|
||||
[
|
||||
"--ignore-words-list",
|
||||
@@ -87,4 +86,4 @@ repos:
|
||||
- id: toml-sort-fix
|
||||
exclude: ".*uv.lock"
|
||||
|
||||
exclude: ^(.github/ISSUE_TEMPLATE|ts/llama_cloud_services/src/client|pnpm-lock.yaml)
|
||||
exclude: .github/ISSUE_TEMPLATE
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
# LlamaCloud Services Examples - Python
|
||||
|
||||
In this folder you will find several TypeScript end-to-end applications that contain examples regarding:
|
||||
|
||||
- [LlamaParse](./parse/)
|
||||
- [LlamaCloud Index](./index/)
|
||||
|
||||
Follow the instructions in each example folder to get started!
|
||||
+8
-6
@@ -1,9 +1,11 @@
|
||||
# LlamaCloud Services Examples - Python
|
||||
# LlamaCloud Services Examples
|
||||
|
||||
In this folder you will find several python notebooks that contain examples regarding:
|
||||
In this folder you will find several python notebooks and two end-to-end typescript applications that contain examples regarding:
|
||||
|
||||
- [LlamaParse](./parse/)
|
||||
- [LlamaExtract](./extract/)
|
||||
- [LlamaReport](./report/)
|
||||
- [LlamaParse - Python](./parse/)
|
||||
- [LlamaParse - TypeScript](./parse-ts/)
|
||||
- [LlamaExtract - Python](./extract/)
|
||||
- [LlamaReport - Python](./report/)
|
||||
- [LlamaCloud Index - TypeScript](./index-ts/)
|
||||
|
||||
Follow the instructions in each notebook to get started!
|
||||
Follow the instructions of each notebook/application to get started!
|
||||
|
||||
+27
-145
@@ -4,11 +4,8 @@ LlamaExtract provides a simple API for extracting structured data from unstructu
|
||||
|
||||
## Quick Start
|
||||
|
||||
The simplest way to get started is to use the stateless API with the extraction configuration and the file/text to extract from:
|
||||
|
||||
```python
|
||||
from llama_cloud_services import LlamaExtract
|
||||
from llama_cloud import ExtractConfig, ExtractMode
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# Initialize client
|
||||
@@ -22,87 +19,29 @@ class Resume(BaseModel):
|
||||
skills: list[str] = Field(description="Technical skills and technologies")
|
||||
|
||||
|
||||
# Configure extraction settings
|
||||
config = ExtractConfig(extraction_mode=ExtractMode.FAST)
|
||||
# Create extraction agent
|
||||
agent = extractor.create_agent(name="resume-parser", data_schema=Resume)
|
||||
|
||||
# Extract data directly from document - no agent needed!
|
||||
result = extractor.extract(Resume, config, "resume.pdf")
|
||||
# Extract data from document
|
||||
result = agent.extract("resume.pdf")
|
||||
print(result.data)
|
||||
```
|
||||
|
||||
### Supported File Types
|
||||
|
||||
LlamaExtract supports the following file formats:
|
||||
|
||||
- **Documents**: PDF (.pdf), Word (.docx)
|
||||
- **Text files**: Plain text (.txt), CSV (.csv), JSON (.json), HTML (.html, .htm), Markdown (.md)
|
||||
- **Images**: PNG (.png), JPEG (.jpg, .jpeg)
|
||||
|
||||
### Different Input Types
|
||||
|
||||
```python
|
||||
# From file path (string or Path)
|
||||
result = extractor.extract(Resume, config, "resume.pdf")
|
||||
|
||||
# From file handle
|
||||
with open("resume.pdf", "rb") as f:
|
||||
result = extractor.extract(Resume, config, f)
|
||||
|
||||
# From bytes with filename
|
||||
with open("resume.pdf", "rb") as f:
|
||||
file_bytes = f.read()
|
||||
from llama_cloud_services.extract import SourceText
|
||||
|
||||
result = extractor.extract(
|
||||
Resume, config, SourceText(file=file_bytes, filename="resume.pdf")
|
||||
)
|
||||
|
||||
# From text content
|
||||
text = "Name: John Doe\nEmail: john@example.com\nSkills: Python, AI"
|
||||
result = extractor.extract(Resume, config, SourceText(text_content=text))
|
||||
```
|
||||
|
||||
### Async Extraction
|
||||
|
||||
For better performance with multiple files or when integrating with async applications:
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
|
||||
|
||||
async def extract_resumes():
|
||||
# Async extraction
|
||||
result = await extractor.aextract(Resume, config, "resume.pdf")
|
||||
print(result.data)
|
||||
|
||||
# Queue extraction jobs (returns immediately)
|
||||
jobs = await extractor.queue_extraction(
|
||||
Resume, config, ["resume1.pdf", "resume2.pdf"]
|
||||
)
|
||||
print(f"Queued {len(jobs)} extraction jobs")
|
||||
|
||||
|
||||
# Run async function
|
||||
asyncio.run(extract_resumes())
|
||||
```
|
||||
|
||||
## Core Concepts
|
||||
|
||||
- **Extraction Agents**: Reusable extractors configured with a specific schema and extraction settings.
|
||||
- **Data Schema**: Structure definition for the data you want to extract in the form of a JSON schema or a Pydantic model.
|
||||
- **Extraction Config**: Settings that control how extraction is performed (e.g., speed vs accuracy trade-offs).
|
||||
- **Extraction Jobs**: Asynchronous extraction tasks that can be monitored.
|
||||
- **Extraction Agents** (Advanced): Reusable extractors configured with a specific schema and extraction settings.
|
||||
|
||||
## Defining Schemas
|
||||
|
||||
Schemas define the structure of data you want to extract. You can use either Pydantic models or JSON Schema:
|
||||
Schemas can be defined using either Pydantic models or JSON Schema:
|
||||
|
||||
### Using Pydantic (Recommended)
|
||||
|
||||
```python
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List, Optional
|
||||
from llama_cloud import ExtractConfig, ExtractMode
|
||||
|
||||
|
||||
class Experience(BaseModel):
|
||||
@@ -115,11 +54,6 @@ class Experience(BaseModel):
|
||||
class Resume(BaseModel):
|
||||
name: str = Field(description="Candidate name")
|
||||
experience: List[Experience] = Field(description="Work history")
|
||||
|
||||
|
||||
# Use the schema for extraction
|
||||
config = ExtractConfig(extraction_mode=ExtractMode.FAST)
|
||||
result = extractor.extract(Resume, config, "resume.pdf")
|
||||
```
|
||||
|
||||
### Using JSON Schema
|
||||
@@ -154,27 +88,7 @@ schema = {
|
||||
},
|
||||
}
|
||||
|
||||
# Use the schema for extraction
|
||||
config = ExtractConfig(extraction_mode=ExtractMode.FAST)
|
||||
result = extractor.extract(schema, config, "resume.pdf")
|
||||
```
|
||||
|
||||
## Extraction Configuration
|
||||
|
||||
Configure how extraction is performed using `ExtractConfig`:
|
||||
|
||||
```python
|
||||
from llama_cloud import ExtractConfig, ExtractMode
|
||||
|
||||
# Fast extraction (lower accuracy, faster processing)
|
||||
fast_config = ExtractConfig(extraction_mode=ExtractMode.FAST)
|
||||
|
||||
# Balanced extraction (good balance of speed and accuracy)
|
||||
balanced_config = ExtractConfig(extraction_mode=ExtractMode.BALANCED)
|
||||
|
||||
# Use different configs for different needs
|
||||
result = extractor.extract(schema, fast_config, "simple_document.pdf")
|
||||
result = extractor.extract(schema, balanced_config, "complex_document.pdf")
|
||||
agent = extractor.create_agent(name="resume-parser", data_schema=schema)
|
||||
```
|
||||
|
||||
### Important restrictions on JSON/Pydantic Schema
|
||||
@@ -194,44 +108,28 @@ be sufficient for a wide variety of use-cases.
|
||||
your extraction workflow to fit within these constraints, e.g. by extracting subset of fields
|
||||
and later merging them together.
|
||||
|
||||
## Extraction Agents (Advanced)
|
||||
## Other Extraction APIs
|
||||
|
||||
For reusable extraction workflows, you can create extraction agents that encapsulate both schema and configuration:
|
||||
### Extraction over bytes or text
|
||||
|
||||
### Creating Agents
|
||||
You can use the `SourceText` class to extract from bytes or text directly without using a file. If passing the file bytes,
|
||||
you will need to pass the filename to the `SourceText` class.
|
||||
|
||||
```python
|
||||
from llama_cloud_services import LlamaExtract
|
||||
from llama_cloud import ExtractConfig, ExtractMode
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# Initialize client
|
||||
extractor = LlamaExtract()
|
||||
|
||||
|
||||
# Define schema
|
||||
class Resume(BaseModel):
|
||||
name: str = Field(description="Full name of candidate")
|
||||
email: str = Field(description="Email address")
|
||||
skills: list[str] = Field(description="Technical skills and technologies")
|
||||
|
||||
|
||||
# Configure extraction settings
|
||||
config = ExtractConfig(extraction_mode=ExtractMode.FAST)
|
||||
|
||||
# Create extraction agent
|
||||
agent = extractor.create_agent(
|
||||
name="resume-parser", data_schema=Resume, config=config
|
||||
)
|
||||
|
||||
# Use the agent
|
||||
result = agent.extract("resume.pdf")
|
||||
print(result.data)
|
||||
with open("resume.pdf", "rb") as f:
|
||||
file_bytes = f.read()
|
||||
result = test_agent.extract(SourceText(file=file_bytes, filename="resume.pdf"))
|
||||
```
|
||||
|
||||
### Agent Batch Processing
|
||||
```python
|
||||
result = test_agent.extract(
|
||||
SourceText(text_content="Candidate Name: Jane Doe")
|
||||
)
|
||||
```
|
||||
|
||||
Process multiple files with an agent:
|
||||
### Batch Processing
|
||||
|
||||
Process multiple files asynchronously:
|
||||
|
||||
```python
|
||||
# Queue multiple files for extraction
|
||||
@@ -246,7 +144,7 @@ for job in jobs:
|
||||
results = [agent.get_extraction_run_for_job(job.id) for job in jobs]
|
||||
```
|
||||
|
||||
### Updating Agent Schemas
|
||||
### Updating Schemas
|
||||
|
||||
Schemas can be modified and updated after creation:
|
||||
|
||||
@@ -271,26 +169,10 @@ agent = extractor.get_agent(name="resume-parser")
|
||||
extractor.delete_agent(agent.id)
|
||||
```
|
||||
|
||||
### When to Use Agents vs Direct Extraction
|
||||
|
||||
**Use Direct Extraction When:**
|
||||
|
||||
- One-off extractions
|
||||
- Different schemas for different documents
|
||||
- Simple workflows
|
||||
- Getting started quickly
|
||||
|
||||
**Use Extraction Agents When:**
|
||||
|
||||
- Repeated extractions with the same schema
|
||||
- Team collaboration (shared, named extractors)
|
||||
- Complex workflows requiring state management
|
||||
- Production systems with consistent extraction patterns
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install llama-cloud-services
|
||||
pip install llama-extract==0.1.0
|
||||
```
|
||||
|
||||
## Tips & Best Practices
|
||||
@@ -311,9 +193,9 @@ At the core of LlamaExtract is the schema, which defines the structure of the da
|
||||
2. **Running Extractions**:
|
||||
- Note that resetting `agent.schema` will not save the schema to the database,
|
||||
until you call `agent.save`, but it will be used for running extractions.
|
||||
- Check extraction results for any errors. Error information is available in the `result.error` field for debugging.
|
||||
- Consider async operations (`aextract` or `queue_extraction`) for large-scale extraction or when processing multiple files.
|
||||
- For repeated extractions with the same schema, consider creating an extraction agent to avoid redefining the schema each time.
|
||||
- Check job status prior to accessing results. Any extraction error should be available as
|
||||
part of `job.error` or `extraction_run.error` fields for debugging.
|
||||
- Consider async operations (`queue_extraction`) for large-scale extraction once you have finalized your schema.
|
||||
|
||||
### Hitting "The response was too long to be processed" Error
|
||||
|
||||
|
||||
Generated
+790
-3229
File diff suppressed because it is too large
Load Diff
@@ -1,2 +0,0 @@
|
||||
packages:
|
||||
- "ts/**"
|
||||
@@ -1,5 +1,4 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import os
|
||||
import time
|
||||
from io import BufferedIOBase, BufferedReader, BytesIO, TextIOWrapper
|
||||
@@ -22,7 +21,6 @@ from llama_cloud import (
|
||||
ExtractJobCreate,
|
||||
ExtractRun,
|
||||
File,
|
||||
FileData,
|
||||
ExtractMode,
|
||||
StatusEnum,
|
||||
ExtractTarget,
|
||||
@@ -49,7 +47,7 @@ SchemaInput = Union[JSONObjectType, Type[BaseModel]]
|
||||
|
||||
DEFAULT_EXTRACT_CONFIG = ExtractConfig(
|
||||
extraction_target=ExtractTarget.PER_DOC,
|
||||
extraction_mode=ExtractMode.MULTIMODAL,
|
||||
extraction_mode=ExtractMode.BALANCED,
|
||||
)
|
||||
|
||||
|
||||
@@ -64,132 +62,6 @@ def _is_retryable_error(exception: BaseException) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
async def _validate_schema(
|
||||
client: AsyncLlamaCloud, data_schema: SchemaInput
|
||||
) -> JSONObjectType:
|
||||
"""Convert SchemaInput to a validated JSON schema dictionary."""
|
||||
processed_schema: JSONObjectType
|
||||
if isinstance(data_schema, dict):
|
||||
# TODO: if we expose a get_validated JSON schema method, we can use it here
|
||||
processed_schema = data_schema # type: ignore
|
||||
elif isinstance(data_schema, type) and issubclass(data_schema, BaseModel):
|
||||
processed_schema = data_schema.model_json_schema()
|
||||
else:
|
||||
raise ValueError("data_schema must be either a dictionary or a Pydantic model")
|
||||
|
||||
# Validate schema via API
|
||||
validated_schema = await client.llama_extract.validate_extraction_schema(
|
||||
data_schema=processed_schema
|
||||
)
|
||||
return validated_schema.data_schema
|
||||
|
||||
|
||||
async def _get_job_with_retry(
|
||||
client: AsyncLlamaCloud,
|
||||
job_id: str,
|
||||
max_attempts: int = 5,
|
||||
initial_wait: float = 1,
|
||||
max_wait: float = 60,
|
||||
jitter: float = 5,
|
||||
) -> ExtractJob:
|
||||
"""Get extraction job with retry logic."""
|
||||
async for attempt in AsyncRetrying(
|
||||
retry=retry_if_exception(_is_retryable_error),
|
||||
stop=stop_after_attempt(max_attempts),
|
||||
wait=wait_exponential_jitter(initial=initial_wait, max=max_wait, jitter=jitter),
|
||||
reraise=True,
|
||||
):
|
||||
with attempt:
|
||||
return await client.llama_extract.get_job(job_id=job_id)
|
||||
|
||||
|
||||
async def _get_run_with_retry(
|
||||
client: AsyncLlamaCloud,
|
||||
job_id: str,
|
||||
project_id: Optional[str] = None,
|
||||
organization_id: Optional[str] = None,
|
||||
max_attempts: int = 3,
|
||||
initial_wait: float = 1,
|
||||
max_wait: float = 20,
|
||||
jitter: float = 3,
|
||||
) -> ExtractRun:
|
||||
"""Get extraction run with retry logic."""
|
||||
async for attempt in AsyncRetrying(
|
||||
retry=retry_if_exception(_is_retryable_error),
|
||||
stop=stop_after_attempt(max_attempts),
|
||||
wait=wait_exponential_jitter(initial=initial_wait, max=max_wait, jitter=jitter),
|
||||
reraise=True,
|
||||
):
|
||||
with attempt:
|
||||
return await client.llama_extract.get_run_by_job_id(
|
||||
job_id=job_id,
|
||||
project_id=project_id,
|
||||
organization_id=organization_id,
|
||||
)
|
||||
|
||||
|
||||
async def _wait_for_job_result(
|
||||
client: AsyncLlamaCloud,
|
||||
job_id: str,
|
||||
check_interval: int = 1,
|
||||
max_timeout: int = 2000,
|
||||
verbose: bool = False,
|
||||
project_id: Optional[str] = None,
|
||||
organization_id: Optional[str] = None,
|
||||
job_retry_attempts: int = 5,
|
||||
job_max_wait: float = 60,
|
||||
job_jitter: float = 5,
|
||||
run_retry_attempts: int = 3,
|
||||
run_max_wait: float = 20,
|
||||
run_jitter: float = 3,
|
||||
) -> Optional[ExtractRun]:
|
||||
"""Wait for and return the results of an extraction job."""
|
||||
start = time.perf_counter()
|
||||
poll_count = 0
|
||||
|
||||
while True:
|
||||
await asyncio.sleep(check_interval)
|
||||
poll_count += 1
|
||||
job = await _get_job_with_retry(
|
||||
client,
|
||||
job_id,
|
||||
max_attempts=job_retry_attempts,
|
||||
max_wait=job_max_wait,
|
||||
jitter=job_jitter,
|
||||
)
|
||||
|
||||
if job.status == StatusEnum.SUCCESS:
|
||||
return await _get_run_with_retry(
|
||||
client,
|
||||
job_id,
|
||||
project_id,
|
||||
organization_id,
|
||||
max_attempts=run_retry_attempts,
|
||||
max_wait=run_max_wait,
|
||||
jitter=run_jitter,
|
||||
)
|
||||
elif job.status == StatusEnum.PENDING:
|
||||
end = time.perf_counter()
|
||||
if end - start > max_timeout:
|
||||
raise Exception(f"Timeout while extracting the file: {job_id}")
|
||||
if verbose and poll_count % 10 == 0:
|
||||
print(".", end="", flush=True)
|
||||
continue
|
||||
else:
|
||||
warnings.warn(
|
||||
f"Failure in job: {job_id}, status: {job.status}, error: {job.error}"
|
||||
)
|
||||
return await _get_run_with_retry(
|
||||
client,
|
||||
job_id,
|
||||
project_id,
|
||||
organization_id,
|
||||
max_attempts=run_retry_attempts,
|
||||
max_wait=run_max_wait,
|
||||
jitter=run_jitter,
|
||||
)
|
||||
|
||||
|
||||
class SourceText:
|
||||
def __init__(
|
||||
self,
|
||||
@@ -272,21 +144,6 @@ def _extraction_config_warning(config: ExtractConfig) -> None:
|
||||
"available in the `extraction_metadata` field for the extraction run.",
|
||||
ExperimentalWarning,
|
||||
)
|
||||
if config.use_reasoning:
|
||||
if config.extraction_mode == ExtractMode.FAST:
|
||||
raise ValueError(
|
||||
"`reasoning` is only supported with BALANCED, MULTIMODAL, or PREMIUM extraction modes."
|
||||
)
|
||||
if config.cite_sources:
|
||||
if config.extraction_mode in (ExtractMode.FAST, ExtractMode.BALANCED):
|
||||
raise ValueError(
|
||||
"`cite_sources` is only supported with MULTIMODAL or PREMIUM extraction modes."
|
||||
)
|
||||
if config.confidence_scores:
|
||||
if config.extraction_mode in (ExtractMode.FAST, ExtractMode.BALANCED):
|
||||
raise ValueError(
|
||||
"`confidence_scores` is only supported with MULTIMODAL or PREMIUM extraction modes."
|
||||
)
|
||||
|
||||
|
||||
class ExtractionAgent:
|
||||
@@ -337,10 +194,22 @@ class ExtractionAgent:
|
||||
|
||||
@data_schema.setter
|
||||
def data_schema(self, data_schema: SchemaInput) -> None:
|
||||
# Use the shared schema processing and validation function
|
||||
self._data_schema = self._run_in_thread(
|
||||
_validate_schema(self._client, data_schema)
|
||||
processed_schema: JSONObjectType
|
||||
if isinstance(data_schema, dict):
|
||||
# TODO: if we expose a get_validated JSON schema method, we can use it here
|
||||
processed_schema = data_schema # type: ignore
|
||||
elif isinstance(data_schema, type) and issubclass(data_schema, BaseModel):
|
||||
processed_schema = data_schema.model_json_schema()
|
||||
else:
|
||||
raise ValueError(
|
||||
"data_schema must be either a dictionary or a Pydantic model"
|
||||
)
|
||||
validated_schema = self._run_in_thread(
|
||||
self._client.llama_extract.validate_extraction_schema(
|
||||
data_schema=processed_schema
|
||||
)
|
||||
)
|
||||
self._data_schema = validated_schema.data_schema
|
||||
|
||||
@property
|
||||
def config(self) -> ExtractConfig:
|
||||
@@ -399,9 +268,7 @@ class ExtractionAgent:
|
||||
project_id=self._project_id, upload_file=file_contents
|
||||
)
|
||||
finally:
|
||||
if file_contents is not None and isinstance(
|
||||
file_contents, (BufferedReader, BytesIO)
|
||||
):
|
||||
if file_contents is not None and isinstance(file_contents, BufferedReader):
|
||||
file_contents.close()
|
||||
|
||||
async def _upload_file(self, file_input: FileInput) -> File:
|
||||
@@ -429,23 +296,60 @@ class ExtractionAgent:
|
||||
|
||||
return await self.upload_file(source_text)
|
||||
|
||||
async def _get_job_with_retry(self, job_id: str) -> ExtractJob:
|
||||
"""Get job with retry logic for transient errors."""
|
||||
async for attempt in AsyncRetrying(
|
||||
retry=retry_if_exception(_is_retryable_error),
|
||||
stop=stop_after_attempt(5),
|
||||
wait=wait_exponential_jitter(initial=1, max=60, jitter=5),
|
||||
reraise=True,
|
||||
):
|
||||
with attempt:
|
||||
return await self._client.llama_extract.get_job(job_id=job_id)
|
||||
|
||||
async def _get_run_with_retry(self, job_id: str) -> ExtractRun:
|
||||
"""Get extraction run with retry logic for transient errors."""
|
||||
async for attempt in AsyncRetrying(
|
||||
retry=retry_if_exception(_is_retryable_error),
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_exponential_jitter(initial=1, max=20, jitter=3),
|
||||
reraise=True,
|
||||
):
|
||||
with attempt:
|
||||
return await self._client.llama_extract.get_run_by_job_id(job_id=job_id)
|
||||
|
||||
async def _wait_for_job_result(self, job_id: str) -> Optional[ExtractRun]:
|
||||
"""Wait for and return the results of an extraction job."""
|
||||
return await _wait_for_job_result(
|
||||
client=self._client,
|
||||
job_id=job_id,
|
||||
check_interval=self.check_interval,
|
||||
max_timeout=self.max_timeout,
|
||||
verbose=self._verbose,
|
||||
project_id=self._project_id,
|
||||
organization_id=self._organization_id,
|
||||
job_retry_attempts=5,
|
||||
job_max_wait=60,
|
||||
job_jitter=5,
|
||||
run_retry_attempts=3,
|
||||
run_max_wait=20,
|
||||
run_jitter=3,
|
||||
)
|
||||
start = time.perf_counter()
|
||||
tries = 0
|
||||
|
||||
while True:
|
||||
await asyncio.sleep(self.check_interval)
|
||||
tries += 1
|
||||
|
||||
try:
|
||||
job = await self._get_job_with_retry(job_id)
|
||||
|
||||
if job.status == StatusEnum.SUCCESS:
|
||||
return await self._get_run_with_retry(job_id)
|
||||
elif job.status == StatusEnum.PENDING:
|
||||
end = time.perf_counter()
|
||||
if end - start > self.max_timeout:
|
||||
raise Exception(f"Timeout while extracting the file: {job_id}")
|
||||
if self._verbose and tries % 10 == 0:
|
||||
print(".", end="", flush=True)
|
||||
continue
|
||||
else:
|
||||
warnings.warn(
|
||||
f"Failure in job: {job_id}, status: {job.status}, error: {job.error}"
|
||||
)
|
||||
return await self._get_run_with_retry(job_id)
|
||||
|
||||
except Exception as e:
|
||||
# If we get a non-retryable error or all retries are exhausted, re-raise
|
||||
if self._verbose:
|
||||
print(f"\nError in job polling for {job_id}: {e}")
|
||||
raise e
|
||||
|
||||
def save(self) -> None:
|
||||
"""Persist the extraction agent's schema and config to the database.
|
||||
@@ -739,14 +643,12 @@ class LlamaExtract(BaseComponent):
|
||||
base_url = os.getenv("LLAMA_CLOUD_BASE_URL", None) or DEFAULT_BASE_URL
|
||||
|
||||
super().__init__(
|
||||
api_key=api_key, # type: ignore
|
||||
base_url=base_url, # type: ignore
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
check_interval=check_interval,
|
||||
max_timeout=max_timeout,
|
||||
num_workers=num_workers,
|
||||
show_progress=show_progress,
|
||||
project_id=project_id,
|
||||
organization_id=organization_id,
|
||||
verify=verify,
|
||||
httpx_timeout=httpx_timeout,
|
||||
verbose=verbose,
|
||||
@@ -800,7 +702,7 @@ class LlamaExtract(BaseComponent):
|
||||
config = DEFAULT_EXTRACT_CONFIG
|
||||
|
||||
if isinstance(data_schema, dict):
|
||||
pass
|
||||
data_schema = data_schema
|
||||
elif issubclass(data_schema, BaseModel):
|
||||
data_schema = data_schema.model_json_schema()
|
||||
else:
|
||||
@@ -901,8 +803,6 @@ class LlamaExtract(BaseComponent):
|
||||
num_workers=self.num_workers,
|
||||
show_progress=self.show_progress,
|
||||
verbose=self.verbose,
|
||||
verify=self.verify,
|
||||
httpx_timeout=self.httpx_timeout,
|
||||
)
|
||||
for agent in agents
|
||||
]
|
||||
@@ -919,245 +819,6 @@ class LlamaExtract(BaseComponent):
|
||||
)
|
||||
)
|
||||
|
||||
async def _wait_for_job_result(self, job_id: str) -> Optional[ExtractRun]:
|
||||
"""Wait for and return the results of an extraction job."""
|
||||
return await _wait_for_job_result(
|
||||
client=self._async_client,
|
||||
job_id=job_id,
|
||||
check_interval=self.check_interval,
|
||||
max_timeout=self.max_timeout,
|
||||
verbose=self.verbose,
|
||||
project_id=self._project_id,
|
||||
organization_id=self._organization_id,
|
||||
job_retry_attempts=3,
|
||||
job_max_wait=4,
|
||||
job_jitter=5,
|
||||
run_retry_attempts=3,
|
||||
run_max_wait=4,
|
||||
run_jitter=3,
|
||||
)
|
||||
|
||||
def _get_mime_type(
|
||||
self,
|
||||
filename: Optional[str] = None,
|
||||
file_path: Optional[Union[str, Path]] = None,
|
||||
) -> str:
|
||||
"""Determine MIME type for a file based on filename or path."""
|
||||
# MIME type mappings for supported formats
|
||||
MIME_TYPE_MAP = {
|
||||
# Text files
|
||||
".txt": "text/plain",
|
||||
".csv": "text/csv",
|
||||
".json": "application/json",
|
||||
".html": "text/html",
|
||||
".htm": "text/html",
|
||||
".md": "text/markdown",
|
||||
# Document files
|
||||
".pdf": "application/pdf",
|
||||
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
# Image files
|
||||
".png": "image/png",
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
}
|
||||
|
||||
# Try to get extension from filename or file_path
|
||||
extension = None
|
||||
if filename:
|
||||
extension = Path(filename).suffix.lower()
|
||||
elif file_path:
|
||||
extension = Path(file_path).suffix.lower()
|
||||
|
||||
# Check if the extension is supported
|
||||
if extension and extension in MIME_TYPE_MAP:
|
||||
return MIME_TYPE_MAP[extension]
|
||||
|
||||
# If we don't have a supported extension, provide helpful error message
|
||||
supported_extensions = [ext[1:] for ext in MIME_TYPE_MAP.keys()] # Remove dots
|
||||
supported_list = ", ".join(sorted(supported_extensions))
|
||||
|
||||
if extension:
|
||||
ext_without_dot = extension[1:] # Remove the leading dot
|
||||
raise ValueError(
|
||||
f"Unsupported file type: '{ext_without_dot}'. "
|
||||
f"Supported formats are: {supported_list}"
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Could not determine file type. Please provide a filename with one of these supported extensions: {supported_list}"
|
||||
)
|
||||
|
||||
def _convert_file_to_file_data(self, file_input: FileInput) -> Union[FileData, str]:
|
||||
"""Convert FileInput to FileData or text string for stateless extraction."""
|
||||
if isinstance(file_input, SourceText):
|
||||
if file_input.text_content is not None:
|
||||
return file_input.text_content
|
||||
elif file_input.file is not None:
|
||||
if isinstance(file_input.file, bytes):
|
||||
data = file_input.file
|
||||
filename = file_input.filename
|
||||
elif isinstance(file_input.file, (str, Path)):
|
||||
with open(file_input.file, "rb") as f:
|
||||
data = f.read()
|
||||
filename = file_input.filename or str(file_input.file)
|
||||
elif isinstance(file_input.file, (BufferedIOBase, TextIOWrapper)):
|
||||
if hasattr(file_input.file, "read"):
|
||||
content = file_input.file.read()
|
||||
if isinstance(content, str):
|
||||
data = content.encode("utf-8")
|
||||
else:
|
||||
data = content
|
||||
else:
|
||||
raise ValueError("File object must have a read method")
|
||||
filename = file_input.filename or getattr(
|
||||
file_input.file, "name", None
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported file type: {type(file_input.file)}")
|
||||
|
||||
# Encode as base64
|
||||
encoded_data = base64.b64encode(data).decode("utf-8")
|
||||
|
||||
# Determine mime type
|
||||
mime_type = self._get_mime_type(filename=filename)
|
||||
|
||||
return FileData(data=encoded_data, mime_type=mime_type)
|
||||
else:
|
||||
raise ValueError("SourceText must have either text_content or file")
|
||||
|
||||
elif isinstance(file_input, (str, Path)):
|
||||
with open(file_input, "rb") as f:
|
||||
data = f.read()
|
||||
encoded_data = base64.b64encode(data).decode("utf-8")
|
||||
mime_type = self._get_mime_type(file_path=file_input)
|
||||
return FileData(data=encoded_data, mime_type=mime_type)
|
||||
|
||||
elif isinstance(file_input, bytes):
|
||||
# For raw bytes, we can't determine the file type, so we need to raise an error
|
||||
raise ValueError(
|
||||
"Cannot determine file type from raw bytes. Please use SourceText with a filename, or provide a file path."
|
||||
)
|
||||
|
||||
elif isinstance(file_input, (BufferedIOBase, TextIOWrapper)):
|
||||
if hasattr(file_input, "read"):
|
||||
content = file_input.read()
|
||||
if isinstance(content, str):
|
||||
data = content.encode("utf-8")
|
||||
else:
|
||||
data = content
|
||||
encoded_data = base64.b64encode(data).decode("utf-8")
|
||||
|
||||
# Try to get filename from the file object
|
||||
filename = getattr(file_input, "name", None)
|
||||
mime_type = self._get_mime_type(filename=filename)
|
||||
|
||||
return FileData(data=encoded_data, mime_type=mime_type)
|
||||
else:
|
||||
raise ValueError("File object must have a read method")
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unsupported file input type: {type(file_input)}")
|
||||
|
||||
async def queue_extraction(
|
||||
self,
|
||||
data_schema: SchemaInput,
|
||||
config: ExtractConfig,
|
||||
files: Union[FileInput, List[FileInput]],
|
||||
) -> Union[ExtractJob, List[ExtractJob]]:
|
||||
"""Queue extraction jobs using stateless extraction (no agent required).
|
||||
|
||||
Args:
|
||||
data_schema: The schema defining what data to extract
|
||||
config: The extraction configuration
|
||||
files: File(s) to extract from
|
||||
|
||||
Returns:
|
||||
ExtractJob or list of ExtractJobs
|
||||
"""
|
||||
_extraction_config_warning(config)
|
||||
processed_schema = await _validate_schema(self._async_client, data_schema)
|
||||
|
||||
if not isinstance(files, list):
|
||||
files = [files]
|
||||
|
||||
jobs = []
|
||||
for file_input in files:
|
||||
file_data_or_text = self._convert_file_to_file_data(file_input)
|
||||
|
||||
if isinstance(file_data_or_text, str):
|
||||
# It's text content
|
||||
job = await self._async_client.llama_extract.extract_stateless(
|
||||
project_id=self._project_id,
|
||||
organization_id=self._organization_id,
|
||||
data_schema=processed_schema,
|
||||
config=config,
|
||||
text=file_data_or_text,
|
||||
)
|
||||
else:
|
||||
# It's FileData
|
||||
job = await self._async_client.llama_extract.extract_stateless(
|
||||
project_id=self._project_id,
|
||||
organization_id=self._organization_id,
|
||||
data_schema=processed_schema,
|
||||
config=config,
|
||||
file=file_data_or_text,
|
||||
)
|
||||
jobs.append(job)
|
||||
|
||||
return jobs[0] if len(jobs) == 1 else jobs
|
||||
|
||||
async def aextract(
|
||||
self,
|
||||
data_schema: SchemaInput,
|
||||
config: ExtractConfig,
|
||||
files: Union[FileInput, List[FileInput]],
|
||||
) -> Union[ExtractRun, List[ExtractRun]]:
|
||||
"""Run stateless extraction and wait for results.
|
||||
|
||||
Args:
|
||||
data_schema: The schema defining what data to extract
|
||||
config: The extraction configuration
|
||||
files: File(s) to extract from
|
||||
|
||||
Returns:
|
||||
ExtractRun or list of ExtractRuns with the extraction results
|
||||
"""
|
||||
jobs = await self.queue_extraction(data_schema, config, files)
|
||||
|
||||
if isinstance(jobs, list):
|
||||
runs = []
|
||||
for job in jobs:
|
||||
run = await self._wait_for_job_result(job.id)
|
||||
if run is None:
|
||||
raise RuntimeError(
|
||||
f"Failed to get extraction result for job {job.id}"
|
||||
)
|
||||
runs.append(run)
|
||||
return runs
|
||||
else:
|
||||
run = await self._wait_for_job_result(jobs.id)
|
||||
if run is None:
|
||||
raise RuntimeError(f"Failed to get extraction result for job {jobs.id}")
|
||||
return run
|
||||
|
||||
def extract(
|
||||
self,
|
||||
data_schema: SchemaInput,
|
||||
config: ExtractConfig,
|
||||
files: Union[FileInput, List[FileInput]],
|
||||
) -> Union[ExtractRun, List[ExtractRun]]:
|
||||
"""Run stateless extraction and wait for results (synchronous version).
|
||||
|
||||
Args:
|
||||
data_schema: The schema defining what data to extract
|
||||
config: The extraction configuration
|
||||
files: File(s) to extract from
|
||||
|
||||
Returns:
|
||||
ExtractRun or list of ExtractRuns with the extraction results
|
||||
"""
|
||||
return self._run_in_thread(self.aextract(data_schema, config, files))
|
||||
|
||||
def __del__(self) -> None:
|
||||
"""Cleanup resources properly."""
|
||||
try:
|
||||
@@ -1172,20 +833,6 @@ if __name__ == "__main__":
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# Example usage:
|
||||
#
|
||||
# # Basic usage with stateless extraction (no agent required)
|
||||
# extractor = LlamaExtract()
|
||||
# schema = {"name": {"type": "string"}, "email": {"type": "string"}}
|
||||
# config = ExtractConfig(extraction_mode=ExtractMode.FAST)
|
||||
# files = ["path/to/document.pdf"]
|
||||
#
|
||||
# # Queue extraction jobs
|
||||
# jobs = extractor.queue_extraction(schema, config, files)
|
||||
#
|
||||
# # Or run extraction and wait for results
|
||||
# results = extractor.extract(schema, config, files)
|
||||
|
||||
data_dir = Path(__file__).parent.parent / "tests" / "data"
|
||||
extractor = LlamaExtract()
|
||||
try:
|
||||
|
||||
@@ -25,7 +25,7 @@ from llama_index.core.readers.base import BasePydanticReader
|
||||
from llama_index.core.readers.file.base import get_default_fs
|
||||
from llama_index.core.schema import Document
|
||||
|
||||
from llama_cloud_services.utils import check_extra_params, check_for_updates
|
||||
from llama_cloud_services.utils import check_extra_params
|
||||
from llama_cloud_services.parse.types import JobResult
|
||||
from llama_cloud_services.parse.utils import (
|
||||
SUPPORTED_FILE_TYPES,
|
||||
@@ -541,11 +541,6 @@ class LlamaParse(BasePydanticReader):
|
||||
description="Whether to use the vendor multimodal API.",
|
||||
)
|
||||
|
||||
check_for_updates: Optional[bool] = Field(
|
||||
default=False,
|
||||
description="Automatically check for Python SDK updates.",
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def warn_extra_params(cls, data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
@@ -601,16 +596,6 @@ class LlamaParse(BasePydanticReader):
|
||||
|
||||
return self._aclient
|
||||
|
||||
_update_checked = False
|
||||
|
||||
async def _check_for_updates(self) -> None:
|
||||
if self.check_for_updates and not self._update_checked:
|
||||
try:
|
||||
await check_for_updates(self.aclient, quiet=False)
|
||||
self._update_checked = True
|
||||
except (ValueError, httpx.HTTPStatusError):
|
||||
pass
|
||||
|
||||
@asynccontextmanager
|
||||
async def client_context(self) -> AsyncGenerator[httpx.AsyncClient, None]:
|
||||
"""Create a context for the HTTPX client."""
|
||||
@@ -660,7 +645,6 @@ class LlamaParse(BasePydanticReader):
|
||||
fs: Optional[AbstractFileSystem] = None,
|
||||
partition_target_pages: Optional[str] = None,
|
||||
) -> str:
|
||||
await self._check_for_updates()
|
||||
files = None
|
||||
file_handle = None
|
||||
input_url = file_input if self._is_input_url(file_input) else None
|
||||
@@ -1550,7 +1534,6 @@ class LlamaParse(BasePydanticReader):
|
||||
self, json_result: List[dict], download_path: str, asset_key: str
|
||||
) -> List[dict]:
|
||||
"""Download assets (images or charts) from the parsed result."""
|
||||
await self._check_for_updates()
|
||||
# Make the download path
|
||||
if not os.path.exists(download_path):
|
||||
os.makedirs(download_path)
|
||||
@@ -1659,7 +1642,6 @@ class LlamaParse(BasePydanticReader):
|
||||
self, json_result: List[dict], download_path: str
|
||||
) -> List[dict]:
|
||||
"""Download xlsx from the parsed result."""
|
||||
await self._check_for_updates()
|
||||
# make the download path
|
||||
if not os.path.exists(download_path):
|
||||
os.makedirs(download_path)
|
||||
|
||||
@@ -1,9 +1,4 @@
|
||||
import os
|
||||
import importlib.metadata
|
||||
|
||||
import difflib
|
||||
import httpx
|
||||
import packaging.version
|
||||
from pydantic import BaseModel
|
||||
from typing import Any, Dict, List, Tuple, Type
|
||||
|
||||
@@ -32,36 +27,3 @@ def check_extra_params(
|
||||
)
|
||||
|
||||
return extra_params, suggestions
|
||||
|
||||
|
||||
async def check_for_updates(client: httpx.AsyncClient, quiet: bool = True) -> bool:
|
||||
"""Check if an SDK update is available.
|
||||
|
||||
Args:
|
||||
client: HTTPX client to use.
|
||||
quiet: If False, update availability will also be printed to stdout.
|
||||
|
||||
Returns: True if an update is available.
|
||||
|
||||
Raises:
|
||||
ValueError: Failed to get a valid release version from PyPI.
|
||||
"""
|
||||
package_name = "llama-cloud-services"
|
||||
r = await client.get(f"https://pypi.org/pypi/{package_name}/json")
|
||||
version = r.json().get("info", {}).get("version", "")
|
||||
if not version:
|
||||
raise ValueError("Failed to fetch package info from PyPI")
|
||||
latest = packaging.version.parse(version)
|
||||
current = packaging.version.parse(importlib.metadata.version(package_name))
|
||||
if current < latest:
|
||||
if not quiet:
|
||||
msg = [
|
||||
f"\u26A0\uFE0F {package_name} is out of date",
|
||||
f"Current version: {current} | Latest: {latest}",
|
||||
"To upgrade: pip install -U --force-reinstall llama-cloud-services",
|
||||
]
|
||||
print(os.linesep.join(msg))
|
||||
return True
|
||||
elif not quiet:
|
||||
print(f"{package_name} is up to date")
|
||||
return False
|
||||
|
||||
@@ -11,13 +11,13 @@ dev = [
|
||||
|
||||
[project]
|
||||
name = "llama-parse"
|
||||
version = "0.6.55"
|
||||
version = "0.6.54"
|
||||
description = "Parse files into RAG-Optimized formats."
|
||||
authors = [{name = "Logan Markewich", email = "logan@llamaindex.ai"}]
|
||||
requires-python = ">=3.9,<4.0"
|
||||
readme = "README.md"
|
||||
license = "MIT"
|
||||
dependencies = ["llama-cloud-services>=0.6.55"]
|
||||
dependencies = ["llama-cloud-services>=0.6.54"]
|
||||
|
||||
[project.scripts]
|
||||
llama-parse = "llama_parse.cli.main:parse"
|
||||
|
||||
+3
-4
@@ -17,7 +17,7 @@ dev = [
|
||||
|
||||
[project]
|
||||
name = "llama-cloud-services"
|
||||
version = "0.6.55"
|
||||
version = "0.6.54"
|
||||
description = "Tailored SDK clients for LlamaCloud services."
|
||||
authors = [{name = "Logan Markewich", email = "logan@runllama.ai"}]
|
||||
requires-python = ">=3.9,<4.0"
|
||||
@@ -25,14 +25,13 @@ readme = "README.md"
|
||||
license = "MIT"
|
||||
dependencies = [
|
||||
"llama-index-core>=0.12.0",
|
||||
"llama-cloud==0.1.37",
|
||||
"llama-cloud==0.1.35",
|
||||
"pydantic>=2.8,!=2.10",
|
||||
"click>=8.1.7,<9",
|
||||
"python-dotenv>=1.0.1,<2",
|
||||
"eval-type-backport>=0.2.0,<0.3 ; python_version < '3.10'",
|
||||
"platformdirs>=4.3.7,<5",
|
||||
"tenacity>=8.5.0, <10.0",
|
||||
"packaging>=25.0"
|
||||
"tenacity>=8.5.0, <10.0"
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
|
||||
@@ -37,7 +37,7 @@ LLAMA_CLOUD_BASE_URL = os.getenv("LLAMA_CLOUD_BASE_URL")
|
||||
LLAMA_DEPLOY_DEPLOYMENT_NAME = os.getenv("LLAMA_DEPLOY_DEPLOYMENT_NAME")
|
||||
|
||||
|
||||
class ExampleData(BaseModel):
|
||||
class TestData(BaseModel):
|
||||
"""Simple test data model for agent data testing"""
|
||||
|
||||
name: str
|
||||
@@ -66,13 +66,13 @@ async def test_agent_data_crud_operations():
|
||||
# Create agent data client with unique collection name
|
||||
agent_data_client = AsyncAgentDataClient(
|
||||
client=client,
|
||||
type=ExampleData,
|
||||
type=TestData,
|
||||
collection=f"test-collection-{test_id[:8]}",
|
||||
agent_url_id=LLAMA_DEPLOY_DEPLOYMENT_NAME,
|
||||
)
|
||||
|
||||
# Create test data
|
||||
test_data = ExampleData(name="test-item", test_id=test_id, value=42)
|
||||
test_data = TestData(name="test-item", test_id=test_id, value=42)
|
||||
|
||||
created_item = None
|
||||
try:
|
||||
@@ -107,7 +107,7 @@ async def test_agent_data_crud_operations():
|
||||
assert aggregate_results.items[0].count == 1
|
||||
|
||||
# Test UPDATE
|
||||
updated_data = ExampleData(name="updated-item", test_id=test_id, value=84)
|
||||
updated_data = TestData(name="updated-item", test_id=test_id, value=84)
|
||||
updated_item = await agent_data_client.update_item(
|
||||
created_item.id, updated_data
|
||||
)
|
||||
|
||||
@@ -6,12 +6,6 @@ from llama_cloud_services.extract import LlamaExtract
|
||||
_TEST_AGENTS_TO_CLEANUP: List[str] = []
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
"""Register custom markers for extract tests."""
|
||||
config.addinivalue_line("markers", "agent_name: custom agent name for test")
|
||||
config.addinivalue_line("markers", "agent_schema: custom agent schema for test")
|
||||
|
||||
|
||||
def pytest_sessionfinish(session, exitstatus):
|
||||
"""Hook that runs after all tests complete - cleanup agents here"""
|
||||
print(
|
||||
|
||||
@@ -23,9 +23,8 @@ LLAMA_CLOUD_API_KEY = os.getenv("LLAMA_CLOUD_API_KEY")
|
||||
LLAMA_CLOUD_BASE_URL = os.getenv("LLAMA_CLOUD_BASE_URL")
|
||||
LLAMA_CLOUD_PROJECT_ID = os.getenv("LLAMA_CLOUD_PROJECT_ID")
|
||||
|
||||
BenchmarkTestCase = namedtuple(
|
||||
"BenchmarkTestCase",
|
||||
["name", "schema_path", "config", "input_file", "expected_output"],
|
||||
TestCase = namedtuple(
|
||||
"TestCase", ["name", "schema_path", "config", "input_file", "expected_output"]
|
||||
)
|
||||
|
||||
|
||||
@@ -33,7 +32,7 @@ def get_test_cases():
|
||||
"""Get all test cases from TEST_DIR.
|
||||
|
||||
Returns:
|
||||
List[BenchmarkTestCase]: List of test cases
|
||||
List[TestCase]: List of test cases
|
||||
"""
|
||||
test_cases = []
|
||||
|
||||
@@ -74,7 +73,7 @@ def get_test_cases():
|
||||
test_name = f"{data_type}/{os.path.basename(input_file)}"
|
||||
for setting in settings:
|
||||
test_cases.append(
|
||||
BenchmarkTestCase(
|
||||
TestCase(
|
||||
name=test_name,
|
||||
schema_path=schema_path,
|
||||
input_file=input_file,
|
||||
@@ -101,7 +100,7 @@ def extractor():
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def extraction_agent(test_case: BenchmarkTestCase, extractor: LlamaExtract):
|
||||
def extraction_agent(test_case: TestCase, extractor: LlamaExtract):
|
||||
"""Fixture to create and cleanup extraction agent for each test."""
|
||||
# Create unique name with random UUID (important for CI to avoid conflicts)
|
||||
unique_id = uuid.uuid4().hex[:8]
|
||||
@@ -131,7 +130,7 @@ def extraction_agent(test_case: BenchmarkTestCase, extractor: LlamaExtract):
|
||||
@pytest.mark.parametrize("test_case", get_test_cases(), ids=lambda x: x.name)
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_extraction(
|
||||
test_case: BenchmarkTestCase, extraction_agent: ExtractionAgent
|
||||
test_case: TestCase, extraction_agent: ExtractionAgent
|
||||
) -> None:
|
||||
start = perf_counter()
|
||||
result = await extraction_agent._run_extraction_test(
|
||||
|
||||
@@ -4,7 +4,6 @@ from pathlib import Path
|
||||
from pydantic import BaseModel
|
||||
|
||||
from llama_cloud_services.extract import LlamaExtract, ExtractionAgent, SourceText
|
||||
from llama_cloud.types import ExtractConfig, ExtractMode, ExtractRun
|
||||
from tests.extract.util import load_test_dotenv
|
||||
from .conftest import register_agent_for_cleanup
|
||||
|
||||
@@ -22,7 +21,7 @@ pytestmark = pytest.mark.skipif(
|
||||
|
||||
|
||||
# Test data
|
||||
class ExampleSchema(BaseModel):
|
||||
class TestSchema(BaseModel):
|
||||
title: str
|
||||
summary: str
|
||||
|
||||
@@ -108,7 +107,7 @@ class TestLlamaExtract:
|
||||
assert isinstance(test_agent, ExtractionAgent)
|
||||
|
||||
@pytest.mark.agent_name("test-pydantic-schema-agent")
|
||||
@pytest.mark.agent_schema((ExampleSchema,))
|
||||
@pytest.mark.agent_schema((TestSchema,))
|
||||
def test_create_agent_with_pydantic_schema(self, test_agent):
|
||||
assert isinstance(test_agent, ExtractionAgent)
|
||||
|
||||
@@ -223,189 +222,7 @@ class TestExtractionAgent:
|
||||
|
||||
def test_delete_extraction_run(self, test_agent: ExtractionAgent):
|
||||
assert test_agent.list_extraction_runs().total == 0
|
||||
run: ExtractRun = test_agent.extract(TEST_PDF)
|
||||
run = test_agent.extract(TEST_PDF)
|
||||
test_agent.delete_extraction_run(run.id)
|
||||
runs = test_agent.list_extraction_runs()
|
||||
assert runs.total == 0
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
"CI" in os.environ, reason="Test locally; functionality is mostly duplicated."
|
||||
)
|
||||
class TestStatelessExtraction:
|
||||
"""Tests for stateless extraction methods that don't require creating an agent."""
|
||||
|
||||
@pytest.fixture
|
||||
def test_config(self):
|
||||
return ExtractConfig(extraction_mode=ExtractMode.FAST)
|
||||
|
||||
@pytest.fixture
|
||||
def test_schema_dict(self):
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string"},
|
||||
"summary": {"type": "string"},
|
||||
},
|
||||
}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aextract_single_file(
|
||||
self, llama_extract, test_schema_dict, test_config
|
||||
):
|
||||
"""Test async stateless extraction with a single file."""
|
||||
result = await llama_extract.aextract(test_schema_dict, test_config, TEST_PDF)
|
||||
assert result.status == "SUCCESS"
|
||||
assert result.data is not None
|
||||
assert isinstance(result.data, dict)
|
||||
assert "title" in result.data
|
||||
assert "summary" in result.data
|
||||
|
||||
def test_extract_single_file(self, llama_extract, test_schema_dict, test_config):
|
||||
"""Test synchronous stateless extraction with a single file."""
|
||||
result = llama_extract.extract(test_schema_dict, test_config, TEST_PDF)
|
||||
assert result.status == "SUCCESS"
|
||||
assert result.data is not None
|
||||
assert isinstance(result.data, dict)
|
||||
assert "title" in result.data
|
||||
assert "summary" in result.data
|
||||
|
||||
def test_extract_from_bytes_with_source_text(
|
||||
self, llama_extract, test_schema_dict, test_config
|
||||
):
|
||||
"""Test stateless extraction from bytes using SourceText with filename."""
|
||||
with open(TEST_PDF, "rb") as f:
|
||||
file_bytes = f.read()
|
||||
source_text = SourceText(file=file_bytes, filename=TEST_PDF.name)
|
||||
result = llama_extract.extract(test_schema_dict, test_config, source_text)
|
||||
assert result.status == "SUCCESS"
|
||||
assert result.data is not None
|
||||
assert isinstance(result.data, dict)
|
||||
assert "title" in result.data
|
||||
assert "summary" in result.data
|
||||
|
||||
def test_extract_from_source_text_with_file(
|
||||
self, llama_extract, test_schema_dict, test_config
|
||||
):
|
||||
"""Test stateless extraction from SourceText with file."""
|
||||
source_text = SourceText(file=TEST_PDF, filename=TEST_PDF.name)
|
||||
result = llama_extract.extract(test_schema_dict, test_config, source_text)
|
||||
assert result.status == "SUCCESS"
|
||||
assert result.data is not None
|
||||
assert isinstance(result.data, dict)
|
||||
assert "title" in result.data
|
||||
assert "summary" in result.data
|
||||
|
||||
def test_extract_from_buffered_io(
|
||||
self, llama_extract, test_schema_dict, test_config
|
||||
):
|
||||
"""Test stateless extraction from BufferedIO file handle."""
|
||||
with open(TEST_PDF, "rb") as file_handle:
|
||||
result = llama_extract.extract(test_schema_dict, test_config, file_handle)
|
||||
assert result.status == "SUCCESS"
|
||||
assert result.data is not None
|
||||
assert isinstance(result.data, dict)
|
||||
assert "title" in result.data
|
||||
assert "summary" in result.data
|
||||
|
||||
def test_extract_from_source_text_with_text_content(
|
||||
self, llama_extract, test_schema_dict, test_config
|
||||
):
|
||||
"""Test stateless extraction from SourceText with text content."""
|
||||
TEST_TEXT = """
|
||||
# Llamas
|
||||
Llamas are social animals and live with others as a herd. Their wool is soft and
|
||||
contains only a small amount of lanolin. Llamas can learn simple tasks after a
|
||||
few repetitions. When using a pack, they can carry about 25 to 30% of their body
|
||||
weight for 8 to 13 km (5–8 miles). The name llama was adopted by European settlers
|
||||
from native Peruvians.
|
||||
"""
|
||||
source_text = SourceText(text_content=TEST_TEXT)
|
||||
result = llama_extract.extract(test_schema_dict, test_config, source_text)
|
||||
assert result.status == "SUCCESS"
|
||||
assert result.data is not None
|
||||
assert isinstance(result.data, dict)
|
||||
assert "title" in result.data
|
||||
assert "summary" in result.data
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_queue_extraction_single_file(
|
||||
self, llama_extract, test_schema_dict, test_config
|
||||
):
|
||||
"""Test queuing extraction job without waiting for completion."""
|
||||
job = await llama_extract.queue_extraction(
|
||||
test_schema_dict, test_config, TEST_PDF
|
||||
)
|
||||
assert hasattr(job, "id")
|
||||
assert hasattr(job, "status")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_multiple_files(
|
||||
self, llama_extract, test_schema_dict, test_config
|
||||
):
|
||||
"""Test stateless extraction with multiple files."""
|
||||
files = [TEST_PDF, TEST_PDF] # Using same file twice for testing
|
||||
results = await llama_extract.aextract(test_schema_dict, test_config, files)
|
||||
|
||||
assert isinstance(results, list)
|
||||
assert len(results) == 2
|
||||
|
||||
for result in results:
|
||||
assert result.status == "SUCCESS"
|
||||
assert result.data is not None
|
||||
assert isinstance(result.data, dict)
|
||||
assert "title" in result.data
|
||||
assert "summary" in result.data
|
||||
|
||||
def test_extract_with_pydantic_schema(self, llama_extract, test_config):
|
||||
"""Test stateless extraction with Pydantic schema."""
|
||||
result = llama_extract.extract(ExampleSchema, test_config, TEST_PDF)
|
||||
assert result.status == "SUCCESS"
|
||||
assert result.data is not None
|
||||
assert isinstance(result.data, dict)
|
||||
assert "title" in result.data
|
||||
assert "summary" in result.data
|
||||
|
||||
def test_extract_from_raw_bytes_raises_error(
|
||||
self, llama_extract, test_schema_dict, test_config
|
||||
):
|
||||
"""Test that raw bytes without filename raises an error."""
|
||||
with open(TEST_PDF, "rb") as f:
|
||||
file_bytes = f.read()
|
||||
|
||||
with pytest.raises(
|
||||
ValueError, match="Cannot determine file type from raw bytes"
|
||||
):
|
||||
llama_extract.extract(test_schema_dict, test_config, file_bytes)
|
||||
|
||||
def test_mime_type_detection(self, llama_extract):
|
||||
"""Test that MIME types are correctly detected for various file types."""
|
||||
# Test PDF
|
||||
assert llama_extract._get_mime_type(filename="test.pdf") == "application/pdf"
|
||||
|
||||
# Test DOCX
|
||||
assert (
|
||||
llama_extract._get_mime_type(filename="test.docx")
|
||||
== "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
|
||||
)
|
||||
|
||||
# Test text files
|
||||
assert llama_extract._get_mime_type(filename="test.txt") == "text/plain"
|
||||
assert llama_extract._get_mime_type(filename="test.csv") == "text/csv"
|
||||
assert llama_extract._get_mime_type(filename="test.json") == "application/json"
|
||||
|
||||
# Test image files
|
||||
assert llama_extract._get_mime_type(filename="test.png") == "image/png"
|
||||
assert llama_extract._get_mime_type(filename="test.jpg") == "image/jpeg"
|
||||
|
||||
# Test file path
|
||||
from pathlib import Path
|
||||
|
||||
assert (
|
||||
llama_extract._get_mime_type(file_path=Path("test.pdf"))
|
||||
== "application/pdf"
|
||||
)
|
||||
|
||||
# Test unsupported file type
|
||||
with pytest.raises(ValueError, match="Unsupported file type: 'xyz'"):
|
||||
llama_extract._get_mime_type(filename="test.xyz")
|
||||
|
||||
@@ -19,9 +19,8 @@ LLAMA_CLOUD_API_KEY = os.getenv("LLAMA_CLOUD_API_KEY")
|
||||
LLAMA_CLOUD_BASE_URL = os.getenv("LLAMA_CLOUD_BASE_URL")
|
||||
LLAMA_CLOUD_PROJECT_ID = os.getenv("LLAMA_CLOUD_PROJECT_ID")
|
||||
|
||||
ExtractionTestCase = namedtuple(
|
||||
"ExtractionTestCase",
|
||||
["name", "schema_path", "config", "input_file", "expected_output"],
|
||||
TestCase = namedtuple(
|
||||
"TestCase", ["name", "schema_path", "config", "input_file", "expected_output"]
|
||||
)
|
||||
|
||||
|
||||
@@ -29,7 +28,7 @@ def get_test_cases():
|
||||
"""Get all test cases from TEST_DIR.
|
||||
|
||||
Returns:
|
||||
List[ExtractionTestCase]: List of test cases
|
||||
List[TestCase]: List of test cases
|
||||
"""
|
||||
test_cases = []
|
||||
|
||||
@@ -70,7 +69,7 @@ def get_test_cases():
|
||||
test_name = f"{data_type}/{os.path.basename(input_file)}"
|
||||
for setting in settings:
|
||||
test_cases.append(
|
||||
ExtractionTestCase(
|
||||
TestCase(
|
||||
name=test_name,
|
||||
schema_path=schema_path,
|
||||
input_file=input_file,
|
||||
@@ -97,7 +96,7 @@ def extractor():
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def extraction_agent(test_case: ExtractionTestCase, extractor: LlamaExtract):
|
||||
def extraction_agent(test_case: TestCase, extractor: LlamaExtract):
|
||||
"""Fixture to create and cleanup extraction agent for each test."""
|
||||
# Create unique name with random UUID (important for CI to avoid conflicts)
|
||||
unique_id = uuid.uuid4().hex[:8]
|
||||
@@ -129,9 +128,7 @@ def extraction_agent(test_case: ExtractionTestCase, extractor: LlamaExtract):
|
||||
reason="LLAMA_CLOUD_API_KEY not set",
|
||||
)
|
||||
@pytest.mark.parametrize("test_case", get_test_cases(), ids=lambda x: x.name)
|
||||
def test_extraction(
|
||||
test_case: ExtractionTestCase, extraction_agent: ExtractionAgent
|
||||
) -> None:
|
||||
def test_extraction(test_case: TestCase, extraction_agent: ExtractionAgent) -> None:
|
||||
result = extraction_agent.extract(test_case.input_file).data # type: ignore
|
||||
with open(test_case.expected_output, "r") as f:
|
||||
expected = json.load(f)
|
||||
|
||||
@@ -1,9 +1,5 @@
|
||||
from unittest.mock import Mock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from pydantic import BaseModel
|
||||
from llama_cloud_services.utils import check_extra_params, check_for_updates
|
||||
from llama_cloud_services.utils import check_extra_params
|
||||
|
||||
|
||||
class MyModel(BaseModel):
|
||||
@@ -68,30 +64,3 @@ def test_check_extra_params_completely_invalid():
|
||||
for suggestion in suggestions:
|
||||
assert "check the documentation or update the package" in suggestion
|
||||
assert "Did you mean" not in suggestion
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_for_updates(capsys: pytest.CaptureFixture):
|
||||
"""Test update checker."""
|
||||
|
||||
mock_response = Mock()
|
||||
mock_client = Mock(spec=httpx.AsyncClient)
|
||||
mock_client.get.return_value = mock_response
|
||||
|
||||
mock_response.json.return_value = {"info": {"version": "0.0.0"}}
|
||||
assert not await check_for_updates(mock_client)
|
||||
out, err = capsys.readouterr()
|
||||
assert not out and not err
|
||||
|
||||
assert not await check_for_updates(mock_client, quiet=False)
|
||||
out, _ = capsys.readouterr()
|
||||
assert "up to date" in out
|
||||
|
||||
mock_response.json.return_value = {"info": {"version": "999.0.0"}}
|
||||
assert await check_for_updates(mock_client)
|
||||
out, err = capsys.readouterr()
|
||||
assert not out and not err
|
||||
|
||||
assert await check_for_updates(mock_client, quiet=False)
|
||||
out, _ = capsys.readouterr()
|
||||
assert "out of date" in out
|
||||
|
||||
Generated
+4
-6
@@ -1573,16 +1573,16 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "llama-cloud"
|
||||
version = "0.1.37"
|
||||
version = "0.1.35"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "certifi" },
|
||||
{ name = "httpx" },
|
||||
{ name = "pydantic" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/9c/dd/c4f2516523778a0d4b284fa4b66a0136afdd59694f105102838cf793e675/llama_cloud-0.1.37.tar.gz", hash = "sha256:b6d62e7386d1aa85905b7e3f7c19a40694be54c1596668bceaa456cd84ede666", size = 108707, upload-time = "2025-08-04T22:00:37.18Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/9b/72/816e6e900448e1b4a8137d90e65876b296c5264a23db6ae888bd3e6660ba/llama_cloud-0.1.35.tar.gz", hash = "sha256:200349d5d57424d7461f304cdb1355a58eea3e6ca1e6b0d75c66b2e937216983", size = 106403, upload-time = "2025-07-28T17:22:06.41Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/59/ca/a7f874b041d2566f000ecc88bf1b76819a8bdbbaea85036ca6905ae3f0f7/llama_cloud-0.1.37-py3-none-any.whl", hash = "sha256:3109ec74575f53311ec4957ca8aea2d6e30556a43d24c6316ceab91c4fed6ab7", size = 314244, upload-time = "2025-08-04T22:00:35.696Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1d/d2/8d18a021ab757cea231428404f21fe3186bf1ebaac3f57a73c379483fd3f/llama_cloud-0.1.35-py3-none-any.whl", hash = "sha256:b7abab4423118e6f638d2f326749e7a07c6426543bea6da99b623c715b22af71", size = 303280, upload-time = "2025-07-28T17:22:04.946Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1595,7 +1595,6 @@ dependencies = [
|
||||
{ name = "eval-type-backport", marker = "python_full_version < '3.10'" },
|
||||
{ name = "llama-cloud" },
|
||||
{ name = "llama-index-core" },
|
||||
{ name = "packaging" },
|
||||
{ name = "platformdirs" },
|
||||
{ name = "pydantic" },
|
||||
{ name = "python-dotenv" },
|
||||
@@ -1620,9 +1619,8 @@ dev = [
|
||||
requires-dist = [
|
||||
{ name = "click", specifier = ">=8.1.7,<9" },
|
||||
{ name = "eval-type-backport", marker = "python_full_version < '3.10'", specifier = ">=0.2.0,<0.3" },
|
||||
{ name = "llama-cloud", specifier = "==0.1.37" },
|
||||
{ name = "llama-cloud", specifier = "==0.1.35" },
|
||||
{ name = "llama-index-core", specifier = ">=0.12.0" },
|
||||
{ name = "packaging", specifier = ">=25.0" },
|
||||
{ name = "platformdirs", specifier = ">=4.3.7,<5" },
|
||||
{ name = "pydantic", specifier = ">=2.8,!=2.10" },
|
||||
{ name = "python-dotenv", specifier = ">=1.0.1,<2" },
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
{
|
||||
"name": "@llamaindex/llama-cloud-services-e2e-tests",
|
||||
"private": true,
|
||||
"version": "0.0.1",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "pnpm run test:node16 && pnpm run test:nodenext && pnpm run test:bundler",
|
||||
"build": "pnpm run build:node16 && pnpm run build:nodenext && pnpm run build:bundler",
|
||||
"build:node16": "tsc -p src/tsconfig.node16.json --outDir dist/node16",
|
||||
"test:node16": "pnpm run build:node16 && node --test dist/node16/index.e2e.js",
|
||||
"build:nodenext": "tsc -p src/tsconfig.nodenext.json --outDir dist/nodenext",
|
||||
"test:nodenext": "pnpm run build:nodenext && node --test dist/nodenext/index.e2e.js",
|
||||
"build:bundler": "tsc -p src/tsconfig.bundler.json --outDir dist/bundler",
|
||||
"test:bundler": "pnpm run build:bundler && node --test dist/bundler/index.e2e.js",
|
||||
"build:node": "tsc -p src/tsconfig.node.json --outDir dist/node",
|
||||
"test:node": "pnpm run build:node && node --test dist/node/index.e2e.js"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.0.13",
|
||||
"llama-cloud-services": "workspace:*",
|
||||
"typescript": "^5.9.2"
|
||||
}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
import { ok } from "node:assert";
|
||||
import { test } from "node:test";
|
||||
|
||||
import { LlamaCloudIndex } from "llama-cloud-services";
|
||||
import { LlamaParseReader } from "llama-cloud-services";
|
||||
|
||||
test("LlamaIndex module resolution test", async (t) => {
|
||||
await t.test("works with ES module", () => {
|
||||
const index = new LlamaCloudIndex({
|
||||
name: "test-index",
|
||||
projectName: "Default",
|
||||
});
|
||||
const reader = new LlamaParseReader({
|
||||
resultType: "markdown",
|
||||
verbose: false,
|
||||
});
|
||||
ok(index !== undefined);
|
||||
ok(reader !== undefined);
|
||||
});
|
||||
|
||||
await t.test("works with dynamic imports", async () => {
|
||||
const mod = await import("llama-cloud-services"); // simulates commonjs
|
||||
ok(mod !== undefined);
|
||||
const index = new mod.LlamaCloudIndex({
|
||||
name: "test-index",
|
||||
projectName: "Default",
|
||||
});
|
||||
ok(index !== undefined);
|
||||
});
|
||||
|
||||
await t.test("all imports work", () => {
|
||||
const allImports = [
|
||||
LlamaCloudIndex,
|
||||
];
|
||||
|
||||
ok(allImports.filter(Boolean).length === allImports.length);
|
||||
});
|
||||
});
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "esnext",
|
||||
"moduleResolution": "bundler",
|
||||
"target": "esnext",
|
||||
"outDir": "dist",
|
||||
"rootDir": ".",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": ["**/*.ts"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "esnext",
|
||||
"moduleResolution": "node",
|
||||
"target": "esnext",
|
||||
"outDir": "dist",
|
||||
"rootDir": ".",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": ["**/*.ts"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "node16",
|
||||
"moduleResolution": "node16",
|
||||
"target": "esnext",
|
||||
"outDir": "dist",
|
||||
"rootDir": ".",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": ["**/*.ts"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "nodenext",
|
||||
"moduleResolution": "nodenext",
|
||||
"target": "esnext",
|
||||
"outDir": "dist",
|
||||
"rootDir": ".",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": ["**/*.ts"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"verbatimModuleSyntax": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"strictNullChecks": true,
|
||||
"exactOptionalPropertyTypes": true,
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist/type",
|
||||
"tsBuildInfoFile": "./dist/.tsbuildinfo",
|
||||
"incremental": true,
|
||||
"lib": [
|
||||
"ES2022",
|
||||
"DOM",
|
||||
"DOM.Iterable",
|
||||
"DOM.AsyncIterable"
|
||||
],
|
||||
"types": [
|
||||
"node"
|
||||
]
|
||||
},
|
||||
"include": [
|
||||
"./src"
|
||||
],
|
||||
"exclude": [
|
||||
"node_modules"
|
||||
]
|
||||
}
|
||||
Generated
+3562
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user