mirror of
https://github.com/run-llama/llama_cloud_services.git
synced 2026-07-21 03:55:22 -04:00
Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 51a7534733 | |||
| 4f5d2bde13 | |||
| 3d05fe5d77 | |||
| c16ca673af | |||
| 6619034bce | |||
| c56fb5d8f7 | |||
| b407a5edb5 |
+93
-21
@@ -2,6 +2,29 @@
|
||||
|
||||
LlamaExtract provides a simple API for extracting structured data from unstructured documents like PDFs, text files and images.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Quick Start](#quick-start)
|
||||
- [Supported File Types](#supported-file-types)
|
||||
- [Different Input Types](#different-input-types)
|
||||
- [Async Extraction](#async-extraction)
|
||||
- [Core Concepts](#core-concepts)
|
||||
- [Defining Schemas](#defining-schemas)
|
||||
- [Using Pydantic (Recommended)](#using-pydantic-recommended)
|
||||
- [Using JSON Schema](#using-json-schema)
|
||||
- [Important restrictions on JSON/Pydantic Schema](#important-restrictions-on-jsonpydantic-schema)
|
||||
- [Extraction Configuration](#extraction-configuration)
|
||||
- [Configuration Options](#configuration-options)
|
||||
- [Extraction Agents (Advanced)](#extraction-agents-advanced)
|
||||
- [Creating Agents](#creating-agents)
|
||||
- [Agent Batch Processing](#agent-batch-processing)
|
||||
- [Updating Agent Schemas](#updating-agent-schemas)
|
||||
- [Managing Agents](#managing-agents)
|
||||
- [When to Use Agents vs Direct Extraction](#when-to-use-agents-vs-direct-extraction)
|
||||
- [Installation](#installation)
|
||||
- [Tips & Best Practices](#tips--best-practices)
|
||||
- [Additional Resources](#additional-resources)
|
||||
|
||||
## Quick Start
|
||||
|
||||
The simplest way to get started is to use the stateless API with the extraction configuration and the file/text to extract from:
|
||||
@@ -12,7 +35,7 @@ from llama_cloud import ExtractConfig, ExtractMode
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# Initialize client
|
||||
extractor = LlamaExtract()
|
||||
extractor = LlamaExtract(api_key="YOUR_API_KEY")
|
||||
|
||||
|
||||
# Define schema using Pydantic
|
||||
@@ -64,7 +87,9 @@ result = extractor.extract(Resume, config, SourceText(text_content=text))
|
||||
|
||||
### Async Extraction
|
||||
|
||||
For better performance with multiple files or when integrating with async applications:
|
||||
For better performance with multiple files or when integrating with async applications.
|
||||
Here `queue_extraction` will enqueue the extraction jobs and exit. Alternatively, you
|
||||
can use `aextract` to poll for the job and return the extraction results.
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
@@ -80,10 +105,18 @@ async def extract_resumes():
|
||||
Resume, config, ["resume1.pdf", "resume2.pdf"]
|
||||
)
|
||||
print(f"Queued {len(jobs)} extraction jobs")
|
||||
return jobs
|
||||
|
||||
|
||||
# Run async function
|
||||
asyncio.run(extract_resumes())
|
||||
jobs = asyncio.run(extract_resumes())
|
||||
# Check job status
|
||||
for job in jobs:
|
||||
status = agent.get_extraction_job(job.id).status
|
||||
print(f"Job {job.id}: {status}")
|
||||
|
||||
# Get results when complete
|
||||
results = [agent.get_extraction_run_for_job(job.id) for job in jobs]
|
||||
```
|
||||
|
||||
## Core Concepts
|
||||
@@ -159,24 +192,6 @@ config = ExtractConfig(extraction_mode=ExtractMode.FAST)
|
||||
result = extractor.extract(schema, config, "resume.pdf")
|
||||
```
|
||||
|
||||
## Extraction Configuration
|
||||
|
||||
Configure how extraction is performed using `ExtractConfig`:
|
||||
|
||||
```python
|
||||
from llama_cloud import ExtractConfig, ExtractMode
|
||||
|
||||
# Fast extraction (lower accuracy, faster processing)
|
||||
fast_config = ExtractConfig(extraction_mode=ExtractMode.FAST)
|
||||
|
||||
# Balanced extraction (good balance of speed and accuracy)
|
||||
balanced_config = ExtractConfig(extraction_mode=ExtractMode.BALANCED)
|
||||
|
||||
# Use different configs for different needs
|
||||
result = extractor.extract(schema, fast_config, "simple_document.pdf")
|
||||
result = extractor.extract(schema, balanced_config, "complex_document.pdf")
|
||||
```
|
||||
|
||||
### Important restrictions on JSON/Pydantic Schema
|
||||
|
||||
_LlamaExtract only supports a subset of the JSON Schema specification._ While limited, it should
|
||||
@@ -194,6 +209,62 @@ be sufficient for a wide variety of use-cases.
|
||||
your extraction workflow to fit within these constraints, e.g. by extracting subset of fields
|
||||
and later merging them together.
|
||||
|
||||
## Extraction Configuration
|
||||
|
||||
Configure how extraction is performed using `ExtractConfig`. The schema is the most important part, but several configuration options can significantly impact the extraction process.
|
||||
|
||||
```python
|
||||
from llama_cloud import ExtractConfig, ExtractMode, ChunkMode, ExtractTarget
|
||||
|
||||
# Basic configuration
|
||||
config = ExtractConfig(
|
||||
extraction_mode=ExtractMode.BALANCED, # FAST, BALANCED, MULTIMODAL, PREMIUM
|
||||
extraction_target=ExtractTarget.PER_DOC, # PER_DOC, PER_PAGE
|
||||
system_prompt="Focus on the most recent data",
|
||||
page_range="1-5,10-15", # Extract from specific pages
|
||||
)
|
||||
|
||||
# Advanced configuration
|
||||
advanced_config = ExtractConfig(
|
||||
extraction_mode=ExtractMode.MULTIMODAL,
|
||||
chunk_mode=ChunkMode.PAGE, # PAGE, SECTION
|
||||
high_resolution_mode=True, # Better OCR accuracy
|
||||
invalidate_cache=False, # Bypass cached results
|
||||
cite_sources=True, # Enable source citations
|
||||
use_reasoning=True, # Enable reasoning (not in FAST mode)
|
||||
confidence_scores=True, # MULTIMODAL/PREMIUM only
|
||||
)
|
||||
```
|
||||
|
||||
### Key Configuration Options
|
||||
|
||||
**Extraction Mode**: Controls processing quality and speed
|
||||
|
||||
- `FAST`: Fastest processing, suitable for simple documents with no OCR
|
||||
- `BALANCED`: Good speed/accuracy tradeoff for text-rich documents
|
||||
- `MULTIMODAL`: For visually rich documents with text, tables, and images (recommended)
|
||||
- `PREMIUM`: Highest accuracy with OCR, complex table/header detection
|
||||
|
||||
**Extraction Target**: Defines extraction scope
|
||||
|
||||
- `PER_DOC`: Apply schema to entire document (default)
|
||||
- `PER_PAGE`: Apply schema to each page, returns array of results
|
||||
|
||||
**Advanced Options**:
|
||||
|
||||
- `system_prompt`: Additional system-level instructions
|
||||
- `page_range`: Specific pages to extract (e.g., "1,3,5-7,9")
|
||||
- `chunk_mode`: Document splitting strategy (`PAGE` or `SECTION`)
|
||||
- `high_resolution_mode`: Better OCR for small text (slower processing)
|
||||
|
||||
**Extensions** (return additional metadata):
|
||||
|
||||
- `cite_sources`: Source tracing for extracted fields
|
||||
- `use_reasoning`: Explanations for extraction decisions
|
||||
- `confidence_scores`: Quantitative confidence measures (MULTIMODAL/PREMIUM only)
|
||||
|
||||
For complete configuration options, advanced settings, and detailed examples, see the [LlamaExtract Configuration Documentation](https://docs.cloud.llamaindex.ai/llamaextract/features/options).
|
||||
|
||||
## Extraction Agents (Advanced)
|
||||
|
||||
For reusable extraction workflows, you can create extraction agents that encapsulate both schema and configuration:
|
||||
@@ -326,6 +397,7 @@ Another option (orthogonal to the above) is to break the document into smaller s
|
||||
|
||||
## Additional Resources
|
||||
|
||||
- [Extract Documentation](https://docs.cloud.llamaindex.ai/llamaextract/getting_started) - Details on Extract features, API and examples.
|
||||
- [Example Notebook](docs/examples-py/extract/resume_screening.ipynb) - Detailed walkthrough of resume parsing
|
||||
- [Example Application with TypeScript](./examples-ts/extract/) - End-to-end examples using LlamaExtract TypeScript client.
|
||||
- [Discord Community](https://discord.com/invite/eN6D2HQ4aX) - Get help and share feedback
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from .client import FileClient
|
||||
|
||||
__all__ = ["FileClient"]
|
||||
@@ -0,0 +1,82 @@
|
||||
from io import BytesIO
|
||||
from typing import BinaryIO
|
||||
import os
|
||||
from llama_cloud.client import AsyncLlamaCloud
|
||||
from llama_cloud.types import File, FileCreate
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class FileClient:
|
||||
"""
|
||||
Higher-level client for interacting with the LlamaCloud Files API.
|
||||
Uses presigned URLs for uploads by default.
|
||||
|
||||
Args:
|
||||
client: The LlamaCloud client to use.
|
||||
project_id: The project ID to use.
|
||||
organization_id: The organization ID to use.
|
||||
use_presigned_url: Whether to use presigned URLs for uploads (set to False when uploading to BYOC deployments).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
client: AsyncLlamaCloud,
|
||||
project_id: Optional[str] = None,
|
||||
organization_id: Optional[str] = None,
|
||||
use_presigned_url: bool = True,
|
||||
):
|
||||
self.client = client
|
||||
self.project_id = project_id
|
||||
self.organization_id = organization_id
|
||||
self.use_presigned_url = use_presigned_url
|
||||
|
||||
async def get_file(self, file_id: str) -> File:
|
||||
return await self.client.files.get_file(
|
||||
file_id, project_id=self.project_id, organization_id=self.organization_id
|
||||
)
|
||||
|
||||
async def upload_file(
|
||||
self, file_path: str, external_file_id: Optional[str] = None
|
||||
) -> File:
|
||||
external_file_id = external_file_id or file_path
|
||||
file_size = os.path.getsize(file_path)
|
||||
with open(file_path, "rb") as file:
|
||||
return await self.upload_buffer(file, external_file_id, file_size)
|
||||
|
||||
async def upload_bytes(self, bytes: bytes, external_file_id: str) -> File:
|
||||
return await self.upload_buffer(BytesIO(bytes), external_file_id, len(bytes))
|
||||
|
||||
async def upload_buffer(
|
||||
self,
|
||||
buffer: BinaryIO,
|
||||
external_file_id: str,
|
||||
file_size: int,
|
||||
) -> File:
|
||||
if self.use_presigned_url:
|
||||
if getattr(buffer, "name", None):
|
||||
name = os.path.basename(str(getattr(buffer, "name", external_file_id)))
|
||||
else:
|
||||
name = external_file_id
|
||||
presigned_url = await self.client.files.generate_presigned_url(
|
||||
project_id=self.project_id,
|
||||
organization_id=self.organization_id,
|
||||
request=FileCreate(
|
||||
name=name,
|
||||
external_file_id=external_file_id,
|
||||
file_size=file_size,
|
||||
),
|
||||
)
|
||||
httpx_client = self.client._client_wrapper.httpx_client
|
||||
httpx_client.post(presigned_url.url, data=buffer)
|
||||
return await self.client.files.get_file(
|
||||
presigned_url.file_id,
|
||||
project_id=self.project_id,
|
||||
organization_id=self.organization_id,
|
||||
)
|
||||
else:
|
||||
return await self.client.files.upload_file(
|
||||
upload_file=buffer,
|
||||
external_file_id=external_file_id,
|
||||
project_id=self.project_id,
|
||||
organization_id=self.organization_id,
|
||||
)
|
||||
@@ -52,6 +52,10 @@ class PageItem(BaseModel):
|
||||
bBox: Optional[BBox] = Field(
|
||||
default=None, description="The bounding box of the item."
|
||||
)
|
||||
html: Optional[str] = Field(
|
||||
default=None,
|
||||
description="The HTML-formatted content of the item. Only applicable for table items when output_tables_as_HTML=True.",
|
||||
)
|
||||
|
||||
|
||||
class ImageItem(BaseModel):
|
||||
@@ -108,7 +112,7 @@ class ChartItem(BaseModel):
|
||||
class Page(BaseModel):
|
||||
"""A page of the document."""
|
||||
|
||||
page: int = Field(description="The page number.")
|
||||
page: int = Field(default=0, description="The page number.")
|
||||
text: Optional[str] = Field(default=None, description="The text of the page.")
|
||||
md: Optional[str] = Field(default=None, description="The markdown of the page.")
|
||||
images: List[ImageItem] = Field(
|
||||
@@ -149,6 +153,12 @@ class Page(BaseModel):
|
||||
noTextContent: bool = Field(
|
||||
default=False, description="Whether the page has no text content."
|
||||
)
|
||||
isAudioTranscript: bool = Field(
|
||||
default=False, description="Whether the page is an audio transcript."
|
||||
)
|
||||
durationInSeconds: Optional[float] = Field(
|
||||
default=None, description="The duration of the audio transcript in seconds."
|
||||
)
|
||||
|
||||
|
||||
class JobResult(BaseModel):
|
||||
|
||||
@@ -11,13 +11,13 @@ dev = [
|
||||
|
||||
[project]
|
||||
name = "llama-parse"
|
||||
version = "0.6.55"
|
||||
version = "0.6.57"
|
||||
description = "Parse files into RAG-Optimized formats."
|
||||
authors = [{name = "Logan Markewich", email = "logan@llamaindex.ai"}]
|
||||
requires-python = ">=3.9,<4.0"
|
||||
readme = "README.md"
|
||||
license = "MIT"
|
||||
dependencies = ["llama-cloud-services>=0.6.55"]
|
||||
dependencies = ["llama-cloud-services>=0.6.56"]
|
||||
|
||||
[project.scripts]
|
||||
llama-parse = "llama_parse.cli.main:parse"
|
||||
|
||||
+3
-2
@@ -12,12 +12,13 @@ dev = [
|
||||
"deepdiff>=8.1.1,<9",
|
||||
"ipython>=8.12.3,<9",
|
||||
"jupyter>=1.1.1,<2",
|
||||
"mypy>=1.14.1,<2"
|
||||
"mypy>=1.14.1,<2",
|
||||
"pydantic-settings>=2.10.1"
|
||||
]
|
||||
|
||||
[project]
|
||||
name = "llama-cloud-services"
|
||||
version = "0.6.55"
|
||||
version = "0.6.57"
|
||||
description = "Tailored SDK clients for LlamaCloud services."
|
||||
authors = [{name = "Logan Markewich", email = "logan@runllama.ai"}]
|
||||
requires-python = ">=3.9,<4.0"
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
[pytest]
|
||||
asyncio_mode = auto
|
||||
@@ -0,0 +1,25 @@
|
||||
import pytest
|
||||
from llama_index.core.constants import DEFAULT_BASE_URL
|
||||
from pydantic import Field, SecretStr
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
|
||||
class EndToEndTestSettings(BaseSettings):
|
||||
LLAMA_CLOUD_BASE_URL: str = Field(
|
||||
description="The base URL of the LlamaCloud API", default=DEFAULT_BASE_URL
|
||||
)
|
||||
LLAMA_CLOUD_API_KEY: SecretStr = Field(
|
||||
description="The API key for the LlamaCloud API"
|
||||
)
|
||||
LLAMA_CLOUD_ORGANIZATION_ID: str | None = Field(
|
||||
default=None, description="The organization ID for the LlamaCloud API"
|
||||
)
|
||||
LLAMA_CLOUD_PROJECT_NAME: str = Field(
|
||||
description="The project name for the LlamaCloud API",
|
||||
default="framework_integration_test",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def e2e_test_settings() -> EndToEndTestSettings:
|
||||
return EndToEndTestSettings()
|
||||
@@ -202,3 +202,12 @@ async def test_get_result(markdown_parser: LlamaParse) -> None:
|
||||
result = await markdown_parser.aget_result(expected.job_id)
|
||||
assert result.job_id == expected.job_id
|
||||
assert len(result.pages) == len(expected.pages)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_audio() -> None:
|
||||
parser = LlamaParse()
|
||||
filepath = "tests/test_files/hello_world.m4a"
|
||||
|
||||
result = await parser.aparse(filepath)
|
||||
assert result.job_id is not None
|
||||
|
||||
Binary file not shown.
@@ -0,0 +1,132 @@
|
||||
import os
|
||||
import pytest
|
||||
from io import BytesIO
|
||||
from llama_cloud.client import AsyncLlamaCloud
|
||||
from llama_cloud.types import Project, File
|
||||
from llama_cloud_services.files.client import FileClient
|
||||
from tests.conftest import EndToEndTestSettings
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def llama_cloud_client(
|
||||
e2e_test_settings: EndToEndTestSettings,
|
||||
) -> AsyncLlamaCloud:
|
||||
return AsyncLlamaCloud(
|
||||
token=e2e_test_settings.LLAMA_CLOUD_API_KEY.get_secret_value(),
|
||||
base_url=e2e_test_settings.LLAMA_CLOUD_BASE_URL,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def project(
|
||||
llama_cloud_client: AsyncLlamaCloud, e2e_test_settings: EndToEndTestSettings
|
||||
) -> Project:
|
||||
projects = await llama_cloud_client.projects.list_projects(
|
||||
project_name=e2e_test_settings.LLAMA_CLOUD_PROJECT_NAME,
|
||||
organization_id=e2e_test_settings.LLAMA_CLOUD_ORGANIZATION_ID,
|
||||
)
|
||||
assert len(projects) == 1
|
||||
return projects[0]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def file_client(
|
||||
llama_cloud_client: AsyncLlamaCloud, project: Project, use_presigned_url: bool
|
||||
) -> FileClient:
|
||||
return FileClient(
|
||||
llama_cloud_client,
|
||||
project_id=project.id,
|
||||
organization_id=project.organization_id,
|
||||
use_presigned_url=use_presigned_url,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_file() -> str:
|
||||
return "tests/test_files/index/Simple PDF Slides.pdf"
|
||||
|
||||
|
||||
parametrize_use_presigned_url = pytest.mark.parametrize(
|
||||
"use_presigned_url", [True, False]
|
||||
)
|
||||
|
||||
|
||||
@parametrize_use_presigned_url
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_file_from_path(file_client: FileClient, test_file: str):
|
||||
"""Test uploading a file from file path"""
|
||||
external_file_id = f"test_upload_path_{os.getpid()}"
|
||||
uploaded_file = await file_client.upload_file(test_file, external_file_id)
|
||||
|
||||
assert isinstance(uploaded_file, File)
|
||||
expected_name = os.path.basename(test_file)
|
||||
assert uploaded_file.name == expected_name
|
||||
assert uploaded_file.external_file_id == external_file_id
|
||||
|
||||
|
||||
@parametrize_use_presigned_url
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_bytes(
|
||||
file_client: FileClient, test_file: str, use_presigned_url: bool
|
||||
):
|
||||
"""Test uploading a file from bytes"""
|
||||
# Read file as bytes
|
||||
with open(test_file, "rb") as f:
|
||||
file_bytes = f.read()
|
||||
|
||||
external_file_id = f"test_upload_bytes_{os.getpid()}"
|
||||
uploaded_file = await file_client.upload_bytes(file_bytes, external_file_id)
|
||||
|
||||
assert isinstance(uploaded_file, File)
|
||||
expected_name = external_file_id if use_presigned_url else "upload"
|
||||
assert uploaded_file.name == expected_name
|
||||
assert uploaded_file.external_file_id == external_file_id
|
||||
|
||||
|
||||
@parametrize_use_presigned_url
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_buffer(
|
||||
file_client: FileClient, test_file: str, use_presigned_url: bool
|
||||
):
|
||||
"""Test uploading a file from buffer"""
|
||||
# Read file as bytes and create buffer
|
||||
with open(test_file, "rb") as f:
|
||||
file_bytes = f.read()
|
||||
|
||||
buffer = BytesIO(file_bytes)
|
||||
file_size = len(file_bytes)
|
||||
external_file_id = f"test_upload_buffer_{os.getpid()}"
|
||||
|
||||
uploaded_file = await file_client.upload_buffer(buffer, external_file_id, file_size)
|
||||
|
||||
assert isinstance(uploaded_file, File)
|
||||
expected_name = external_file_id if use_presigned_url else "upload"
|
||||
assert uploaded_file.name == expected_name
|
||||
assert uploaded_file.external_file_id == external_file_id
|
||||
|
||||
|
||||
@parametrize_use_presigned_url
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_file(file_client: FileClient, test_file: str):
|
||||
"""Test retrieving a file by ID"""
|
||||
# Upload a file first
|
||||
external_file_id = f"test_get_file_{os.getpid()}"
|
||||
uploaded_file = await file_client.upload_file(test_file, external_file_id)
|
||||
|
||||
# Retrieve the file by ID
|
||||
retrieved_file = await file_client.get_file(uploaded_file.id)
|
||||
|
||||
assert isinstance(retrieved_file, File)
|
||||
assert retrieved_file == uploaded_file
|
||||
|
||||
|
||||
@parametrize_use_presigned_url
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_with_default_external_id(file_client: FileClient, test_file: str):
|
||||
"""Test uploading file with default external_file_id"""
|
||||
# Upload file without specifying external_file_id
|
||||
uploaded_file = await file_client.upload_file(test_file)
|
||||
|
||||
assert isinstance(uploaded_file, File)
|
||||
assert uploaded_file.name == os.path.basename(test_file)
|
||||
assert uploaded_file.external_file_id == test_file
|
||||
Generated
+2266
-2250
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "llama-cloud-services",
|
||||
"version": "0.3.0",
|
||||
"version": "0.3.1",
|
||||
"type": "module",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
|
||||
@@ -14,7 +14,8 @@ import {
|
||||
getJobTextResultApiV1ParsingJobJobIdResultTextGet,
|
||||
uploadFileApiV1ParsingUploadPost,
|
||||
} from "./api";
|
||||
import { sleep } from "./utils";
|
||||
import { sleep, getSavePath } from "./utils";
|
||||
import type { ParseResult } from "./type";
|
||||
|
||||
export type Language = ParserLanguages;
|
||||
export type ResultType = "text" | "markdown" | "json";
|
||||
@@ -594,15 +595,15 @@ export class LlamaParseReader extends FileReader {
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads data from a file and returns an array of JSON objects.
|
||||
* Loads data from a file and returns an array of ParseResult objects.
|
||||
* To be used with resultType "json".
|
||||
*
|
||||
* @param filePathOrContent - The file path or the file content as a Uint8Array.
|
||||
* @returns A Promise that resolves to an array of JSON objects.
|
||||
* @returns A Promise that resolves to an array of ParseResult objects, encapsulating the JSON array resulting from the parsing within the pages attribute.
|
||||
*/
|
||||
async loadJson(
|
||||
filePathOrContent: string | Uint8Array,
|
||||
): Promise<Record<string, any>[]> {
|
||||
): Promise<ParseResult[]> {
|
||||
let jobId;
|
||||
const isFilePath =
|
||||
typeof filePathOrContent === "string" &&
|
||||
@@ -628,7 +629,7 @@ export class LlamaParseReader extends FileReader {
|
||||
const resultJson = await this.getJobResult(jobId, "json");
|
||||
resultJson.job_id = jobId;
|
||||
resultJson.file_path = isFilePath ? filePathOrContent : undefined;
|
||||
return [resultJson];
|
||||
return [resultJson] as ParseResult[];
|
||||
} catch (e) {
|
||||
console.error(`Error while parsing the file under job id ${jobId}`, e);
|
||||
if (this.ignoreErrors) {
|
||||
@@ -639,6 +640,80 @@ export class LlamaParseReader extends FileReader {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads data from a file or file bytes (or an array of those) and returns an array of ParseResult objects.
|
||||
*
|
||||
* @param filePathOrContent - The file path or the file content as a Uint8Array, or an array of one of those two types.
|
||||
* @returns A Promise that resolves to an array of ParseResult objects, encapsulating the JSON array resulting from the parsing within the pages attribute.
|
||||
*/
|
||||
async parse(
|
||||
filePathOrContent: string | Uint8Array | string[] | Uint8Array[],
|
||||
): Promise<ParseResult[]> {
|
||||
const jsonResults: Record<string, any>[][] = [];
|
||||
if (!Array.isArray(filePathOrContent)) {
|
||||
const jsonResult = await this.loadJson(filePathOrContent);
|
||||
jsonResults.push(jsonResult);
|
||||
} else {
|
||||
for (let i = 0; i < filePathOrContent.length; i++) {
|
||||
console.log(
|
||||
`Processing file ${i + 1} of ${filePathOrContent.length}...`,
|
||||
);
|
||||
const jsonResult = await this.loadJson(
|
||||
filePathOrContent[i] as string | Uint8Array,
|
||||
);
|
||||
jsonResults.push(jsonResult);
|
||||
}
|
||||
}
|
||||
const parseResults: ParseResult[] = [];
|
||||
for (const jsonResult of jsonResults) {
|
||||
for (const result of jsonResult) {
|
||||
const parseResult = {
|
||||
pages: result.pages,
|
||||
job_metadata: result.job_metadata,
|
||||
job_id: result.job_id,
|
||||
file_path: result?.file_path ?? "",
|
||||
is_completed: true,
|
||||
} as ParseResult;
|
||||
parseResults.push(parseResult);
|
||||
}
|
||||
}
|
||||
return parseResults;
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads and saves tables from a given array of ParseResult to a specified download path.
|
||||
*
|
||||
* @param jsonResults - The array of ParseResult containing table information.
|
||||
* @param downloadPath - The path where the downloaded tables will be saved as CSV files.
|
||||
* @returns A Promise that resolves to an array of strings representing the paths to the tables.
|
||||
*/
|
||||
async getTables(
|
||||
jsonResults: ParseResult[],
|
||||
downloadPath: string,
|
||||
): Promise<string[]> {
|
||||
const tables: string[] = [];
|
||||
for (const result of jsonResults) {
|
||||
for (const page of result.pages) {
|
||||
if ("items" in page && Array.isArray(page.items)) {
|
||||
for (let i = 0; i < page.items.length; i++) {
|
||||
if (
|
||||
"type" in page.items[i] &&
|
||||
page.items[i].type === "table" &&
|
||||
"csv" in page.items[i] &&
|
||||
typeof page.items[i].csv === "string" &&
|
||||
page.items[i].csv != ""
|
||||
) {
|
||||
const savePath = getSavePath(downloadPath, i);
|
||||
await fs.writeFile(savePath, page.items[i].csv);
|
||||
tables.push(savePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return tables;
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads and saves images from a given JSON result to a specified download path.
|
||||
* Currently only supports resultType "json".
|
||||
@@ -648,7 +723,7 @@ export class LlamaParseReader extends FileReader {
|
||||
* @returns A Promise that resolves to an array of image objects.
|
||||
*/
|
||||
async getImages(
|
||||
jsonResult: Record<string, any>[],
|
||||
jsonResult: ParseResult[],
|
||||
downloadPath: string,
|
||||
): Promise<Record<string, any>[]> {
|
||||
try {
|
||||
|
||||
@@ -49,3 +49,13 @@ export type ExtractResult = {
|
||||
| null;
|
||||
};
|
||||
};
|
||||
|
||||
export type ParseResult = {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
pages: Record<string, any>[];
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
job_metadata: Record<string, any>;
|
||||
job_id: string;
|
||||
is_completed: boolean;
|
||||
file_path: string;
|
||||
};
|
||||
|
||||
@@ -6,6 +6,8 @@ import {
|
||||
import { DEFAULT_BASE_URL } from "@llamaindex/core/global";
|
||||
import { getEnv } from "@llamaindex/env";
|
||||
import type { ClientParams } from "./type.js";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
|
||||
function getBaseUrl(baseUrl?: string): string {
|
||||
return baseUrl ?? getEnv("LLAMA_CLOUD_BASE_URL") ?? DEFAULT_BASE_URL;
|
||||
@@ -99,3 +101,19 @@ export async function getPipelineId(
|
||||
export async function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
export function getSavePath(downloadPath: string, i: number): string {
|
||||
const now = new Date();
|
||||
const formatted =
|
||||
now.toISOString().replace(/[-:T]/g, "_").replace(/\..+/, "") +
|
||||
"_" +
|
||||
now.getMilliseconds().toString().padStart(3, "0");
|
||||
|
||||
let savePath = path.join(downloadPath, `table_${formatted}.csv`);
|
||||
|
||||
if (fs.existsSync(savePath)) {
|
||||
savePath = savePath.replace(".csv", "_") + i.toString() + ".csv";
|
||||
}
|
||||
|
||||
return savePath;
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import { LlamaExtract, LlamaExtractAgent } from "../src/LlamaExtract.js";
|
||||
import { Document } from "@llamaindex/core/schema";
|
||||
import { fs } from "@llamaindex/env";
|
||||
import { ExtractConfig } from "../src/api.js";
|
||||
import { ParseResult } from "../src/type.js";
|
||||
|
||||
// Integration tests that require actual API keys and files
|
||||
describe("Integration Tests", () => {
|
||||
@@ -284,6 +285,78 @@ describe("Integration Tests", () => {
|
||||
},
|
||||
60000,
|
||||
);
|
||||
|
||||
it.skipIf(skipIfNoApiKey)(
|
||||
"should parse a file and return a ParseResult array",
|
||||
async () => {
|
||||
const parseReader = new LlamaParseReader({
|
||||
apiKey: process.env.LLAMA_CLOUD_API_KEY!,
|
||||
verbose: false,
|
||||
});
|
||||
|
||||
const testContent = "Test document for JSON parsing";
|
||||
const testFilePath = "test-json.txt";
|
||||
|
||||
await fs.writeFile(testFilePath, new TextEncoder().encode(testContent));
|
||||
|
||||
try {
|
||||
const result = await parseReader.parse(testFilePath);
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(Array.isArray(result)).toBe(true);
|
||||
expect(result.length).toBeGreaterThan(0);
|
||||
expect(result[0]).toHaveProperty("job_id");
|
||||
expect(result[0]).toHaveProperty("job_metadata");
|
||||
expect(result[0]).toHaveProperty("file_path");
|
||||
expect(result[0]).toHaveProperty("pages");
|
||||
|
||||
await fs.unlink(testFilePath);
|
||||
} catch (error) {
|
||||
try {
|
||||
await fs.unlink(testFilePath);
|
||||
} catch {}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
60000,
|
||||
);
|
||||
|
||||
it.skipIf(skipIfNoApiKey)(
|
||||
"should extract tables correctly from a JSON result",
|
||||
async () => {
|
||||
const parseReader = new LlamaParseReader({
|
||||
apiKey: process.env.LLAMA_CLOUD_API_KEY!,
|
||||
verbose: false,
|
||||
});
|
||||
const pseudoJsonResult = [
|
||||
{
|
||||
pages: [
|
||||
{
|
||||
items: [
|
||||
{
|
||||
type: "table",
|
||||
csv: "Name,Age,Height (cm)\nAnna,12,140\nBob,22,175\nClaire,33,173\nDenis,44,185\n",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
job_id: "jobId",
|
||||
job_metadata: { job_id: "jobId" },
|
||||
file_path: "table.csv",
|
||||
is_completed: true,
|
||||
},
|
||||
] as ParseResult[];
|
||||
|
||||
const tmpdir = await fs.mkdtemp("tables");
|
||||
const result = await parseReader.getTables(pseudoJsonResult, tmpdir);
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(Array.isArray(result)).toBe(true);
|
||||
expect(result.length).toBeGreaterThan(0);
|
||||
expect(typeof result[0] === "string").toBe(true);
|
||||
},
|
||||
60000,
|
||||
);
|
||||
});
|
||||
|
||||
describe("LlamaCloudIndex Integration", () => {
|
||||
|
||||
Reference in New Issue
Block a user