mirror of
https://github.com/run-llama/llama_cloud_services.git
synced 2026-07-21 03:55:22 -04:00
Compare commits
14 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0c69f99cf0 | |||
| e48f544ddc | |||
| 4aa7ad5642 | |||
| c39cdbcd01 | |||
| 71eaa8bcc6 | |||
| 1e1cbdfc79 | |||
| cc8af4a43a | |||
| 43fbd48ab8 | |||
| 5ec66e9452 | |||
| 211521c82e | |||
| 4ddaab1efb | |||
| 53e5ce2e83 | |||
| 9f4bd1cb64 | |||
| 456863752b |
@@ -0,0 +1,11 @@
|
||||
# Please see the documentation for all configuration options:
|
||||
# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
|
||||
# and
|
||||
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
|
||||
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
@@ -21,9 +21,9 @@ jobs:
|
||||
os: [ubuntu-latest, windows-latest]
|
||||
python-version: ["3.9"]
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v4
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install Poetry
|
||||
|
||||
@@ -1,14 +1,3 @@
|
||||
# For most projects, this workflow file will not need changing; you simply need
|
||||
# to commit it to your repository.
|
||||
#
|
||||
# You may wish to alter this file to override the set of languages analyzed,
|
||||
# or to provide custom queries or build logic.
|
||||
#
|
||||
# ******** NOTE ********
|
||||
# We have attempted to detect the languages in your repository. Please check
|
||||
# the `language` matrix defined below to confirm you have the correct set of
|
||||
# supported CodeQL languages.
|
||||
#
|
||||
name: "CodeQL"
|
||||
|
||||
on:
|
||||
@@ -28,54 +17,25 @@ jobs:
|
||||
# - https://gh.io/supported-runners-and-hardware-resources
|
||||
# - https://gh.io/using-larger-runners
|
||||
# Consider using larger runners for possible analysis time improvements.
|
||||
runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
|
||||
timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
|
||||
runs-on: "ubuntu-latest"
|
||||
timeout-minutes: 360
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
language: ["python"]
|
||||
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby', 'swift' ]
|
||||
# Use only 'java' to analyze code written in Java, Kotlin or both
|
||||
# Use only 'javascript' to analyze code written in JavaScript, TypeScript or both
|
||||
# Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v2
|
||||
uses: github/codeql-action/init@v3
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
# By default, queries listed here will override any specified in a config file.
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
|
||||
# For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
|
||||
# queries: security-extended,security-and-quality
|
||||
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
|
||||
# If this step fails, then you should remove it and run the build manually (see below)
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v2
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||
|
||||
# If the Autobuild fails above, remove it and uncomment the following three lines.
|
||||
# modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
|
||||
|
||||
# - run: |
|
||||
# echo "Run, Build Application using script"
|
||||
# ./location_of_script_within_repo/buildscript.sh
|
||||
languages: python
|
||||
dependency-caching: true
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v2
|
||||
uses: github/codeql-action/analyze@v3
|
||||
with:
|
||||
category: "/language:${{matrix.language}}"
|
||||
category: "/language:python"
|
||||
|
||||
@@ -18,11 +18,11 @@ jobs:
|
||||
matrix:
|
||||
python-version: ["3.9"]
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }}
|
||||
- name: Set up python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v4
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install Poetry
|
||||
|
||||
@@ -18,9 +18,9 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up python ${{ env.PYTHON_VERSION }}
|
||||
uses: actions/setup-python@v4
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
@@ -39,10 +39,18 @@ jobs:
|
||||
pypi_token: ${{ secrets.LLAMA_PARSE_PYPI_TOKEN }}
|
||||
poetry_install_options: "--without dev"
|
||||
|
||||
- name: Wait for PyPI to update
|
||||
run: |
|
||||
sleep 60
|
||||
|
||||
- name: Update llama-parse lock file
|
||||
run: |
|
||||
cd llama_parse && poetry lock
|
||||
|
||||
- name: Build and publish llama-parse
|
||||
uses: JRubics/poetry-publish@v2.1
|
||||
with:
|
||||
working_directory: "llama_parse"
|
||||
package_directory: "./llama_parse"
|
||||
pypi_token: ${{ secrets.LLAMA_PARSE_PYPI_TOKEN }}
|
||||
poetry_install_options: "--without dev"
|
||||
|
||||
|
||||
@@ -19,11 +19,11 @@ jobs:
|
||||
matrix:
|
||||
python-version: ["3.9", "3.10", "3.11", "3.12"]
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Set up python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v4
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install Poetry
|
||||
|
||||
@@ -1,7 +1,17 @@
|
||||
from llama_cloud_services.extract.extract import (
|
||||
LlamaExtract,
|
||||
ExtractConfig,
|
||||
ExtractionAgent,
|
||||
SourceText,
|
||||
ExtractTarget,
|
||||
ExtractMode,
|
||||
)
|
||||
|
||||
__all__ = ["LlamaExtract", "ExtractionAgent", "SourceText"]
|
||||
__all__ = [
|
||||
"LlamaExtract",
|
||||
"ExtractionAgent",
|
||||
"SourceText",
|
||||
"ExtractConfig",
|
||||
"ExtractTarget",
|
||||
"ExtractMode",
|
||||
]
|
||||
|
||||
@@ -10,13 +10,10 @@ import httpx
|
||||
from pydantic import BaseModel
|
||||
from llama_cloud import (
|
||||
ExtractAgent as CloudExtractAgent,
|
||||
ExtractAgentCreate,
|
||||
ExtractConfig,
|
||||
ExtractJob,
|
||||
ExtractJobCreate,
|
||||
ExtractRun,
|
||||
ExtractSchemaValidateRequest,
|
||||
ExtractAgentUpdate,
|
||||
File,
|
||||
ExtractMode,
|
||||
StatusEnum,
|
||||
@@ -26,7 +23,11 @@ from llama_cloud import (
|
||||
PaginatedExtractRunsResponse,
|
||||
)
|
||||
from llama_cloud.client import AsyncLlamaCloud
|
||||
from llama_cloud_services.extract.utils import JSONObjectType, augment_async_errors
|
||||
from llama_cloud_services.extract.utils import (
|
||||
JSONObjectType,
|
||||
augment_async_errors,
|
||||
ExperimentalWarning,
|
||||
)
|
||||
from llama_index.core.schema import BaseComponent
|
||||
from llama_index.core.async_utils import run_jobs
|
||||
from llama_index.core.bridge.pydantic import Field, PrivateAttr
|
||||
@@ -118,6 +119,25 @@ def run_in_thread(
|
||||
return thread_pool.submit(run_coro).result()
|
||||
|
||||
|
||||
def _extraction_config_warning(config: ExtractConfig) -> None:
|
||||
if config.extraction_mode == ExtractMode.ACCURATE:
|
||||
warnings.warn("ACCURATE extraction mode is deprecated. Using BALANCED instead.")
|
||||
config.extraction_mode = ExtractMode.BALANCED
|
||||
if config.use_reasoning:
|
||||
warnings.warn(
|
||||
"`use_reasoning` is an experimental feature. Results will be available in "
|
||||
"the `extraction_metadata` field for the extraction run.",
|
||||
ExperimentalWarning,
|
||||
)
|
||||
if config.cite_sources:
|
||||
warnings.warn(
|
||||
"`cite_sources` is an experimental feature. This may greatly increase the "
|
||||
"size of the response, and slow down the extraction. Results will be "
|
||||
"available in the `extraction_metadata` field for the extraction run.",
|
||||
ExperimentalWarning,
|
||||
)
|
||||
|
||||
|
||||
class ExtractionAgent:
|
||||
"""Class representing a single extraction agent with methods for extraction operations."""
|
||||
|
||||
@@ -178,7 +198,7 @@ class ExtractionAgent:
|
||||
)
|
||||
validated_schema = self._run_in_thread(
|
||||
self._client.llama_extract.validate_extraction_schema(
|
||||
request=ExtractSchemaValidateRequest(data_schema=processed_schema)
|
||||
data_schema=processed_schema
|
||||
)
|
||||
)
|
||||
self._data_schema = validated_schema.data_schema
|
||||
@@ -189,6 +209,7 @@ class ExtractionAgent:
|
||||
|
||||
@config.setter
|
||||
def config(self, config: ExtractConfig) -> None:
|
||||
_extraction_config_warning(config)
|
||||
self._config = config
|
||||
|
||||
def _run_in_thread(self, coro: Coroutine[Any, Any, T]) -> T:
|
||||
@@ -307,10 +328,8 @@ class ExtractionAgent:
|
||||
self._agent = self._run_in_thread(
|
||||
self._client.llama_extract.update_extraction_agent(
|
||||
extraction_agent_id=self.id,
|
||||
request=ExtractAgentUpdate(
|
||||
data_schema=self.data_schema,
|
||||
config=self.config,
|
||||
),
|
||||
data_schema=self.data_schema,
|
||||
config=self.config,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -602,7 +621,7 @@ class LlamaExtract(BaseComponent):
|
||||
httpx_timeout=httpx_timeout,
|
||||
verbose=verbose,
|
||||
)
|
||||
self._httpx_client = httpx.AsyncClient(verify=verify, timeout=httpx_timeout)
|
||||
self._httpx_client = httpx.AsyncClient(verify=verify, timeout=httpx_timeout) # type: ignore
|
||||
self.verify = verify
|
||||
self.httpx_timeout = httpx_timeout
|
||||
|
||||
@@ -659,11 +678,7 @@ class LlamaExtract(BaseComponent):
|
||||
ExtractionAgent: The created extraction agent
|
||||
"""
|
||||
if config is not None:
|
||||
if config.extraction_mode == ExtractMode.ACCURATE:
|
||||
warnings.warn(
|
||||
"ACCURATE extraction mode is deprecated. Using BALANCED instead."
|
||||
)
|
||||
config.extraction_mode = ExtractMode.BALANCED
|
||||
_extraction_config_warning(config)
|
||||
else:
|
||||
config = DEFAULT_EXTRACT_CONFIG
|
||||
|
||||
@@ -680,11 +695,9 @@ class LlamaExtract(BaseComponent):
|
||||
self._async_client.llama_extract.create_extraction_agent(
|
||||
project_id=self._project_id,
|
||||
organization_id=self._organization_id,
|
||||
request=ExtractAgentCreate(
|
||||
name=name,
|
||||
data_schema=data_schema,
|
||||
config=config,
|
||||
),
|
||||
name=name,
|
||||
data_schema=data_schema,
|
||||
config=config,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -32,3 +32,9 @@ def augment_async_errors() -> Generator[None, None, None]:
|
||||
|
||||
JSONType = Union[Dict[str, Any], List[Any], str, int, float, bool, None]
|
||||
JSONObjectType = Dict[str, JSONType]
|
||||
|
||||
|
||||
class ExperimentalWarning(Warning):
|
||||
"""Warning for experimental features."""
|
||||
|
||||
pass
|
||||
|
||||
@@ -7,7 +7,7 @@ from copy import deepcopy
|
||||
from enum import Enum
|
||||
from io import BufferedIOBase
|
||||
from pathlib import Path, PurePath, PurePosixPath
|
||||
from typing import Any, AsyncGenerator, Dict, List, Optional, Union
|
||||
from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple, Union
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
@@ -115,7 +115,7 @@ class LlamaParse(BasePydanticReader):
|
||||
num_workers: int = Field(
|
||||
default=4,
|
||||
gt=0,
|
||||
lt=10,
|
||||
lt=20,
|
||||
description="The number of workers to use sending API requests for parsing.",
|
||||
)
|
||||
result_type: ResultType = Field(
|
||||
@@ -145,6 +145,10 @@ class LlamaParse(BasePydanticReader):
|
||||
default=False,
|
||||
description="If set to true, the parser will automatically select the best mode to extract text from documents based on the rules provide. Will use the 'accurate' default mode by default and will upgrade page that match the rule to Premium mode.",
|
||||
)
|
||||
auto_mode_configuration_json: Optional[str] = Field(
|
||||
default=None,
|
||||
description="A JSON string containing the configuration for the auto mode. If set, the parser will use the provided configuration for the auto mode.",
|
||||
)
|
||||
auto_mode_trigger_on_image_in_page: Optional[bool] = Field(
|
||||
default=False,
|
||||
description="If auto_mode is set to true, the parser will upgrade the page that contain an image to Premium mode.",
|
||||
@@ -317,11 +321,15 @@ class LlamaParse(BasePydanticReader):
|
||||
default=False,
|
||||
description="Use our best parser mode if set to True.",
|
||||
)
|
||||
preset: Optional[str] = Field(
|
||||
default=None,
|
||||
description="The preset to use for the parser. If set, the parser will use the preset configuration. See LlamaParse documentation for available presets. Preset override most other parameters.",
|
||||
)
|
||||
preserve_layout_alignment_across_pages: Optional[bool] = Field(
|
||||
default=False,
|
||||
description="Preserve grid alignment across page in text mode.",
|
||||
)
|
||||
replace_failed_page_mode: Optional[Union[FailedPageMode, str]] = Field(
|
||||
replace_failed_page_mode: Optional[FailedPageMode] = Field(
|
||||
default=None,
|
||||
description="The mode to use to replace the failed page, see FailedPageMode enum for possible value. If set, the parser will replace the failed page with the specified mode. If not set, the default mode (raw_text) will be used.",
|
||||
)
|
||||
@@ -582,6 +590,9 @@ class LlamaParse(BasePydanticReader):
|
||||
if self.auto_mode:
|
||||
data["auto_mode"] = self.auto_mode
|
||||
|
||||
if self.auto_mode_configuration_json is not None:
|
||||
data["auto_mode_configuration_json"] = self.auto_mode_configuration_json
|
||||
|
||||
if self.auto_mode_trigger_on_image_in_page:
|
||||
data[
|
||||
"auto_mode_trigger_on_image_in_page"
|
||||
@@ -772,8 +783,12 @@ class LlamaParse(BasePydanticReader):
|
||||
"preserve_layout_alignment_across_pages"
|
||||
] = self.preserve_layout_alignment_across_pages
|
||||
|
||||
if self.preset is not None:
|
||||
data["preset"] = self.preset
|
||||
|
||||
if self.replace_failed_page_mode is not None:
|
||||
data["replace_failed_page_mode"] = self.replace_failed_page_mode
|
||||
data["replace_failed_page_mode"] = self.replace_failed_page_mode.value
|
||||
|
||||
if self.replace_failed_page_with_error_message_prefix is not None:
|
||||
data[
|
||||
"replace_failed_page_with_error_message_prefix"
|
||||
@@ -944,6 +959,22 @@ class LlamaParse(BasePydanticReader):
|
||||
)
|
||||
current_interval = self._calculate_backoff(current_interval)
|
||||
|
||||
async def _parse_one(
|
||||
self,
|
||||
file_path: FileInput,
|
||||
extra_info: Optional[dict] = None,
|
||||
fs: Optional[AbstractFileSystem] = None,
|
||||
result_type: Optional[str] = None,
|
||||
) -> Tuple[str, Dict[str, Any]]:
|
||||
"""Create one parse job and wait for the result."""
|
||||
job_id = await self._create_job(file_path, extra_info=extra_info, fs=fs)
|
||||
if self.verbose:
|
||||
print("Started parsing the file under job_id %s" % job_id)
|
||||
result = await self._get_job_result(
|
||||
job_id, result_type or self.result_type.value, verbose=self.verbose
|
||||
)
|
||||
return job_id, result
|
||||
|
||||
async def _aload_data(
|
||||
self,
|
||||
file_path: FileInput,
|
||||
@@ -953,14 +984,9 @@ class LlamaParse(BasePydanticReader):
|
||||
) -> List[Document]:
|
||||
"""Load data from the input path."""
|
||||
try:
|
||||
job_id = await self._create_job(file_path, extra_info=extra_info, fs=fs)
|
||||
if verbose:
|
||||
print("Started parsing the file under job_id %s" % job_id)
|
||||
|
||||
result = await self._get_job_result(
|
||||
job_id, self.result_type.value, verbose=verbose
|
||||
_job_id, result = await self._parse_one(
|
||||
file_path, extra_info=extra_info, fs=fs
|
||||
)
|
||||
|
||||
docs = [
|
||||
Document(
|
||||
text=result[self.result_type.value],
|
||||
@@ -1058,10 +1084,6 @@ class LlamaParse(BasePydanticReader):
|
||||
"""
|
||||
|
||||
if isinstance(file_path, (str, PurePosixPath, Path, bytes, BufferedIOBase)):
|
||||
job_id = await self._create_job(file_path, extra_info=extra_info, fs=fs)
|
||||
if self.verbose:
|
||||
print("Started parsing the file under job_id %s" % job_id)
|
||||
|
||||
if isinstance(file_path, (bytes, BufferedIOBase)):
|
||||
if not extra_info or "file_name" not in extra_info:
|
||||
raise ValueError(
|
||||
@@ -1071,8 +1093,11 @@ class LlamaParse(BasePydanticReader):
|
||||
else:
|
||||
file_name = str(file_path)
|
||||
|
||||
job_result = await self._get_job_result(
|
||||
job_id, ResultType.JSON.value, verbose=self.verbose
|
||||
job_id, job_result = await self._parse_one(
|
||||
file_path,
|
||||
extra_info=extra_info,
|
||||
fs=fs,
|
||||
result_type=ResultType.JSON.value,
|
||||
)
|
||||
return JobResult(
|
||||
job_id=job_id,
|
||||
@@ -1085,14 +1110,6 @@ class LlamaParse(BasePydanticReader):
|
||||
)
|
||||
|
||||
elif isinstance(file_path, list):
|
||||
jobs = [
|
||||
self._create_job(
|
||||
f,
|
||||
extra_info=extra_info,
|
||||
fs=fs,
|
||||
)
|
||||
for f in file_path
|
||||
]
|
||||
file_names = []
|
||||
for f in file_path:
|
||||
if isinstance(f, (bytes, BufferedIOBase)):
|
||||
@@ -1105,19 +1122,15 @@ class LlamaParse(BasePydanticReader):
|
||||
file_names.append(str(f))
|
||||
|
||||
try:
|
||||
job_ids = await run_jobs(
|
||||
jobs,
|
||||
workers=self.num_workers,
|
||||
desc="Creating parsing jobs",
|
||||
show_progress=self.show_progress,
|
||||
)
|
||||
|
||||
job_results = await run_jobs(
|
||||
[
|
||||
self._get_job_result(
|
||||
job_id, ResultType.JSON.value, verbose=self.verbose
|
||||
self._parse_one(
|
||||
f,
|
||||
extra_info=extra_info,
|
||||
fs=fs,
|
||||
result_type=ResultType.JSON.value,
|
||||
)
|
||||
for job_id in job_ids
|
||||
for f in file_path
|
||||
],
|
||||
workers=self.num_workers,
|
||||
desc="Getting job results",
|
||||
@@ -1125,21 +1138,19 @@ class LlamaParse(BasePydanticReader):
|
||||
)
|
||||
|
||||
# Create JobResults just using the job_ids and job_results
|
||||
job_results = [
|
||||
return [
|
||||
JobResult(
|
||||
job_id=job_id,
|
||||
file_name=file_names[i],
|
||||
job_result=job_results[i],
|
||||
job_result=job_result,
|
||||
api_key=self.api_key,
|
||||
base_url=self.base_url,
|
||||
client=self.aclient,
|
||||
page_separator=self.page_separator or _DEFAULT_SEPARATOR,
|
||||
)
|
||||
for i, job_id in enumerate(job_ids)
|
||||
for i, (job_id, job_result) in enumerate(job_results)
|
||||
]
|
||||
|
||||
return job_results
|
||||
|
||||
except RuntimeError as e:
|
||||
if nest_asyncio_err in str(e):
|
||||
raise RuntimeError(nest_asyncio_msg)
|
||||
@@ -1183,10 +1194,11 @@ class LlamaParse(BasePydanticReader):
|
||||
) -> List[dict]:
|
||||
"""Load data from the input path."""
|
||||
try:
|
||||
job_id = await self._create_job(file_path, extra_info=extra_info)
|
||||
if self.verbose:
|
||||
print("Started parsing the file under job_id %s" % job_id)
|
||||
result = await self._get_job_result(job_id, "json")
|
||||
job_id, result = await self._parse_one(
|
||||
file_path,
|
||||
extra_info=extra_info,
|
||||
result_type=ResultType.JSON.value,
|
||||
)
|
||||
result["job_id"] = job_id
|
||||
|
||||
if not isinstance(file_path, (bytes, BufferedIOBase)):
|
||||
|
||||
@@ -49,19 +49,31 @@ class PageItem(BaseModel):
|
||||
rows: Optional[List[List[str]]] = Field(
|
||||
default=None, description="The rows of the item."
|
||||
)
|
||||
bBox: BBox = Field(description="The bounding box of the item.")
|
||||
bBox: Optional[BBox] = Field(
|
||||
default=None, description="The bounding box of the item."
|
||||
)
|
||||
|
||||
|
||||
class ImageItem(BaseModel):
|
||||
"""An image in a page."""
|
||||
|
||||
name: str = Field(description="The name of the image.")
|
||||
height: float = Field(description="The height of the image.")
|
||||
width: float = Field(description="The width of the image.")
|
||||
x: float = Field(description="The x-coordinate of the image.")
|
||||
y: float = Field(description="The y-coordinate of the image.")
|
||||
original_width: int = Field(description="The original width of the image.")
|
||||
original_height: int = Field(description="The original height of the image.")
|
||||
height: Optional[float] = Field(
|
||||
default=None, description="The height of the image."
|
||||
)
|
||||
width: Optional[float] = Field(default=None, description="The width of the image.")
|
||||
x: Optional[float] = Field(
|
||||
default=None, description="The x-coordinate of the image."
|
||||
)
|
||||
y: Optional[float] = Field(
|
||||
default=None, description="The y-coordinate of the image."
|
||||
)
|
||||
original_width: Optional[int] = Field(
|
||||
default=None, description="The original width of the image."
|
||||
)
|
||||
original_height: Optional[int] = Field(
|
||||
default=None, description="The original height of the image."
|
||||
)
|
||||
type: Optional[str] = Field(default=None, description="The type of the image.")
|
||||
|
||||
|
||||
@@ -71,7 +83,9 @@ class LayoutItem(BaseModel):
|
||||
image: str = Field(description="The name of the image containing the layout item")
|
||||
confidence: float = Field(description="The confidence of the layout item.")
|
||||
label: str = Field(description="The label of the layout item.")
|
||||
bbox: BBox = Field(description="The bounding box of the layout item.")
|
||||
bbox: Optional[BBox] = Field(
|
||||
default=None, description="The bounding box of the layout item."
|
||||
)
|
||||
isLikelyNoise: bool = Field(description="Whether the layout item is likely noise.")
|
||||
|
||||
|
||||
@@ -79,18 +93,24 @@ class ChartItem(BaseModel):
|
||||
"""A chart in a page."""
|
||||
|
||||
name: str = Field(description="The name of the chart.")
|
||||
x: float = Field(description="The x-coordinate of the chart.")
|
||||
y: float = Field(description="The y-coordinate of the chart.")
|
||||
width: float = Field(description="The width of the chart.")
|
||||
height: float = Field(description="The height of the chart.")
|
||||
x: Optional[float] = Field(
|
||||
default=None, description="The x-coordinate of the chart."
|
||||
)
|
||||
y: Optional[float] = Field(
|
||||
default=None, description="The y-coordinate of the chart."
|
||||
)
|
||||
width: Optional[float] = Field(default=None, description="The width of the chart.")
|
||||
height: Optional[float] = Field(
|
||||
default=None, description="The height of the chart."
|
||||
)
|
||||
|
||||
|
||||
class Page(BaseModel):
|
||||
"""A page of the document."""
|
||||
|
||||
page: int = Field(description="The page number.")
|
||||
text: str = Field(description="The text of the page.")
|
||||
md: str = Field(description="The markdown of the page.")
|
||||
text: Optional[str] = Field(default=None, description="The text of the page.")
|
||||
md: Optional[str] = Field(default=None, description="The markdown of the page.")
|
||||
images: List[ImageItem] = Field(
|
||||
default_factory=list,
|
||||
description="The names of the image IDs in the page, including both objects and page screenshots.",
|
||||
@@ -111,8 +131,8 @@ class Page(BaseModel):
|
||||
links: List[SerializeAsAny[Any]] = Field(
|
||||
default_factory=list, description="The links in the page."
|
||||
)
|
||||
width: float = Field(description="The width of the page.")
|
||||
height: float = Field(description="The height of the page.")
|
||||
width: Optional[float] = Field(default=None, description="The width of the page.")
|
||||
height: Optional[float] = Field(default=None, description="The height of the page.")
|
||||
triggeredAutoMode: bool = Field(
|
||||
description="Whether the page triggered auto mode (thus increasing the cost)."
|
||||
)
|
||||
@@ -185,7 +205,9 @@ class JobResult(BaseModel):
|
||||
for page in self.pages
|
||||
]
|
||||
else:
|
||||
text = self._page_separator.join([page.text for page in self.pages])
|
||||
text = self._page_separator.join(
|
||||
[page.text if page.text is not None else "" for page in self.pages]
|
||||
)
|
||||
return [Document(text=text, metadata={"file_name": self.file_name})]
|
||||
|
||||
async def aget_text_documents(self, split_by_page: bool = False) -> List[Document]:
|
||||
@@ -230,7 +252,9 @@ class JobResult(BaseModel):
|
||||
else:
|
||||
return [
|
||||
Document(
|
||||
text=self._page_separator.join([page.md for page in self.pages]),
|
||||
text=self._page_separator.join(
|
||||
[page.md if page.md is not None else "" for page in self.pages]
|
||||
),
|
||||
metadata={"file_name": self.file_name},
|
||||
)
|
||||
]
|
||||
|
||||
Generated
+3087
File diff suppressed because it is too large
Load Diff
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.poetry]
|
||||
name = "llama-parse"
|
||||
version = "0.6.14"
|
||||
version = "0.6.21"
|
||||
description = "Parse files into RAG-Optimized formats."
|
||||
authors = ["Logan Markewich <logan@llamaindex.ai>"]
|
||||
license = "MIT"
|
||||
@@ -13,7 +13,7 @@ packages = [{include = "llama_parse"}]
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = ">=3.9,<4.0"
|
||||
llama-cloud-services = ">=0.6.14"
|
||||
llama-cloud-services = ">=0.6.21"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
pytest = "^8.0.0"
|
||||
|
||||
Generated
+642
-620
File diff suppressed because it is too large
Load Diff
+2
-2
@@ -8,7 +8,7 @@ python_version = "3.10"
|
||||
|
||||
[tool.poetry]
|
||||
name = "llama-cloud-services"
|
||||
version = "0.6.14"
|
||||
version = "0.6.21"
|
||||
description = "Tailored SDK clients for LlamaCloud services."
|
||||
authors = ["Logan Markewich <logan@runllama.ai>"]
|
||||
license = "MIT"
|
||||
@@ -18,7 +18,7 @@ packages = [{include = "llama_cloud_services"}]
|
||||
[tool.poetry.dependencies]
|
||||
python = ">=3.9,<4.0"
|
||||
llama-index-core = ">=0.11.0"
|
||||
llama-cloud = "^0.1.18"
|
||||
llama-cloud = "==0.1.19"
|
||||
pydantic = "!=2.10"
|
||||
click = "^8.1.7"
|
||||
python-dotenv = "^1.0.1"
|
||||
|
||||
Reference in New Issue
Block a user