mirror of
https://github.com/run-llama/llama_cloud_services.git
synced 2026-07-21 03:55:22 -04:00
Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b22bb1b4b7 | |||
| f20cf5c8b9 | |||
| 5edf5f914a | |||
| 22e4975cb2 | |||
| bc2f04379b | |||
| f9f951d5d8 | |||
| 355129fea5 |
@@ -29,7 +29,7 @@ repos:
|
||||
- id: black-jupyter
|
||||
name: black-src
|
||||
alias: black
|
||||
exclude: ".*uv.lock"
|
||||
exclude: ".*uv.lock|examples/extract/solar_panel_e2e_comparison.ipynb"
|
||||
- repo: https://github.com/pre-commit/mirrors-mypy
|
||||
rev: v1.0.1
|
||||
hooks:
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
"source": [
|
||||
"# Extraction and Analysis over a Fidelity Multi-Fund Annual Report\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services-demo/blob/main/examples/extract/asset_manager_fund_analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/extract/asset_manager_fund_analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"\n",
|
||||
"In this notebook we show you how to create an agentic document workflow over a complex document that contains annual reports for multiple funds - each fund reports financials in a standardized reporting structure, and it's all consolidated in the same document.\n",
|
||||
"\n",
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
"source": [
|
||||
"# Automotive Equity Research: A Multi-Step Agentic Workflow\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services-demo/blob/main/examples/extract/automotive_sector_analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/extract/automotive_sector_analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"\n",
|
||||
"This notebook demonstrates an end‑to‑end agentic workflow using LlamaExtract and the LlamaIndex event‑driven workflow framework for automotive sector analysis.\n",
|
||||
"\n",
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
"source": [
|
||||
"# Dynamic Section Retrieval with LlamaParse\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services-demo/blob/main/examples/parse/advanced_rag/dynamic_section_retrieval.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/parse/advanced_rag/dynamic_section_retrieval.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"\n",
|
||||
"This notebook showcases a concept called \"dynamic section retrieval\".\n",
|
||||
"\n",
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"source": [
|
||||
"# Advanced RAG with LlamaParse\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_parse/blob/main/examples/demo_advanced.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_parse/blob/main/examples/parse/demo_advanced.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"\n",
|
||||
"This notebook is a complete walkthrough for using LlamaParse with advanced indexing/retrieval techniques in LlamaIndex over the Apple 10K Filing. \n",
|
||||
"\n",
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"source": [
|
||||
"# RAG with Excel Spreadsheet using LlamaPrase\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/demo_excel.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/parse/demo_excel.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"\n",
|
||||
"This notebook shows you using LlamaParse with Excel Spreadsheet.\n",
|
||||
"\n",
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
"source": [
|
||||
"# Download Charts\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/demo_get_charts.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/parse/demo_get_charts.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"\n",
|
||||
"This notebook demonstrates how to download charts from a document using the result object.\n",
|
||||
"\n",
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"source": [
|
||||
"# LlamaParse - Fast checking Insurance Contract for Coverage\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/demo_insurance.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/parse/demo_insurance.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"\n",
|
||||
"In this notebook we will look at how LlamaParse can be used to extract structured coverage information from an insurance policy.\n",
|
||||
"\n",
|
||||
@@ -36,7 +36,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Download an insurance policy fron IRDAI\n",
|
||||
"## Download an insurance policy from IRDAI\n",
|
||||
"\n",
|
||||
"The Insurance Regulatory and Development Authority of India (IRDAI) maintains a great resource: https://policyholder.gov.in/web/guest/non-life-insurance-products where all insurance policies available in India are publicly available for download! Let's download a complex health insurance policy as an example."
|
||||
]
|
||||
@@ -228,11 +228,11 @@
|
||||
" result_type=\"markdown\",\n",
|
||||
" system_prompt_append=\"\"\"\n",
|
||||
"This document is an insurance policy.\n",
|
||||
"When a benefits/coverage/exlusion is describe in the document ammend to it add a text in the follwing benefits string format (where coverage could be an exclusion).\n",
|
||||
"When a benefits/coverage/exlusion is describe in the document amend to it add a text in the following benefits string format (where coverage could be an exclusion).\n",
|
||||
"\n",
|
||||
"For {nameofrisk} and in this condition {whenDoesThecoverageApply} the coverage is {coverageDescription}. \n",
|
||||
" \n",
|
||||
"If the document contain a benefits TABLE that describe coverage amounts, do not ouput it as a table, but instead as a list of benefits string.\n",
|
||||
"If the document contain a benefits TABLE that describe coverage amounts, do not output it as a table, but instead as a list of benefits string.\n",
|
||||
" \n",
|
||||
"\"\"\",\n",
|
||||
").aparse(\"./policy.pdf\")\n",
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
"source": [
|
||||
"# LlamaParse `JobResult` Tour\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/demo_json.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/parse/demo_json.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"\n",
|
||||
"The `JobResult` object is the main object returned by the LlamaParse API. It contains all the information about the job, including the parsed data, metadata, and any errors.\n",
|
||||
"\n",
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
"\n",
|
||||
"LlamaParse supports users to specify a `language` parameter before uploading documents, giving users better OCR capabilities over non-English PDFs, parsing images into more accurate representations.\n",
|
||||
"\n",
|
||||
"You can specify 80+ different languages: see this file for a full list of supported languages: https://github.com/run-llama/llama_cloud_services/blob/main/llama_parse/base.py.\n",
|
||||
"You can specify 80+ different languages: see this file for a full list of supported languages: https://github.com/run-llama/llama_cloud_services/blob/main/py/llama_cloud_services/parse/base.py.\n",
|
||||
"\n",
|
||||
"This notebook shows a demo of this in action. \n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/excel/o1_excel_rag.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/parse/excel/o1_excel_rag.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -740,7 +740,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"In this example, these pages aren't going to be that different when parsed, but we can verify which pages triggered auto-made by looking at the [JSON output](https://github.com/run-llama/llama_cloud_services/blob/main/examples/demo_json_tour.ipynb) of LlamaParse:"
|
||||
"In this example, these pages aren't going to be that different when parsed, but we can verify which pages triggered auto-made by looking at the [JSON output](https://github.com/run-llama/llama_cloud_services/blob/main/examples/parse/demo_json_tour.ipynb) of LlamaParse:"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -86,7 +86,7 @@ class AsyncAgentDataClient(Generic[AgentDataT]):
|
||||
client=llama_client,
|
||||
type=ExtractedPerson,
|
||||
collection="extracted_people",
|
||||
agent_url_id="person-extraction-agent"
|
||||
deployment_name="person-extraction-agent"
|
||||
)
|
||||
|
||||
# Create data
|
||||
@@ -109,10 +109,12 @@ class AsyncAgentDataClient(Generic[AgentDataT]):
|
||||
self,
|
||||
type: Type[AgentDataT],
|
||||
collection: str = "default",
|
||||
agent_url_id: Optional[str] = None,
|
||||
deployment_name: Optional[str] = None,
|
||||
client: Optional[AsyncLlamaCloud] = None,
|
||||
token: Optional[str] = None,
|
||||
base_url: Optional[str] = None,
|
||||
# deprecated, use deployment_name instead
|
||||
agent_url_id: Optional[str] = None,
|
||||
):
|
||||
"""
|
||||
Initialize the AsyncAgentDataClient.
|
||||
@@ -123,11 +125,11 @@ class AsyncAgentDataClient(Generic[AgentDataT]):
|
||||
collection: Named collection within the agent for organizing data.
|
||||
Defaults to "default". Collections allow logical separation of
|
||||
different data types or workflows within the same agent.
|
||||
agent_url_id: Unique identifier for the agent. This normally appears in the
|
||||
url of an agent within the llama cloud platform. If not provided,
|
||||
will attempt to use the LLAMA_DEPLOY_DEPLOYMENT_NAME environment
|
||||
variable. Data can only be added to an already existing agent in the
|
||||
platform.
|
||||
deployment_name: Unique identifier for the agent deployment. This normally
|
||||
appears in the URL of an agent within the Llama Cloud platform. If not
|
||||
provided, will attempt to use the LLAMA_DEPLOY_DEPLOYMENT_NAME
|
||||
environment variable. Data can only be added to an already existing
|
||||
agent in the platform.
|
||||
client: AsyncLlamaCloud client instance for API communication. If not provided, will
|
||||
construct one from the provided api token and base url
|
||||
token: Llama Cloud API token. Reads from LLAMA_CLOUD_API_KEY if not provided
|
||||
@@ -135,15 +137,14 @@ class AsyncAgentDataClient(Generic[AgentDataT]):
|
||||
defaults to https://api.cloud.llamaindex.ai
|
||||
|
||||
Raises:
|
||||
ValueError: If agent_url_id is not provided and the
|
||||
ValueError: If deployment_name is not provided and the
|
||||
LLAMA_DEPLOY_DEPLOYMENT_NAME environment variable is not set
|
||||
|
||||
Note:
|
||||
The client automatically applies retry logic to all API calls with
|
||||
exponential backoff for timeout, connection, and HTTP status errors.
|
||||
"""
|
||||
|
||||
self.agent_url_id = agent_url_id or get_default_agent_id()
|
||||
self.deployment_name = deployment_name or agent_url_id or get_default_agent_id()
|
||||
|
||||
self.collection = collection
|
||||
if not client:
|
||||
@@ -164,7 +165,7 @@ class AsyncAgentDataClient(Generic[AgentDataT]):
|
||||
@agent_data_retry
|
||||
async def create_item(self, data: AgentDataT) -> TypedAgentData[AgentDataT]:
|
||||
raw_data = await self.client.beta.create_agent_data(
|
||||
agent_slug=self.agent_url_id,
|
||||
deployment_name=self.deployment_name,
|
||||
collection=self.collection,
|
||||
data=data.model_dump(),
|
||||
)
|
||||
@@ -211,7 +212,7 @@ class AsyncAgentDataClient(Generic[AgentDataT]):
|
||||
include_total: Whether to include the total count in the response. Defaults to False to improve performance. It's recommended to only request on the first page.
|
||||
"""
|
||||
raw = await self.client.beta.search_agent_data_api_v_1_beta_agent_data_search_post(
|
||||
agent_slug=self.agent_url_id,
|
||||
deployment_name=self.deployment_name,
|
||||
collection=self.collection,
|
||||
filter=filter,
|
||||
order_by=order_by,
|
||||
@@ -254,7 +255,7 @@ class AsyncAgentDataClient(Generic[AgentDataT]):
|
||||
page_size: Maximum number of groups to return per page.
|
||||
"""
|
||||
raw = await self.client.beta.aggregate_agent_data_api_v_1_beta_agent_data_aggregate_post(
|
||||
agent_slug=self.agent_url_id,
|
||||
deployment_name=self.deployment_name,
|
||||
collection=self.collection,
|
||||
page_size=page_size,
|
||||
filter=filter,
|
||||
|
||||
@@ -10,7 +10,7 @@ CRUD operations, search capabilities, filtering, and aggregation functionality
|
||||
for managing agent-generated data at scale.
|
||||
|
||||
Key Concepts:
|
||||
- Agent Slug: Unique identifier for an agent instance
|
||||
- Deployment Name: Unique identifier for an agent deployment
|
||||
- Collection: Named grouping of data within an agent (defaults to "default"). Data within a collection should be of the same type.
|
||||
- Agent Data: Individual structured data records with metadata and timestamps
|
||||
|
||||
@@ -26,7 +26,7 @@ Example Usage:
|
||||
client=async_llama_cloud,
|
||||
type=Person,
|
||||
collection="people",
|
||||
agent_url_id="my-extraction-agent-xyz"
|
||||
deployment_name="my-extraction-agent-xyz"
|
||||
)
|
||||
|
||||
# Create typed data
|
||||
@@ -78,7 +78,7 @@ class TypedAgentData(BaseModel, Generic[AgentDataT]):
|
||||
|
||||
Attributes:
|
||||
id: Unique identifier for this data record
|
||||
agent_url_id: Identifier of the agent that created this data
|
||||
deployment_name: Identifier of the agent deployment that created this data
|
||||
collection: Named collection within the agent (used for organization)
|
||||
data: The actual structured data payload (typed as AgentDataT)
|
||||
created_at: Timestamp when the record was first created
|
||||
@@ -94,8 +94,8 @@ class TypedAgentData(BaseModel, Generic[AgentDataT]):
|
||||
"""
|
||||
|
||||
id: Optional[str] = Field(description="Unique identifier for this data record")
|
||||
agent_url_id: str = Field(
|
||||
description="Identifier of the agent that created this data"
|
||||
deployment_name: str = Field(
|
||||
description="Identifier of the agent deployment that created this data"
|
||||
)
|
||||
collection: Optional[str] = Field(
|
||||
description="Named collection within the agent for data organization"
|
||||
@@ -124,7 +124,7 @@ class TypedAgentData(BaseModel, Generic[AgentDataT]):
|
||||
|
||||
return cls(
|
||||
id=raw_data.id,
|
||||
agent_url_id=raw_data.agent_slug,
|
||||
deployment_name=raw_data.deployment_name,
|
||||
collection=raw_data.collection,
|
||||
data=data,
|
||||
created_at=raw_data.created_at,
|
||||
@@ -222,12 +222,16 @@ def parse_extracted_field_metadata(
|
||||
return {
|
||||
k: _parse_extracted_field_metadata_recursive(v)
|
||||
for k, v in field_metadata.items()
|
||||
if k not in _METADATA_FIELDS_SIBLING_TO_LEAF
|
||||
and k not in _ADDITIONAL_ROOT_METADATA_FIELDS
|
||||
if not _is_reasoning_field(k, v) and k not in _ADDITIONAL_ROOT_METADATA_FIELDS
|
||||
}
|
||||
|
||||
|
||||
_METADATA_FIELDS_SIBLING_TO_LEAF = {"reasoning"}
|
||||
def _is_reasoning_field(field_name: str, field_value: Any) -> bool:
|
||||
# There can either be a user specified reasoning field (from the schema), or a reasoning metadata field for the
|
||||
# dict of values
|
||||
return field_name == "reasoning" and isinstance(field_value, str)
|
||||
|
||||
|
||||
_ADDITIONAL_ROOT_METADATA_FIELDS = {"error"}
|
||||
|
||||
|
||||
@@ -257,14 +261,12 @@ def _parse_extracted_field_metadata_recursive(
|
||||
except ValidationError:
|
||||
pass
|
||||
additional_fields = {
|
||||
k: v
|
||||
for k, v in field_value.items()
|
||||
if k in _METADATA_FIELDS_SIBLING_TO_LEAF
|
||||
k: v for k, v in field_value.items() if _is_reasoning_field(k, v)
|
||||
}
|
||||
return {
|
||||
k: _parse_extracted_field_metadata_recursive(v, additional_fields)
|
||||
for k, v in field_value.items()
|
||||
if k not in _METADATA_FIELDS_SIBLING_TO_LEAF
|
||||
if not _is_reasoning_field(k, v)
|
||||
}
|
||||
elif isinstance(field_value, list):
|
||||
return [_parse_extracted_field_metadata_recursive(item) for item in field_value]
|
||||
|
||||
@@ -489,6 +489,7 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
name: str,
|
||||
project_name: str = DEFAULT_PROJECT_NAME,
|
||||
organization_id: Optional[str] = None,
|
||||
project_id: Optional[str] = None,
|
||||
api_key: Optional[str] = None,
|
||||
base_url: Optional[str] = None,
|
||||
app_url: Optional[str] = None,
|
||||
@@ -504,15 +505,15 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
app_url = app_url or os.environ.get("LLAMA_CLOUD_APP_URL", DEFAULT_APP_URL)
|
||||
client = get_client(api_key, base_url, app_url, timeout)
|
||||
|
||||
# create project if it doesn't exist
|
||||
project = client.projects.upsert_project(
|
||||
organization_id=organization_id, request=ProjectCreate(name=project_name)
|
||||
)
|
||||
if project.id is None:
|
||||
raise ValueError(f"Failed to create/get project {project_name}")
|
||||
|
||||
if verbose:
|
||||
print(f"Created project {project.id} with name {project.name}")
|
||||
if project_id is None:
|
||||
# create project if it doesn't exist
|
||||
project = client.projects.upsert_project(
|
||||
organization_id=organization_id,
|
||||
request=ProjectCreate(name=project_name),
|
||||
)
|
||||
project_id = project.id
|
||||
if verbose:
|
||||
print(f"Created project {project_id} with name {project_name}")
|
||||
|
||||
# create pipeline
|
||||
pipeline_create = PipelineCreate(
|
||||
@@ -523,7 +524,7 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
llama_parse_parameters=llama_parse_parameters or LlamaParseParameters(),
|
||||
)
|
||||
pipeline = client.pipelines.upsert_pipeline(
|
||||
project_id=project.id, request=pipeline_create
|
||||
project_id=project_id, request=pipeline_create
|
||||
)
|
||||
if pipeline.id is None:
|
||||
raise ValueError(f"Failed to create/get pipeline {name}")
|
||||
@@ -532,8 +533,7 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
|
||||
return cls(
|
||||
name,
|
||||
project_name=project.name,
|
||||
organization_id=project.organization_id,
|
||||
project_id=project_id,
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
app_url=app_url,
|
||||
@@ -606,6 +606,7 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
name: str,
|
||||
project_name: str = DEFAULT_PROJECT_NAME,
|
||||
organization_id: Optional[str] = None,
|
||||
project_id: Optional[str] = None,
|
||||
api_key: Optional[str] = None,
|
||||
base_url: Optional[str] = None,
|
||||
app_url: Optional[str] = None,
|
||||
@@ -631,6 +632,7 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
verbose=verbose,
|
||||
embedding_config=embedding_config,
|
||||
transform_config=transform_config,
|
||||
project_id=project_id,
|
||||
)
|
||||
|
||||
app_url = app_url or os.environ.get("LLAMA_CLOUD_APP_URL", DEFAULT_APP_URL)
|
||||
|
||||
@@ -420,6 +420,10 @@ class LlamaParse(BasePydanticReader):
|
||||
default=False,
|
||||
description="If set to true, the parser will extract sub-tables from the spreadsheet when possible (more than one table per sheet).",
|
||||
)
|
||||
spreadsheet_force_formula_computation: Optional[bool] = Field(
|
||||
default=False,
|
||||
description="If set to true, the parser will re-compute values for all spreadsheet cells containing formulas.",
|
||||
)
|
||||
specialized_chart_parsing_agentic: Optional[bool] = Field(
|
||||
default=False,
|
||||
description="If set to true, the parser will use a specialized agentic chart parsing model to extract data from charts. This model is able to understand the chart type and extract the data accordingly.",
|
||||
@@ -965,6 +969,11 @@ class LlamaParse(BasePydanticReader):
|
||||
if self.spreadsheet_extract_sub_tables:
|
||||
data["spreadsheet_extract_sub_tables"] = self.spreadsheet_extract_sub_tables
|
||||
|
||||
if self.spreadsheet_force_formula_computation:
|
||||
data[
|
||||
"spreadsheet_force_formula_computation"
|
||||
] = self.spreadsheet_force_formula_computation
|
||||
|
||||
if self.specialized_chart_parsing_agentic:
|
||||
data[
|
||||
"specialized_chart_parsing_agentic"
|
||||
|
||||
@@ -11,13 +11,13 @@ dev = [
|
||||
|
||||
[project]
|
||||
name = "llama-parse"
|
||||
version = "0.6.65"
|
||||
version = "0.6.69"
|
||||
description = "Parse files into RAG-Optimized formats."
|
||||
authors = [{name = "Logan Markewich", email = "logan@llamaindex.ai"}]
|
||||
requires-python = ">=3.9,<4.0"
|
||||
readme = "README.md"
|
||||
license = "MIT"
|
||||
dependencies = ["llama-cloud-services>=0.6.64"]
|
||||
dependencies = ["llama-cloud-services>=0.6.69"]
|
||||
|
||||
[project.scripts]
|
||||
llama-parse = "llama_parse.cli.main:parse"
|
||||
|
||||
+2
-2
@@ -19,7 +19,7 @@ dev = [
|
||||
|
||||
[project]
|
||||
name = "llama-cloud-services"
|
||||
version = "0.6.65"
|
||||
version = "0.6.69"
|
||||
description = "Tailored SDK clients for LlamaCloud services."
|
||||
authors = [{name = "Logan Markewich", email = "logan@runllama.ai"}]
|
||||
requires-python = ">=3.9,<4.0"
|
||||
@@ -27,7 +27,7 @@ readme = "README.md"
|
||||
license = "MIT"
|
||||
dependencies = [
|
||||
"llama-index-core>=0.12.0",
|
||||
"llama-cloud==0.1.41",
|
||||
"llama-cloud==0.1.42",
|
||||
"pydantic>=2.8,!=2.10",
|
||||
"click>=8.1.7,<9",
|
||||
"python-dotenv>=1.0.1,<2",
|
||||
|
||||
@@ -68,7 +68,7 @@ async def test_agent_data_crud_operations():
|
||||
client=client,
|
||||
type=ExampleData,
|
||||
collection=f"test-collection-{test_id[:8]}",
|
||||
agent_url_id=LLAMA_DEPLOY_DEPLOYMENT_NAME,
|
||||
deployment_name=LLAMA_DEPLOY_DEPLOYMENT_NAME,
|
||||
)
|
||||
|
||||
# Create test data
|
||||
|
||||
@@ -38,7 +38,7 @@ def test_typed_agent_data_from_raw():
|
||||
"""Test TypedAgentData.from_raw class method."""
|
||||
raw_data = AgentData(
|
||||
id="456",
|
||||
agent_slug="extraction-agent",
|
||||
deployment_name="extraction-agent",
|
||||
collection="employees",
|
||||
data={"name": "Jane Smith", "age": 25, "email": "jane@company.com"},
|
||||
created_at=datetime.now(),
|
||||
@@ -48,7 +48,7 @@ def test_typed_agent_data_from_raw():
|
||||
typed_data = TypedAgentData.from_raw(raw_data, Person)
|
||||
|
||||
assert typed_data.id == "456"
|
||||
assert typed_data.agent_url_id == "extraction-agent"
|
||||
assert typed_data.deployment_name == "extraction-agent"
|
||||
assert typed_data.collection == "employees"
|
||||
assert typed_data.data.name == "Jane Smith"
|
||||
assert typed_data.data.age == 25
|
||||
@@ -59,7 +59,7 @@ def test_typed_agent_data_from_raw_validation_error():
|
||||
"""Test TypedAgentData.from_raw with invalid data."""
|
||||
raw_data = AgentData(
|
||||
id="789",
|
||||
agent_slug="test-agent",
|
||||
deployment_name="test-agent",
|
||||
collection="people",
|
||||
data={"name": "Invalid Person", "age": "not_a_number"}, # Invalid age
|
||||
created_at=datetime.now(),
|
||||
@@ -613,3 +613,51 @@ def test_parses_field_metadata_with_error_field():
|
||||
}
|
||||
assert parsed.metadata.get("field_errors") == "This is an error"
|
||||
assert parsed.metadata.get("job_id") == "job-123"
|
||||
|
||||
|
||||
REASONING_IN_SCHEMA = {
|
||||
"majority_opinion": {
|
||||
"type": {
|
||||
"citation": [
|
||||
{
|
||||
"page": 4,
|
||||
"matching_text": "BARRETT, J., delivered the opinion for a unanimous Court.",
|
||||
},
|
||||
{"page": 11, "matching_text": "Opinion of the Court"},
|
||||
],
|
||||
"parsing_confidence": 1.0,
|
||||
"extraction_confidence": 0.9999998919950147,
|
||||
"confidence": 0.9999998919950147,
|
||||
},
|
||||
"reasoning": {
|
||||
"citation": [
|
||||
{
|
||||
"page": 15,
|
||||
"matching_text": "We hold that §5110(b)(1) is not subject to equitable tolling and affirm the judg...",
|
||||
}
|
||||
],
|
||||
"parsing_confidence": 1.0,
|
||||
"extraction_confidence": 0.414292785946868,
|
||||
"confidence": 0.414292785946868,
|
||||
},
|
||||
},
|
||||
"reasoning": {
|
||||
"citation": [
|
||||
{
|
||||
"page": 15,
|
||||
"matching_text": "We hold that §5110(b)(1) is not subject to equitable tolling and affirm the judg...",
|
||||
}
|
||||
],
|
||||
"parsing_confidence": 1.0,
|
||||
"extraction_confidence": 0.414292785946868,
|
||||
"confidence": 0.414292785946868,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_field_conflict_in_schema():
|
||||
extracted = parse_extracted_field_metadata(REASONING_IN_SCHEMA)
|
||||
assert isinstance(extracted["reasoning"], ExtractedFieldMetadata)
|
||||
assert isinstance(
|
||||
extracted["majority_opinion"]["reasoning"], ExtractedFieldMetadata
|
||||
)
|
||||
|
||||
@@ -1,16 +1,155 @@
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import llama_cloud_services.index.base as base
|
||||
from llama_cloud import (
|
||||
PipelineEmbeddingConfig_ManagedOpenaiEmbedding,
|
||||
Project,
|
||||
Pipeline,
|
||||
CloudDocument,
|
||||
)
|
||||
from llama_index.core.constants import DEFAULT_PROJECT_NAME
|
||||
from llama_index.core.indices.managed.base import BaseManagedIndex
|
||||
from llama_cloud_services.index import (
|
||||
LlamaCloudIndex,
|
||||
from llama_index.core.schema import Document
|
||||
from llama_cloud_services.index import LlamaCloudIndex
|
||||
|
||||
|
||||
# Simple test data as values, not fixtures
|
||||
TEST_PROJECT = Project(id="proj-123", name="test-project", organization_id="org-123")
|
||||
|
||||
EMBEDDING_CONFIG = PipelineEmbeddingConfig_ManagedOpenaiEmbedding(
|
||||
type="MANAGED_OPENAI_EMBEDDING"
|
||||
)
|
||||
TEST_PIPELINE = Pipeline(
|
||||
id="pipe-456",
|
||||
name="test-pipeline",
|
||||
project_id="proj-123",
|
||||
embedding_config=PipelineEmbeddingConfig_ManagedOpenaiEmbedding(
|
||||
type="MANAGED_OPENAI_EMBEDDING"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_class():
|
||||
@pytest.fixture
|
||||
def mock_client() -> MagicMock:
|
||||
"""Mock client with sensible defaults."""
|
||||
client = MagicMock()
|
||||
client.projects.upsert_project.return_value = Project(
|
||||
id="default-proj", name=DEFAULT_PROJECT_NAME, organization_id="default-org"
|
||||
)
|
||||
client.pipelines.upsert_pipeline.return_value = Pipeline(
|
||||
id="default-pipe",
|
||||
name="default",
|
||||
project_id="default-proj",
|
||||
embedding_config=EMBEDDING_CONFIG,
|
||||
)
|
||||
client.pipelines.upsert_batch_pipeline_documents.return_value = [
|
||||
CloudDocument(id="doc-1", text="test", metadata={})
|
||||
]
|
||||
return client
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def base_patches(mock_client: MagicMock) -> None:
|
||||
"""Auto-applied patches for all tests."""
|
||||
with (
|
||||
patch.object(base, "get_client", return_value=mock_client),
|
||||
patch.object(
|
||||
base,
|
||||
"resolve_project_and_pipeline",
|
||||
return_value=(TEST_PROJECT, TEST_PIPELINE),
|
||||
),
|
||||
patch.object(base.LlamaCloudIndex, "wait_for_completion"),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
def test_class() -> None:
|
||||
names_of_base_classes = [b.__name__ for b in LlamaCloudIndex.__mro__]
|
||||
assert BaseManagedIndex.__name__ in names_of_base_classes
|
||||
|
||||
|
||||
def test_conflicting_index_identifiers():
|
||||
def test_conflicting_index_identifiers() -> None:
|
||||
with pytest.raises(ValueError):
|
||||
LlamaCloudIndex(name="test", pipeline_id="test", index_id="test")
|
||||
|
||||
|
||||
def test_from_documents_uses_provided_project_id(mock_client: MagicMock) -> None:
|
||||
provided_project_id = "proj-123"
|
||||
organization_id = "org-abc"
|
||||
index_name = "my_new_index"
|
||||
|
||||
# Override resolve to return project with provided ID
|
||||
test_project = Project(
|
||||
id=provided_project_id, name="my_project", organization_id=organization_id
|
||||
)
|
||||
test_pipeline = Pipeline(
|
||||
id="pipe-xyz",
|
||||
name=index_name,
|
||||
project_id=provided_project_id,
|
||||
embedding_config=EMBEDDING_CONFIG,
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
base, "resolve_project_and_pipeline", return_value=(test_project, test_pipeline)
|
||||
):
|
||||
docs = [Document(text="hello")]
|
||||
index = LlamaCloudIndex.from_documents(
|
||||
documents=docs,
|
||||
name=index_name,
|
||||
project_id=provided_project_id,
|
||||
)
|
||||
|
||||
# Assert - project upsert not called; pipeline uses provided project_id
|
||||
mock_client.projects.upsert_project.assert_not_called()
|
||||
assert mock_client.pipelines.upsert_pipeline.call_count == 1
|
||||
assert (
|
||||
mock_client.pipelines.upsert_pipeline.call_args.kwargs["project_id"]
|
||||
== provided_project_id
|
||||
)
|
||||
assert index.project.id == provided_project_id
|
||||
|
||||
|
||||
def test_from_documents_upserts_project_when_project_id_missing(
|
||||
mock_client: MagicMock,
|
||||
) -> None:
|
||||
organization_id = "org-xyz"
|
||||
index_name = "my_new_index"
|
||||
|
||||
# Project is created when project_id is not provided
|
||||
upserted_project = Project(
|
||||
id="proj-999", name=DEFAULT_PROJECT_NAME, organization_id=organization_id
|
||||
)
|
||||
mock_client.projects.upsert_project.return_value = upserted_project
|
||||
|
||||
test_pipeline = Pipeline(
|
||||
id="pipe-xyz",
|
||||
name=index_name,
|
||||
project_id=upserted_project.id,
|
||||
embedding_config=EMBEDDING_CONFIG,
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
base,
|
||||
"resolve_project_and_pipeline",
|
||||
return_value=(upserted_project, test_pipeline),
|
||||
):
|
||||
docs = [Document(text="world")]
|
||||
index = LlamaCloudIndex.from_documents(
|
||||
documents=docs,
|
||||
name=index_name,
|
||||
organization_id=organization_id,
|
||||
)
|
||||
|
||||
# Assert - project was upserted with org id and default project name
|
||||
mock_client.projects.upsert_project.assert_called_once()
|
||||
kwargs = mock_client.projects.upsert_project.call_args.kwargs
|
||||
assert kwargs["organization_id"] == organization_id
|
||||
assert kwargs["request"].name == DEFAULT_PROJECT_NAME
|
||||
|
||||
# Pipeline created under the upserted project id
|
||||
assert (
|
||||
mock_client.pipelines.upsert_pipeline.call_args.kwargs["project_id"]
|
||||
== upserted_project.id
|
||||
)
|
||||
assert index.project.id == upserted_project.id
|
||||
|
||||
Generated
+5
-5
@@ -1582,21 +1582,21 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "llama-cloud"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "certifi" },
|
||||
{ name = "httpx" },
|
||||
{ name = "pydantic" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/62/6c/b2e84eebed376aea34c446cab745da5fc4e9dc53309180672299083219d5/llama_cloud-0.1.41.tar.gz", hash = "sha256:dcb741b779e3e740cd64928cfffc8ef70ed0e9bae9ef26acbe1d7e32aa737bdc", size = 109854, upload-time = "2025-09-05T22:45:13.069Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/21/04/ae0694b582d6aab4d6e7957febb7bff048897ac231ad80ba1bd71547d944/llama_cloud-0.1.42.tar.gz", hash = "sha256:485aa0e364ea648e3aaa3b2c54af7bcb6f2242c50b4f86ec022e137413fff464", size = 112480, upload-time = "2025-09-16T20:25:42.631Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/1e/4d/f0af76b389310840ce3483a92560a152025b0eefe4eee0c81102bf3317e6/llama_cloud-0.1.41-py3-none-any.whl", hash = "sha256:c847f288f0d3f4b23f47345088006deae5f2cf3f223ac1819d4c1531e9aaa13e", size = 307646, upload-time = "2025-09-05T22:45:11.597Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6a/61/85d115699a59d03f0783e119aaf6d534fca95dbe1a4531a8056e6a4774ed/llama_cloud-0.1.42-py3-none-any.whl", hash = "sha256:4ed3edde4a277ff52eeb831188c8476eb079b5e4605ad3142157a0f054b27d96", size = 311857, upload-time = "2025-09-16T20:25:41.479Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "llama-cloud-services"
|
||||
version = "0.6.65"
|
||||
version = "0.6.68"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
|
||||
@@ -1631,7 +1631,7 @@ dev = [
|
||||
requires-dist = [
|
||||
{ name = "click", specifier = ">=8.1.7,<9" },
|
||||
{ name = "eval-type-backport", marker = "python_full_version < '3.10'", specifier = ">=0.2.0,<0.3" },
|
||||
{ name = "llama-cloud", specifier = "==0.1.41" },
|
||||
{ name = "llama-cloud", specifier = "==0.1.42" },
|
||||
{ name = "llama-index-core", specifier = ">=0.12.0" },
|
||||
{ name = "packaging", specifier = ">=25.0" },
|
||||
{ name = "platformdirs", specifier = ">=4.3.7,<5" },
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "llama-cloud-services",
|
||||
"version": "0.3.5",
|
||||
"version": "0.3.6",
|
||||
"type": "module",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
|
||||
@@ -25,20 +25,23 @@ import type {
|
||||
export class AgentClient<T = unknown> {
|
||||
private client: ReturnType<typeof createClient>;
|
||||
private collection: string;
|
||||
private agentUrlId: string;
|
||||
private deploymentName: string;
|
||||
|
||||
constructor({
|
||||
client = defaultClient,
|
||||
collection = "default",
|
||||
agentUrlId = "_public",
|
||||
deploymentName = "_public",
|
||||
agentUrlId,
|
||||
}: {
|
||||
client?: ReturnType<typeof createClient>;
|
||||
collection?: string;
|
||||
deploymentName?: string;
|
||||
// deprecated, use deploymentName instead
|
||||
agentUrlId?: string;
|
||||
}) {
|
||||
this.client = client;
|
||||
this.collection = collection;
|
||||
this.agentUrlId = agentUrlId;
|
||||
this.deploymentName = agentUrlId || deploymentName;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -48,7 +51,7 @@ export class AgentClient<T = unknown> {
|
||||
const response = await createAgentDataApiV1BetaAgentDataPost({
|
||||
throwOnError: true,
|
||||
body: {
|
||||
agent_slug: this.agentUrlId,
|
||||
deployment_name: this.deploymentName,
|
||||
collection: this.collection,
|
||||
data: data as Record<string, unknown>,
|
||||
},
|
||||
@@ -118,7 +121,7 @@ export class AgentClient<T = unknown> {
|
||||
const response = await searchAgentDataApiV1BetaAgentDataSearchPost({
|
||||
throwOnError: true,
|
||||
body: {
|
||||
agent_slug: this.agentUrlId,
|
||||
deployment_name: this.deploymentName,
|
||||
...(this.collection !== undefined && {
|
||||
collection: this.collection,
|
||||
}),
|
||||
@@ -165,7 +168,7 @@ export class AgentClient<T = unknown> {
|
||||
const response = await aggregateAgentDataApiV1BetaAgentDataAggregatePost({
|
||||
throwOnError: true,
|
||||
body: {
|
||||
agent_slug: this.agentUrlId,
|
||||
deployment_name: this.deploymentName,
|
||||
...(this.collection !== undefined && {
|
||||
collection: this.collection,
|
||||
}),
|
||||
@@ -209,7 +212,7 @@ export class AgentClient<T = unknown> {
|
||||
private transformResponse(data: AgentData): TypedAgentData<T> {
|
||||
const result: TypedAgentData<T> = {
|
||||
id: data.id!,
|
||||
agentUrlId: data.agent_slug,
|
||||
deploymentName: data.deployment_name,
|
||||
data: data.data as T,
|
||||
createdAt: new Date(data.created_at!),
|
||||
updatedAt: new Date(data.updated_at!),
|
||||
@@ -250,10 +253,10 @@ export interface AgentDataClientOptions {
|
||||
/** Base URL for the client */
|
||||
/** Base URL of the llama cloud api */
|
||||
baseUrl?: string;
|
||||
/** If running in an agent runtime, optionally provide the window url to infer the agent url id */
|
||||
/** If running in an agent runtime, optionally provide the window url to infer the deployment name */
|
||||
windowUrl?: string;
|
||||
/** Agent URL ID for the client, if not provided, it will be inferred from the window url, or fall back to "default" */
|
||||
agentUrlId?: string;
|
||||
/** Deployment name for the client, if not provided, it will be inferred from the window url, or fall back to "default" */
|
||||
deploymentName?: string;
|
||||
/** Collection name for the client, defaults to "default" */
|
||||
collection?: string;
|
||||
}
|
||||
@@ -267,22 +270,25 @@ export function createAgentDataClient<T = unknown>({
|
||||
client = defaultClient,
|
||||
windowUrl,
|
||||
env,
|
||||
deploymentName,
|
||||
agentUrlId,
|
||||
collection = "default",
|
||||
}: {
|
||||
client?: ReturnType<typeof createClient>;
|
||||
windowUrl?: string;
|
||||
env?: Record<string, string>;
|
||||
deploymentName?: string;
|
||||
// deprecated, use deploymentName instead
|
||||
agentUrlId?: string;
|
||||
collection?: string;
|
||||
} = {}): AgentClient<T> {
|
||||
if (env && !agentUrlId) {
|
||||
agentUrlId =
|
||||
if (env && !deploymentName) {
|
||||
deploymentName =
|
||||
env.LLAMA_DEPLOY_DEPLOYMENT_NAME ||
|
||||
env.NEXT_PUBLIC_LLAMA_DEPLOY_DEPLOYMENT_NAME ||
|
||||
env.VITE_LLAMA_DEPLOY_DEPLOYMENT_NAME;
|
||||
}
|
||||
if (windowUrl && !agentUrlId) {
|
||||
if (windowUrl && !deploymentName) {
|
||||
try {
|
||||
const url = new URL(windowUrl);
|
||||
const path = url.pathname;
|
||||
@@ -291,17 +297,18 @@ export function createAgentDataClient<T = unknown>({
|
||||
url.hostname.includes("127.0.0.1");
|
||||
if (path.startsWith("/deployments/") && !isLocalhost) {
|
||||
// /deployments/<agent-url-id>/ui/ -> ["", "deployments", "<agent-url-id>", "ui"]
|
||||
agentUrlId = path.split("/")[2];
|
||||
deploymentName = path.split("/")[2];
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
"Failed to infer agent url id from window url, falling back to default",
|
||||
"Failed to infer deployment name from window url, falling back to default",
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return new AgentClient({
|
||||
...(deploymentName && { deploymentName }),
|
||||
...(agentUrlId && { agentUrlId }),
|
||||
collection,
|
||||
client,
|
||||
|
||||
@@ -87,8 +87,8 @@ export interface ExtractedData<T = unknown> {
|
||||
export interface TypedAgentData<T = unknown> {
|
||||
/** The unique ID of the agent data record. */
|
||||
id: string;
|
||||
/** The ID of the agent that created the data. */
|
||||
agentUrlId: string;
|
||||
/** The deployment name of the agent that created the data. */
|
||||
deploymentName: string;
|
||||
/** The collection of the agent data. */
|
||||
collection?: string;
|
||||
/** The data of the agent data. Usually an ExtractedData<SomeOtherType> */
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user