mirror of
https://github.com/run-llama/llama_cloud_services.git
synced 2026-07-21 20:15:23 -04:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 21ddd8d886 | |||
| db88b4db56 | |||
| da0b516135 | |||
| 77f339bd23 | |||
| 2c5c793c69 |
@@ -12,7 +12,6 @@ env:
|
||||
jobs:
|
||||
test_e2e:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
# You can use PyPy versions in python-version.
|
||||
# For example, pypy-2.7 and pypy-3.8
|
||||
|
||||
@@ -280,7 +280,7 @@
|
||||
"source": [
|
||||
"## Phase 2: Document Classification\n",
|
||||
"\n",
|
||||
"Next, let's classify our documents based on their content using `LlamaClassify`."
|
||||
"Next, let's classify our documents based on their content using the ClassifyClient."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -298,14 +298,14 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from llama_cloud_services.beta.classifier.client import LlamaClassify\n",
|
||||
"from llama_cloud_services.beta.classifier.client import ClassifyClient\n",
|
||||
"from llama_cloud.types import ClassifierRule\n",
|
||||
"from llama_cloud_services.files.client import FileClient\n",
|
||||
"from llama_cloud.client import AsyncLlamaCloud\n",
|
||||
"\n",
|
||||
"# Initialize the classify client\n",
|
||||
"api_key = os.environ[\"LLAMA_CLOUD_API_KEY\"]\n",
|
||||
"classify_client = LlamaClassify.from_api_key(api_key)\n",
|
||||
"classify_client = ClassifyClient.from_api_key(api_key)\n",
|
||||
"\n",
|
||||
"print(\"🏷️ Setting up document classification...\")\n",
|
||||
"\n",
|
||||
@@ -1097,7 +1097,7 @@
|
||||
" - Preserves document structure and formatting\n",
|
||||
" - Handles various file types (PDF, DOCX, etc.)\n",
|
||||
"\n",
|
||||
"2. **LlamaClassify** (`llama_cloud_services.beta.classifier.client.LlamaClassify`):\n",
|
||||
"2. **ClassifyClient** (`llama_cloud_services.beta.classifier.client.ClassifyClient`):\n",
|
||||
" - Automatically categorizes documents based on content\n",
|
||||
" - Uses customizable rules for classification\n",
|
||||
" - Provides confidence scores for classifications\n",
|
||||
|
||||
@@ -24,8 +24,8 @@ from workflows import Context
|
||||
|
||||
dotenv.load_dotenv()
|
||||
|
||||
# Global context for executed code
|
||||
_code_context: Dict[str, Any] = {}
|
||||
# Global context for loaded dataframes
|
||||
_dataframe_context: Dict[str, Any] = {}
|
||||
|
||||
|
||||
# Helper function for initial agent context
|
||||
@@ -79,30 +79,36 @@ def list_extracted_data(data_dir: str = "data") -> str:
|
||||
|
||||
|
||||
# Agent tool for code execution against dataframes
|
||||
def execute_code(code: str) -> str:
|
||||
def execute_dataframe_code(
|
||||
code: str, load_files: Optional[Dict[str, str]] = None
|
||||
) -> str:
|
||||
"""
|
||||
Execute Python pandas code against LlamaSheets extracted data.
|
||||
Execute Python pandas code against LlamaSheets extracted data.
|
||||
|
||||
This tool allows flexible data analysis by executing arbitrary pandas code.
|
||||
You can load parquet files, manipulate dataframes, and return results.
|
||||
This tool allows flexible data analysis by executing arbitrary pandas code.
|
||||
You can load parquet files, manipulate dataframes, and return results.
|
||||
|
||||
The code executes in a context where:
|
||||
- pandas is available as 'pd'
|
||||
- json is available for formatting output
|
||||
The code executes in a context where:
|
||||
- pandas is available as 'pd'
|
||||
- json is available for formatting output
|
||||
- Previously loaded dataframes are accessible by their variable names
|
||||
|
||||
Args:
|
||||
code: Python code to execute. Any print() statements or stdout/stderr
|
||||
will be captured and returned. Optionally set a 'result' variable
|
||||
for structured output.
|
||||
Args:
|
||||
code: Python code to execute. Any print() statements or stdout/stderr
|
||||
will be captured and returned. Optionally set a 'result' variable
|
||||
for structured output.
|
||||
load_files: Optional dict mapping variable names to file paths to load
|
||||
Example: {"df": "data/sales_region_1.parquet",
|
||||
"meta": "data/sales_metadata_1.parquet"}
|
||||
|
||||
Returns:
|
||||
String containing:
|
||||
- Any stdout/stderr output from the code execution
|
||||
- The 'result' variable if it was set (formatted appropriately)
|
||||
- Error message if execution failed
|
||||
Returns:
|
||||
String containing:
|
||||
- Any stdout/stderr output from the code execution
|
||||
- The 'result' variable if it was set (formatted appropriately)
|
||||
- Error message if execution failed
|
||||
|
||||
Example usage:
|
||||
code = '''
|
||||
Example usage:
|
||||
code = '''
|
||||
# Load and inspect data
|
||||
df = pd.read_parquet("data/sales_region_1.parquet")
|
||||
print(f"Loaded {len(df)} rows")
|
||||
@@ -112,9 +118,9 @@ def execute_code(code: str) -> str:
|
||||
"columns": list(df.columns),
|
||||
"sample": df.head(3).to_dict(orient="records")
|
||||
}
|
||||
'''
|
||||
'''
|
||||
"""
|
||||
global _code_context
|
||||
global _dataframe_context
|
||||
|
||||
# Capture stdout and stderr
|
||||
stdout_capture = io.StringIO()
|
||||
@@ -132,17 +138,24 @@ def execute_code(code: str) -> str:
|
||||
"pd": pd,
|
||||
"json": json,
|
||||
"Path": Path,
|
||||
**_code_context, # Include previously loaded dataframes
|
||||
**_dataframe_context, # Include previously loaded dataframes
|
||||
}
|
||||
|
||||
# Load any requested files into context
|
||||
if load_files:
|
||||
for var_name, file_path in load_files.items():
|
||||
if file_path.endswith(".parquet"):
|
||||
exec_context[var_name] = pd.read_parquet(file_path)
|
||||
# Also save to global context for future calls
|
||||
_dataframe_context[var_name] = exec_context[var_name]
|
||||
elif file_path.endswith(".json"):
|
||||
with open(file_path, "r") as f:
|
||||
exec_context[var_name] = json.load(f)
|
||||
_dataframe_context[var_name] = exec_context[var_name]
|
||||
|
||||
# Execute the code
|
||||
exec(code, exec_context)
|
||||
|
||||
# Update global context with any new variables (excluding built-ins and modules)
|
||||
for key, value in exec_context.items():
|
||||
if not key.startswith("_") and key not in ["pd", "json", "Path"]:
|
||||
_code_context[key] = value
|
||||
|
||||
# Restore stdout/stderr
|
||||
sys.stdout = old_stdout
|
||||
sys.stderr = old_stderr
|
||||
@@ -210,8 +223,8 @@ def create_llamasheets_agent(
|
||||
# Initialize LLM
|
||||
llm = OpenAI(model=llm_model, api_key=api_key)
|
||||
|
||||
# Create tools list
|
||||
tools = [execute_code]
|
||||
# Create tools - just 4 simple but powerful tools
|
||||
tools = [execute_dataframe_code]
|
||||
|
||||
# System prompt to guide the agent
|
||||
available_regions = list_extracted_data()
|
||||
@@ -225,8 +238,11 @@ LlamaSheets extracts messy spreadsheets into clean parquet files with two types
|
||||
- Type detection: data_type, is_date_like, is_percentage, is_currency
|
||||
- Layout: is_in_first_row, is_merged_cell, horizontal_alignment
|
||||
|
||||
You have access to tools that allow you to execute Python pandas code against these files.
|
||||
Use these tools to load the parquet files, analyze the data, and return results.
|
||||
Your approach:
|
||||
1. Use list_extracted_data() to discover available files
|
||||
2. Use execute_dataframe_code() to load and analyze data with pandas
|
||||
3. Use metadata to understand structure (bold = headers, colors = groups)
|
||||
4. Use save_dataframe() to export results
|
||||
|
||||
Key tips:
|
||||
- Bold cells in metadata often indicate headers
|
||||
@@ -283,7 +299,7 @@ async def main():
|
||||
print(ev.delta, end="", flush=True)
|
||||
|
||||
_ = await handler
|
||||
print("\n=== End Query ===\n")
|
||||
print("=== End Query ===\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,23 +1,5 @@
|
||||
# llama-cloud-services-py
|
||||
|
||||
## 0.6.82
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- bfaec79: Update for new page number params
|
||||
|
||||
## 0.6.81
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- f3233de: Propagate retrieval metadata to retriever nodes
|
||||
|
||||
## 0.6.80
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 0506c88: Moved ClassifyClient to LlamaClassify (backward compatible)
|
||||
|
||||
## 0.6.79
|
||||
|
||||
### Patch Changes
|
||||
|
||||
+1
-1
@@ -15,4 +15,4 @@ test: ## Run unit tests via pytest
|
||||
|
||||
.PHONY: e2e
|
||||
e2e: ## Run all tests. Run with high parallelism using xdist since tests are bottlenecked bound by the slow backend parsing
|
||||
uv run pytest -v -n 32 --timeout=300 --session-timeout=1740 tests/
|
||||
uv run pytest -v -n 32 tests/
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
from llama_cloud_services.beta.classifier.client import LlamaClassify, ClassifyClient
|
||||
from llama_cloud_services.beta.classifier.client import ClassifyClient
|
||||
from llama_cloud_services.beta.classifier.types import ClassifyJobResultsWithFiles
|
||||
from llama_cloud_services.utils import SourceText, FileInput
|
||||
|
||||
__all__ = [
|
||||
"LlamaClassify",
|
||||
"ClassifyClient",
|
||||
"ClassifyJobResultsWithFiles",
|
||||
"SourceText",
|
||||
|
||||
@@ -31,7 +31,7 @@ class ClassificationOutput(BaseModel):
|
||||
classification: str
|
||||
|
||||
|
||||
class LlamaClassify:
|
||||
class ClassifyClient:
|
||||
"""
|
||||
Experimental - Client for interacting with the LlamaCloud Classifier API.
|
||||
The Classification API is currently in beta and may change in the future without notice.
|
||||
@@ -366,6 +366,3 @@ class LlamaClassify:
|
||||
job_id, project_id=self.project_id
|
||||
)
|
||||
return job
|
||||
|
||||
|
||||
ClassifyClient = LlamaClassify
|
||||
|
||||
@@ -258,7 +258,6 @@ def page_screenshot_nodes_to_node_with_score(
|
||||
client: LlamaCloud,
|
||||
raw_image_nodes: Optional[List[PageScreenshotNodeWithScore]],
|
||||
project_id: str,
|
||||
metadata: Optional[dict] = None,
|
||||
) -> List[NodeWithScore]:
|
||||
if not raw_image_nodes:
|
||||
return []
|
||||
@@ -274,7 +273,6 @@ def page_screenshot_nodes_to_node_with_score(
|
||||
image_base64 = base64.b64encode(image_bytes).decode("utf-8")
|
||||
image_node_metadata: Dict[str, Any] = {
|
||||
**(raw_image_node.node.metadata or {}),
|
||||
**(metadata or {}),
|
||||
"file_id": raw_image_node.node.file_id,
|
||||
"page_index": raw_image_node.node.page_index,
|
||||
}
|
||||
@@ -291,7 +289,6 @@ def image_nodes_to_node_with_score(
|
||||
client: LlamaCloud,
|
||||
raw_image_nodes: Optional[List[PageScreenshotNodeWithScore]],
|
||||
project_id: str,
|
||||
metadata: Optional[dict] = None,
|
||||
) -> List[NodeWithScore]:
|
||||
"""
|
||||
Legacy method to alias page_screenshot_nodes_to_node_with_score.
|
||||
@@ -300,10 +297,7 @@ def image_nodes_to_node_with_score(
|
||||
return []
|
||||
|
||||
return page_screenshot_nodes_to_node_with_score(
|
||||
client=client,
|
||||
raw_image_nodes=raw_image_nodes,
|
||||
project_id=project_id,
|
||||
metadata=metadata,
|
||||
client=client, raw_image_nodes=raw_image_nodes, project_id=project_id
|
||||
)
|
||||
|
||||
|
||||
@@ -311,7 +305,6 @@ def page_figure_nodes_to_node_with_score(
|
||||
client: LlamaCloud,
|
||||
raw_figure_nodes: Optional[List[PageFigureNodeWithScore]],
|
||||
project_id: str,
|
||||
metadata: Optional[dict] = None,
|
||||
) -> List[NodeWithScore]:
|
||||
if not raw_figure_nodes:
|
||||
return []
|
||||
@@ -328,7 +321,6 @@ def page_figure_nodes_to_node_with_score(
|
||||
figure_base64 = base64.b64encode(figure_bytes).decode("utf-8")
|
||||
figure_node_metadata: Dict[str, Any] = {
|
||||
**(raw_figure_node.node.metadata or {}),
|
||||
**(metadata or {}),
|
||||
"file_id": raw_figure_node.node.file_id,
|
||||
"page_index": raw_figure_node.node.page_index,
|
||||
"figure_name": raw_figure_node.node.figure_name,
|
||||
@@ -345,7 +337,6 @@ async def apage_screenshot_nodes_to_node_with_score(
|
||||
client: AsyncLlamaCloud,
|
||||
raw_image_nodes: Optional[List[PageScreenshotNodeWithScore]],
|
||||
project_id: str,
|
||||
metadata: Optional[dict] = None,
|
||||
) -> List[NodeWithScore]:
|
||||
if not raw_image_nodes:
|
||||
return []
|
||||
@@ -366,7 +357,6 @@ async def apage_screenshot_nodes_to_node_with_score(
|
||||
image_base64 = base64.b64encode(image_bytes).decode("utf-8")
|
||||
image_node_metadata: Dict[str, Any] = {
|
||||
**(raw_image_node.node.metadata or {}),
|
||||
**(metadata or {}),
|
||||
"file_id": raw_image_node.node.file_id,
|
||||
"page_index": raw_image_node.node.page_index,
|
||||
}
|
||||
@@ -382,7 +372,6 @@ async def aimage_nodes_to_node_with_score(
|
||||
client: AsyncLlamaCloud,
|
||||
raw_image_nodes: Optional[List[PageScreenshotNodeWithScore]],
|
||||
project_id: str,
|
||||
metadata: Optional[dict] = None,
|
||||
) -> List[NodeWithScore]:
|
||||
"""
|
||||
Legacy method to alias apage_screenshot_nodes_to_node_with_score.
|
||||
@@ -391,10 +380,7 @@ async def aimage_nodes_to_node_with_score(
|
||||
return []
|
||||
|
||||
return await apage_screenshot_nodes_to_node_with_score(
|
||||
client=client,
|
||||
raw_image_nodes=raw_image_nodes,
|
||||
project_id=project_id,
|
||||
metadata=metadata,
|
||||
client=client, raw_image_nodes=raw_image_nodes, project_id=project_id
|
||||
)
|
||||
|
||||
|
||||
@@ -402,7 +388,6 @@ async def apage_figure_nodes_to_node_with_score(
|
||||
client: AsyncLlamaCloud,
|
||||
raw_figure_nodes: Optional[List[PageFigureNodeWithScore]],
|
||||
project_id: str,
|
||||
metadata: Optional[dict] = None,
|
||||
) -> List[NodeWithScore]:
|
||||
if not raw_figure_nodes:
|
||||
return []
|
||||
@@ -424,7 +409,6 @@ async def apage_figure_nodes_to_node_with_score(
|
||||
figure_base64 = base64.b64encode(figure_bytes).decode("utf-8")
|
||||
figure_node_metadata: Dict[str, Any] = {
|
||||
**(raw_figure_node.node.metadata or {}),
|
||||
**(metadata or {}),
|
||||
"file_id": raw_figure_node.node.file_id,
|
||||
"page_index": raw_figure_node.node.page_index,
|
||||
"figure_name": raw_figure_node.node.figure_name,
|
||||
|
||||
@@ -129,12 +129,11 @@ class LlamaCloudRetriever(BaseRetriever):
|
||||
)
|
||||
|
||||
def _result_nodes_to_node_with_score(
|
||||
self, result_nodes: List[TextNodeWithScore], metadata: Optional[dict] = None
|
||||
self, result_nodes: List[TextNodeWithScore]
|
||||
) -> List[NodeWithScore]:
|
||||
nodes = []
|
||||
for res in result_nodes:
|
||||
text_node = TextNode.model_validate(res.node.dict())
|
||||
text_node.metadata.update(metadata or {})
|
||||
text_node = TextNode.parse_obj(res.node.dict())
|
||||
nodes.append(NodeWithScore(node=text_node, score=res.score))
|
||||
|
||||
return nodes
|
||||
@@ -162,25 +161,17 @@ class LlamaCloudRetriever(BaseRetriever):
|
||||
search_filters_inference_schema=search_filters_inference_schema,
|
||||
)
|
||||
|
||||
result_nodes = self._result_nodes_to_node_with_score(
|
||||
results.retrieval_nodes, metadata=results.metadata
|
||||
)
|
||||
result_nodes = self._result_nodes_to_node_with_score(results.retrieval_nodes)
|
||||
if self._retrieve_page_screenshot_nodes:
|
||||
result_nodes.extend(
|
||||
page_screenshot_nodes_to_node_with_score(
|
||||
self._client,
|
||||
results.image_nodes,
|
||||
self.project.id,
|
||||
metadata=results.metadata,
|
||||
self._client, results.image_nodes, self.project.id
|
||||
)
|
||||
)
|
||||
if self._retrieve_page_figure_nodes:
|
||||
result_nodes.extend(
|
||||
page_figure_nodes_to_node_with_score(
|
||||
self._client,
|
||||
results.page_figure_nodes,
|
||||
self.project.id,
|
||||
metadata=results.metadata,
|
||||
self._client, results.page_figure_nodes, self.project.id
|
||||
)
|
||||
)
|
||||
|
||||
@@ -209,25 +200,17 @@ class LlamaCloudRetriever(BaseRetriever):
|
||||
search_filters_inference_schema=search_filters_inference_schema,
|
||||
)
|
||||
|
||||
result_nodes = self._result_nodes_to_node_with_score(
|
||||
results.retrieval_nodes, metadata=results.metadata
|
||||
)
|
||||
result_nodes = self._result_nodes_to_node_with_score(results.retrieval_nodes)
|
||||
if self._retrieve_page_screenshot_nodes:
|
||||
result_nodes.extend(
|
||||
await apage_screenshot_nodes_to_node_with_score(
|
||||
self._aclient,
|
||||
results.image_nodes,
|
||||
self.project.id,
|
||||
metadata=results.metadata,
|
||||
self._aclient, results.image_nodes, self.project.id
|
||||
)
|
||||
)
|
||||
if self._retrieve_page_figure_nodes:
|
||||
result_nodes.extend(
|
||||
await apage_figure_nodes_to_node_with_score(
|
||||
self._aclient,
|
||||
results.page_figure_nodes,
|
||||
self.project.id,
|
||||
metadata=results.metadata,
|
||||
self._aclient, results.page_figure_nodes, self.project.id
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -285,7 +285,7 @@ class LlamaParse(BasePydanticReader):
|
||||
description="Note: Non compatible with gpt-4o. If set to true, the parser will use a faster mode to extract text from documents. This mode will skip OCR of images, and table/heading reconstruction.",
|
||||
)
|
||||
|
||||
guess_xlsx_sheet_name: Optional[bool] = Field(
|
||||
guess_xlsx_sheet_names: Optional[bool] = Field(
|
||||
default=False,
|
||||
description="Whether to guess the sheet names of the xlsx file.",
|
||||
)
|
||||
@@ -313,10 +313,6 @@ class LlamaParse(BasePydanticReader):
|
||||
default=False,
|
||||
description="If set to true, the parser will ignore document elements for layout detection and only rely on a vision model.",
|
||||
)
|
||||
inline_images_in_markdown: Optional[bool] = Field(
|
||||
default=False,
|
||||
description="If set to true, the parser will inline images in the markdown output.",
|
||||
)
|
||||
input_s3_region: Optional[str] = Field(
|
||||
default=None,
|
||||
description="The region of the input S3 bucket if input_s3_path is specified.",
|
||||
@@ -333,10 +329,6 @@ class LlamaParse(BasePydanticReader):
|
||||
default=None,
|
||||
description="The maximum timeout in seconds to wait for the parsing to finish. Override default timeout of 30 minutes. Minimum is 120 seconds.",
|
||||
)
|
||||
keep_page_separator_when_merging_tables: Optional[bool] = Field(
|
||||
default=False,
|
||||
description="If set to true, the parser will keep the page separator when merging tables across pages.",
|
||||
)
|
||||
language: Optional[str] = Field(
|
||||
default="en", description="The language of the text to parse."
|
||||
)
|
||||
@@ -408,10 +400,6 @@ class LlamaParse(BasePydanticReader):
|
||||
default=False,
|
||||
description="If set, the parser will try to preserve very small text lines. This can be useful for documents containing vector graphics with very small text lines that may not be recognized by OCR or a vision model (such as in CAD drawings).",
|
||||
)
|
||||
presentation_out_of_bounds_content: Optional[bool] = Field(
|
||||
default=False,
|
||||
description="If set to true, the parser will include out-of-bounds content in presentation files.",
|
||||
)
|
||||
precise_bounding_box: Optional[bool] = Field(
|
||||
default=False,
|
||||
description="If set to true, the parser will use a more precise bounding box to extract text from documents. This will increase the accuracy of the parsing job, but reduce the speed.",
|
||||
@@ -428,14 +416,6 @@ class LlamaParse(BasePydanticReader):
|
||||
default=None,
|
||||
description="A suffix to add after error message in failed pages. If not set, no suffix will be used.",
|
||||
)
|
||||
remove_hidden_text: Optional[bool] = Field(
|
||||
default=False,
|
||||
description="If set to true, the parser will remove hidden text from the document.",
|
||||
)
|
||||
save_images: Optional[bool] = Field(
|
||||
default=True,
|
||||
description="If set to true, the parser will save images extracted from the document.",
|
||||
)
|
||||
skip_diagonal_text: Optional[bool] = Field(
|
||||
default=False,
|
||||
description="If set to true, the parser will ignore diagonal text (when the text rotation in degrees modulo 90 is not 0).",
|
||||
@@ -460,10 +440,6 @@ class LlamaParse(BasePydanticReader):
|
||||
default=False,
|
||||
description="If set to true, the parser will use a specialized one-shot chart parsing model to extract data from charts. This model is able to understand the chart type and extract the data accordingly. It is more accurate than the efficient model, but also more expensive.",
|
||||
)
|
||||
specialized_image_parsing: Optional[bool] = Field(
|
||||
default=False,
|
||||
description="If set to true, the parser will use a specialized image parsing model to extract data from images.",
|
||||
)
|
||||
strict_mode_buggy_font: Optional[bool] = Field(
|
||||
default=False,
|
||||
description="If set to true, the parser will fail if it can't extract text from a document because of a buggy font.",
|
||||
@@ -560,10 +536,6 @@ class LlamaParse(BasePydanticReader):
|
||||
default=None,
|
||||
description="A prefix to add to the page footer in the output markdown.",
|
||||
)
|
||||
extract_printed_page_number: Optional[bool] = Field(
|
||||
default=None,
|
||||
description="Whether to extract the printed page numbers from pages in the document.",
|
||||
)
|
||||
|
||||
# Deprecated
|
||||
bounding_box: Optional[str] = Field(
|
||||
@@ -608,23 +580,6 @@ class LlamaParse(BasePydanticReader):
|
||||
description="Automatically check for Python SDK updates.",
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def handle_deprecated_params(cls, data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
# Handle deprecated guess_xlsx_sheet_names -> guess_xlsx_sheet_name
|
||||
if "guess_xlsx_sheet_names" in data:
|
||||
warnings.warn(
|
||||
"The parameter 'guess_xlsx_sheet_names' is deprecated and will be removed in a future release. "
|
||||
"Use 'guess_xlsx_sheet_name' instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
# Only set the new parameter if it's not already explicitly set
|
||||
if "guess_xlsx_sheet_name" not in data:
|
||||
data["guess_xlsx_sheet_name"] = data["guess_xlsx_sheet_names"]
|
||||
del data["guess_xlsx_sheet_names"]
|
||||
return data
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def warn_extra_params(cls, data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
@@ -865,8 +820,8 @@ class LlamaParse(BasePydanticReader):
|
||||
)
|
||||
data["formatting_instruction"] = self.formatting_instruction
|
||||
|
||||
if self.guess_xlsx_sheet_name:
|
||||
data["guess_xlsx_sheet_name"] = self.guess_xlsx_sheet_name
|
||||
if self.guess_xlsx_sheet_names:
|
||||
data["guess_xlsx_sheet_names"] = self.guess_xlsx_sheet_names
|
||||
|
||||
if self.html_make_all_elements_visible:
|
||||
data["html_make_all_elements_visible"] = self.html_make_all_elements_visible
|
||||
@@ -890,9 +845,6 @@ class LlamaParse(BasePydanticReader):
|
||||
"ignore_document_elements_for_layout_detection"
|
||||
] = self.ignore_document_elements_for_layout_detection
|
||||
|
||||
if self.inline_images_in_markdown:
|
||||
data["inline_images_in_markdown"] = self.inline_images_in_markdown
|
||||
|
||||
if input_url is not None:
|
||||
files = None
|
||||
data["input_url"] = str(input_url)
|
||||
@@ -921,11 +873,6 @@ class LlamaParse(BasePydanticReader):
|
||||
if self.job_timeout_in_seconds is not None:
|
||||
data["job_timeout_in_seconds"] = self.job_timeout_in_seconds
|
||||
|
||||
if self.keep_page_separator_when_merging_tables:
|
||||
data[
|
||||
"keep_page_separator_when_merging_tables"
|
||||
] = self.keep_page_separator_when_merging_tables
|
||||
|
||||
if self.language:
|
||||
data["language"] = self.language
|
||||
|
||||
@@ -1004,11 +951,6 @@ class LlamaParse(BasePydanticReader):
|
||||
if self.preserve_very_small_text:
|
||||
data["preserve_very_small_text"] = self.preserve_very_small_text
|
||||
|
||||
if self.presentation_out_of_bounds_content:
|
||||
data[
|
||||
"presentation_out_of_bounds_content"
|
||||
] = self.presentation_out_of_bounds_content
|
||||
|
||||
if self.preset is not None:
|
||||
data["preset"] = self.preset
|
||||
|
||||
@@ -1028,11 +970,6 @@ class LlamaParse(BasePydanticReader):
|
||||
"replace_failed_page_with_error_message_suffix"
|
||||
] = self.replace_failed_page_with_error_message_suffix
|
||||
|
||||
if self.remove_hidden_text:
|
||||
data["remove_hidden_text"] = self.remove_hidden_text
|
||||
|
||||
data["save_images"] = self.save_images
|
||||
|
||||
if self.skip_diagonal_text:
|
||||
data["skip_diagonal_text"] = self.skip_diagonal_text
|
||||
|
||||
@@ -1057,9 +994,6 @@ class LlamaParse(BasePydanticReader):
|
||||
if self.specialized_chart_parsing_plus:
|
||||
data["specialized_chart_parsing_plus"] = self.specialized_chart_parsing_plus
|
||||
|
||||
if self.specialized_image_parsing:
|
||||
data["specialized_image_parsing"] = self.specialized_image_parsing
|
||||
|
||||
if self.strict_mode_buggy_font:
|
||||
data["strict_mode_buggy_font"] = self.strict_mode_buggy_font
|
||||
|
||||
@@ -1115,9 +1049,6 @@ class LlamaParse(BasePydanticReader):
|
||||
"markdown_table_multiline_header_separator"
|
||||
] = self.markdown_table_multiline_header_separator
|
||||
|
||||
if self.extract_printed_page_number is not None:
|
||||
data["extract_printed_page_number"] = self.extract_printed_page_number
|
||||
|
||||
# Deprecated
|
||||
if self.bounding_box is not None:
|
||||
data["bounding_box"] = self.bounding_box
|
||||
|
||||
@@ -250,19 +250,6 @@ class Page(SafeBaseModel):
|
||||
slideSpeakerNotes: Optional[str] = Field(
|
||||
default=None, description="The speaker notes for the slide."
|
||||
)
|
||||
confidence: Optional[float] = Field(
|
||||
default=None, description="The confidence of the page parsing."
|
||||
)
|
||||
printedPageNumber: Optional[str] = Field(
|
||||
default=None,
|
||||
description="The printed page number on the page, if found and extractPrintedPageNumber is set to true.",
|
||||
)
|
||||
pageHeaderMarkdown: Optional[str] = Field(
|
||||
default=None, description="The page header in markdown format."
|
||||
)
|
||||
pageFooterMarkdown: Optional[str] = Field(
|
||||
default=None, description="The page footer in markdown format."
|
||||
)
|
||||
|
||||
|
||||
class JobResult(SafeBaseModel):
|
||||
|
||||
@@ -1,26 +1,5 @@
|
||||
# llama_parse
|
||||
|
||||
## 0.6.82
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [bfaec79]
|
||||
- llama-cloud-services-py@0.6.82
|
||||
|
||||
## 0.6.81
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [f3233de]
|
||||
- llama-cloud-services-py@0.6.81
|
||||
|
||||
## 0.6.80
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [0506c88]
|
||||
- llama-cloud-services-py@0.6.80
|
||||
|
||||
## 0.6.79
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "llama_parse",
|
||||
"version": "0.6.82",
|
||||
"version": "0.6.79",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"private": false,
|
||||
|
||||
@@ -11,13 +11,13 @@ dev = [
|
||||
|
||||
[project]
|
||||
name = "llama-parse"
|
||||
version = "0.6.83"
|
||||
version = "0.6.79"
|
||||
description = "Parse files into RAG-Optimized formats."
|
||||
authors = [{name = "Logan Markewich", email = "logan@llamaindex.ai"}]
|
||||
requires-python = ">=3.9,<4.0"
|
||||
readme = "README.md"
|
||||
license = "MIT"
|
||||
dependencies = ["llama-cloud-services>=0.6.82"]
|
||||
dependencies = ["llama-cloud-services>=0.6.79"]
|
||||
|
||||
[project.scripts]
|
||||
llama-parse = "llama_parse.cli.main:parse"
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "llama-cloud-services-py",
|
||||
"version": "0.6.82",
|
||||
"version": "0.6.79",
|
||||
"private": false,
|
||||
"license": "MIT",
|
||||
"scripts": {},
|
||||
|
||||
+1
-2
@@ -7,7 +7,6 @@ dev = [
|
||||
"pytest>=8.0.0,<9",
|
||||
"pytest-xdist>=3.6.1,<4",
|
||||
"pytest-asyncio",
|
||||
"pytest-timeout>=2.3.1",
|
||||
"ipykernel>=6.29.0,<7",
|
||||
"pre-commit==3.2.0",
|
||||
"autoevals>=0.0.114,<0.0.115",
|
||||
@@ -23,7 +22,7 @@ dev = [
|
||||
|
||||
[project]
|
||||
name = "llama-cloud-services"
|
||||
version = "0.6.83"
|
||||
version = "0.6.79"
|
||||
description = "Tailored SDK clients for LlamaCloud services."
|
||||
authors = [{name = "Logan Markewich", email = "logan@runllama.ai"}]
|
||||
requires-python = ">=3.9,<4.0"
|
||||
|
||||
@@ -10,10 +10,8 @@ from llama_cloud_services.beta.sheets.types import SpreadsheetParsingConfig
|
||||
@pytest.fixture
|
||||
def sheets_client():
|
||||
"""Create a LlamaSheets client for testing."""
|
||||
api_key = os.getenv(
|
||||
"LLAMA_CLOUD_API_KEY", "llx-3AEorIw5v0lnJPzEOI9xSl0N8yFx3fguw0Zn8QJHzGWmwg5r"
|
||||
)
|
||||
base_url = os.getenv("LLAMA_CLOUD_BASE_URL", "https://api.staging.llamaindex.ai")
|
||||
api_key = os.getenv("LLAMA_CLOUD_API_KEY")
|
||||
base_url = os.getenv("LLAMA_CLOUD_BASE_URL", "https://api.cloud.llamaindex.ai")
|
||||
|
||||
client = LlamaSheets(
|
||||
api_key=api_key,
|
||||
@@ -51,10 +49,7 @@ def sample_excel_file():
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
os.environ.get(
|
||||
"LLAMA_CLOUD_API_KEY", "llx-3AEorIw5v0lnJPzEOI9xSl0N8yFx3fguw0Zn8QJHzGWmwg5r"
|
||||
)
|
||||
== "",
|
||||
os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
|
||||
reason="LLAMA_CLOUD_API_KEY not set",
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
@@ -70,30 +65,30 @@ async def test_spreadsheet_extraction_e2e(
|
||||
4. Verifies the extracted data matches the original data
|
||||
"""
|
||||
# Extract tables from the spreadsheet
|
||||
result = await sheets_client.aextract_regions(sample_excel_file)
|
||||
result = await sheets_client.aextract_tables(sample_excel_file)
|
||||
|
||||
# Verify job completed successfully
|
||||
assert result.status in ("SUCCESS", "PARTIAL_SUCCESS")
|
||||
assert result.success is True
|
||||
|
||||
# Verify we extracted at least one table
|
||||
assert len(result.regions) > 0, "Expected at least one table to be extracted"
|
||||
assert len(result.tables) > 0, "Expected at least one table to be extracted"
|
||||
|
||||
# Get the first table
|
||||
first_table = result.regions[0]
|
||||
first_table = result.tables[0]
|
||||
assert first_table.sheet_name == "TestSheet"
|
||||
|
||||
# Download the table as a DataFrame
|
||||
extracted_df = await sheets_client.adownload_region_as_dataframe(
|
||||
extracted_df = await sheets_client.adownload_table_as_dataframe(
|
||||
job_id=result.id,
|
||||
region_id=first_table.region_id,
|
||||
result_type=first_table.region_type,
|
||||
table_id=first_table.table_id,
|
||||
)
|
||||
|
||||
# Load the original dataframe for comparison
|
||||
original_df = pd.read_excel(sample_excel_file)
|
||||
|
||||
# Verify the extracted DataFrame has the expected shape
|
||||
breakpoint()
|
||||
assert extracted_df.shape[0] == original_df.shape[0], (
|
||||
f"Row count mismatch: extracted {extracted_df.shape[0]}, "
|
||||
f"original {original_df.shape[0]}"
|
||||
@@ -134,10 +129,7 @@ async def test_spreadsheet_extraction_e2e(
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
os.environ.get(
|
||||
"LLAMA_CLOUD_API_KEY", "llx-3AEorIw5v0lnJPzEOI9xSl0N8yFx3fguw0Zn8QJHzGWmwg5r"
|
||||
)
|
||||
== "",
|
||||
os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
|
||||
reason="LLAMA_CLOUD_API_KEY not set",
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
@@ -153,7 +145,7 @@ async def test_spreadsheet_extraction_with_config(
|
||||
)
|
||||
|
||||
# Extract tables with the config
|
||||
result = await sheets_client.aextract_regions(sample_excel_file, config=config)
|
||||
result = await sheets_client.aextract_tables(sample_excel_file, config=config)
|
||||
|
||||
# Verify job completed successfully
|
||||
assert result.status in ("SUCCESS", "PARTIAL_SUCCESS")
|
||||
@@ -165,7 +157,7 @@ async def test_spreadsheet_extraction_with_config(
|
||||
assert result.worksheet_metadata[0].description is not None
|
||||
|
||||
# Verify we extracted at least one table
|
||||
assert len(result.regions) > 0
|
||||
assert len(result.tables) > 0
|
||||
|
||||
# Verify the sheet name matches
|
||||
assert result.regions[0].sheet_name == "TestSheet"
|
||||
assert result.tables[0].sheet_name == "TestSheet"
|
||||
|
||||
@@ -1,17 +1,5 @@
|
||||
# llama-cloud-services
|
||||
|
||||
## 0.4.2
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- bfaec79: Update for new page number params
|
||||
|
||||
## 0.4.1
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- f3233de: Propagate retrieval metadata to retriever nodes
|
||||
|
||||
## 0.4.0
|
||||
|
||||
### Minor Changes
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "llama-cloud-services",
|
||||
"version": "0.4.2",
|
||||
"version": "0.4.0",
|
||||
"type": "module",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
|
||||
@@ -34,15 +34,12 @@ export class LlamaCloudRetriever extends BaseRetriever {
|
||||
|
||||
private resultNodesToNodeWithScore(
|
||||
nodes: TextNodeWithScore[],
|
||||
metadata: Record<string, string> | undefined,
|
||||
): NodeWithScore[] {
|
||||
return nodes.map((node: TextNodeWithScore) => {
|
||||
const textNode = jsonToNode(node.node, ObjectType.TEXT);
|
||||
const extra_metadata = metadata || {};
|
||||
textNode.metadata = {
|
||||
...textNode.metadata,
|
||||
...node.node.extra_info, // append LlamaCloud extra_info to node metadata (file_name, pipeline_id, etc.)
|
||||
...extra_metadata, // append retrieval-level metadata
|
||||
};
|
||||
return {
|
||||
// Currently LlamaCloud only supports text nodes
|
||||
@@ -66,7 +63,6 @@ export class LlamaCloudRetriever extends BaseRetriever {
|
||||
private async pageScreenshotNodesToNodeWithScore(
|
||||
nodes: PageScreenshotNodeWithScore[] | undefined,
|
||||
projectId: string,
|
||||
metadata: Record<string, string> | undefined,
|
||||
): Promise<NodeWithScore[]> {
|
||||
if (!nodes || nodes.length === 0) return [];
|
||||
|
||||
@@ -91,7 +87,6 @@ export class LlamaCloudRetriever extends BaseRetriever {
|
||||
image: base64,
|
||||
metadata: {
|
||||
...(n.node.metadata ?? {}),
|
||||
...(metadata || {}),
|
||||
file_id: n.node.file_id,
|
||||
page_index: n.node.page_index,
|
||||
},
|
||||
@@ -106,7 +101,6 @@ export class LlamaCloudRetriever extends BaseRetriever {
|
||||
private async pageFigureNodesToNodeWithScore(
|
||||
nodes: PageFigureNodeWithScore[] | undefined,
|
||||
projectId: string,
|
||||
metadata: Record<string, string> | undefined,
|
||||
): Promise<NodeWithScore[]> {
|
||||
if (!nodes || nodes.length === 0) return [];
|
||||
|
||||
@@ -132,7 +126,6 @@ export class LlamaCloudRetriever extends BaseRetriever {
|
||||
image: base64,
|
||||
metadata: {
|
||||
...(n.node.metadata ?? {}),
|
||||
...(metadata || {}),
|
||||
file_id: n.node.file_id,
|
||||
page_index: n.node.page_index,
|
||||
figure_name: n.node.figure_name,
|
||||
@@ -229,10 +222,7 @@ export class LlamaCloudRetriever extends BaseRetriever {
|
||||
},
|
||||
});
|
||||
|
||||
const textNodes = this.resultNodesToNodeWithScore(
|
||||
results.retrieval_nodes,
|
||||
results.metadata,
|
||||
);
|
||||
const textNodes = this.resultNodesToNodeWithScore(results.retrieval_nodes);
|
||||
|
||||
const needScreenshots = (this.retrieveParams as RetrievalParams)
|
||||
.retrieve_page_screenshot_nodes;
|
||||
@@ -250,14 +240,12 @@ export class LlamaCloudRetriever extends BaseRetriever {
|
||||
? this.pageScreenshotNodesToNodeWithScore(
|
||||
results.image_nodes,
|
||||
projectId,
|
||||
results.metadata,
|
||||
)
|
||||
: Promise.resolve([] as NodeWithScore[]),
|
||||
needFigures
|
||||
? this.pageFigureNodesToNodeWithScore(
|
||||
results.page_figure_nodes,
|
||||
projectId,
|
||||
results.metadata,
|
||||
)
|
||||
: Promise.resolve([] as NodeWithScore[]),
|
||||
]);
|
||||
|
||||
@@ -185,7 +185,6 @@ export class LlamaParseReader extends FileReader {
|
||||
page_footer_prefix?: string | undefined;
|
||||
page_footer_suffix?: string | undefined;
|
||||
merge_tables_across_pages_in_markdown?: boolean | undefined;
|
||||
extract_printed_page_number?: boolean | undefined;
|
||||
|
||||
constructor(
|
||||
params: Partial<Omit<LlamaParseReader, "language" | "apiKey">> & {
|
||||
@@ -382,7 +381,6 @@ export class LlamaParseReader extends FileReader {
|
||||
page_footer_suffix: this.page_footer_suffix,
|
||||
merge_tables_across_pages_in_markdown:
|
||||
this.merge_tables_across_pages_in_markdown,
|
||||
extract_printed_page_number: this.extract_printed_page_number,
|
||||
} satisfies {
|
||||
[Key in keyof BodyUploadFileApiParsingUploadPost]-?:
|
||||
| BodyUploadFileApiParsingUploadPost[Key]
|
||||
|
||||
Reference in New Issue
Block a user