mirror of
https://github.com/run-llama/llama_cloud_services.git
synced 2026-07-21 12:05:23 -04:00
Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| fd45127678 | |||
| 0506c88735 | |||
| 4bc9eb6c0d | |||
| 5a3dac655c | |||
| 519254efbe | |||
| 6ab56b79f3 | |||
| e020e3e2b1 | |||
| f293547910 |
@@ -34,7 +34,7 @@ repos:
|
||||
rev: v1.0.1
|
||||
hooks:
|
||||
- id: mypy
|
||||
exclude: ^py/tests|^py/unit_tests
|
||||
exclude: ^py/tests|^py/unit_tests|^examples
|
||||
additional_dependencies:
|
||||
[
|
||||
"types-requests",
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
"@tanstack/react-router": "^1.133.22",
|
||||
"@tanstack/react-router-devtools": "^1.133.22",
|
||||
"@tanstack/react-start": "^1.133.22",
|
||||
"llama-cloud-services": "^0.3.10",
|
||||
"llama-cloud-services": "file:../../ts/llama_cloud_services",
|
||||
"react": "^19.0.0",
|
||||
"react-dom": "^19.0.0",
|
||||
"tailwind-merge": "^2.6.0",
|
||||
|
||||
@@ -15,7 +15,7 @@ export const Route = createFileRoute('/api/classify')({
|
||||
const rawRes = await classifier.classify(
|
||||
classificationRules,
|
||||
parsingConfig,
|
||||
[new Uint8Array(buff)],
|
||||
{ fileContents: [new Uint8Array(buff)] },
|
||||
)
|
||||
const results = rawRes.items
|
||||
let classification = ""
|
||||
|
||||
@@ -280,7 +280,7 @@
|
||||
"source": [
|
||||
"## Phase 2: Document Classification\n",
|
||||
"\n",
|
||||
"Next, let's classify our documents based on their content using the ClassifyClient."
|
||||
"Next, let's classify our documents based on their content using `LlamaClassify`."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -298,14 +298,14 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from llama_cloud_services.beta.classifier.client import ClassifyClient\n",
|
||||
"from llama_cloud_services.beta.classifier.client import LlamaClassify\n",
|
||||
"from llama_cloud.types import ClassifierRule\n",
|
||||
"from llama_cloud_services.files.client import FileClient\n",
|
||||
"from llama_cloud.client import AsyncLlamaCloud\n",
|
||||
"\n",
|
||||
"# Initialize the classify client\n",
|
||||
"api_key = os.environ[\"LLAMA_CLOUD_API_KEY\"]\n",
|
||||
"classify_client = ClassifyClient.from_api_key(api_key)\n",
|
||||
"classify_client = LlamaClassify.from_api_key(api_key)\n",
|
||||
"\n",
|
||||
"print(\"🏷️ Setting up document classification...\")\n",
|
||||
"\n",
|
||||
@@ -1097,7 +1097,7 @@
|
||||
" - Preserves document structure and formatting\n",
|
||||
" - Handles various file types (PDF, DOCX, etc.)\n",
|
||||
"\n",
|
||||
"2. **ClassifyClient** (`llama_cloud_services.beta.classifier.client.ClassifyClient`):\n",
|
||||
"2. **LlamaClassify** (`llama_cloud_services.beta.classifier.client.LlamaClassify`):\n",
|
||||
" - Automatically categorizes documents based on content\n",
|
||||
" - Uses customizable rules for classification\n",
|
||||
" - Provides confidence scores for classifications\n",
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
This project uses LlamaSheets to extract data from spreadsheets for analysis.
|
||||
|
||||
## Current Project Structure
|
||||
|
||||
- `data/` - Contains extracted parquet files from LlamaSheets
|
||||
- `{name}_region_{N}.parquet` - Table data files
|
||||
- `{name}_metadata_{N}.parquet` - Cell metadata files
|
||||
- `{name}_job_metadata.json` - Extraction job information
|
||||
- `scripts/` - Analysis and helper scripts
|
||||
- `reports/` - Your generated reports and outputs
|
||||
|
||||
## Working with LlamaSheets Data
|
||||
|
||||
### Understanding the Files
|
||||
|
||||
When a spreadsheet is extracted, you'll find:
|
||||
|
||||
1. **Table parquet files** (`region_*.parquet`): The actual table data
|
||||
- Columns correspond to spreadsheet columns
|
||||
- Data types are preserved (dates, numbers, strings, booleans)
|
||||
|
||||
2. **Metadata parquet files** (`metadata_*.parquet`): Rich cell-level metadata
|
||||
- Formatting: `font_bold`, `font_italic`, `font_size`, `background_color_rgb`
|
||||
- Position: `row_number`, `column_number`, `coordinate` (e.g., "A1")
|
||||
- Type detection: `data_type`, `is_date_like`, `is_percentage`, `is_currency`
|
||||
- Layout: `is_in_first_row`, `is_merged_cell`, `horizontal_alignment`
|
||||
- Content: `cell_value`, `raw_cell_value`
|
||||
|
||||
3. **Job metadata JSON** (`job_metadata.json`): Overall extraction results
|
||||
- `regions[]`: List of extracted regions with IDs, locations, and titles/descriptions
|
||||
- `worksheet_metadata[]`: Generated titles and descriptions
|
||||
- `status`: Success/failure status
|
||||
|
||||
### Key Principles
|
||||
|
||||
1. **Use metadata to understand structure**: Bold cells often indicate headers, colors indicate groupings
|
||||
2. **Validate before analysis**: Check data types, look for missing values
|
||||
3. **Preserve formatting context**: The metadata tells you what the spreadsheet author emphasized
|
||||
4. **Save intermediate results**: Store cleaned data as new parquet files
|
||||
|
||||
### Common Patterns
|
||||
|
||||
**Loading data:**
|
||||
```python
|
||||
import pandas as pd
|
||||
|
||||
df = pd.read_parquet("data/region_1_Sheet1.parquet")
|
||||
meta_df = pd.read_parquet("data/metadata_1_Sheet1.parquet")
|
||||
```
|
||||
|
||||
**Finding headers:**
|
||||
```python
|
||||
headers = meta_df[meta_df["font_bold"] == True]["cell_value"].tolist()
|
||||
```
|
||||
|
||||
**Finding date columns:**
|
||||
```python
|
||||
date_cols = meta_df[meta_df["is_date_like"] == True]["column_number"].unique()
|
||||
```
|
||||
|
||||
## Tools Available
|
||||
|
||||
- **Python 3.11+**: For data analysis
|
||||
- **pandas**: DataFrame manipulation
|
||||
- **pyarrow**: Parquet file reading
|
||||
- **matplotlib**: Visualization (optional)
|
||||
|
||||
## Guidelines
|
||||
|
||||
- Always read the job_metadata.json first to understand what was extracted
|
||||
- Check both table data and metadata before making assumptions
|
||||
- Write reusable functions for common operations
|
||||
- Document any data quality issues discovered
|
||||
@@ -0,0 +1,278 @@
|
||||
"""
|
||||
Generate sample spreadsheets for LlamaSheets + Claude workflows.
|
||||
|
||||
This script creates example Excel files that demonstrate different use cases:
|
||||
1. Simple data table (for Workflow 1)
|
||||
2. Regional sales data (for Workflow 2)
|
||||
3. Complex budget with formatting (for Workflow 3)
|
||||
4. Weekly sales report (for Workflow 4)
|
||||
|
||||
Usage:
|
||||
python generate_sample_data.py
|
||||
"""
|
||||
|
||||
import random
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
from openpyxl import Workbook
|
||||
from openpyxl.styles import Font, PatternFill, Alignment
|
||||
|
||||
|
||||
def generate_workflow_1_data(output_dir: Path) -> None:
|
||||
"""Generate simple financial report for Workflow 1."""
|
||||
print("📊 Generating Workflow 1: financial_report_q1.xlsx")
|
||||
|
||||
# Create sample quarterly data
|
||||
months = ["January", "February", "March"]
|
||||
categories = ["Revenue", "Cost of Goods Sold", "Operating Expenses", "Net Income"]
|
||||
|
||||
data = []
|
||||
for category in categories:
|
||||
row: dict[str, str | int] = {"Category": category}
|
||||
for month in months:
|
||||
if category == "Revenue":
|
||||
value = random.randint(80000, 120000)
|
||||
elif category == "Cost of Goods Sold":
|
||||
value = random.randint(30000, 50000)
|
||||
elif category == "Operating Expenses":
|
||||
value = random.randint(20000, 35000)
|
||||
else: # Net Income
|
||||
value = int(
|
||||
int(row.get("January", 0))
|
||||
+ int(row.get("February", 0))
|
||||
+ int(row.get("March", 0))
|
||||
)
|
||||
value = random.randint(15000, 40000)
|
||||
row[month] = value
|
||||
data.append(row)
|
||||
|
||||
df = pd.DataFrame(data)
|
||||
|
||||
# Write to Excel
|
||||
output_file = output_dir / "financial_report_q1.xlsx"
|
||||
with pd.ExcelWriter(output_file, engine="openpyxl") as writer:
|
||||
df.to_excel(writer, sheet_name="Q1 Summary", index=False)
|
||||
|
||||
# Format it nicely
|
||||
worksheet = writer.sheets["Q1 Summary"]
|
||||
for cell in worksheet[1]: # Header row
|
||||
cell.font = Font(bold=True)
|
||||
cell.fill = PatternFill(
|
||||
start_color="4F81BD", end_color="4F81BD", fill_type="solid"
|
||||
)
|
||||
cell.font = Font(color="FFFFFF", bold=True)
|
||||
|
||||
print(f" ✅ Created {output_file}")
|
||||
|
||||
|
||||
def generate_workflow_2_data(output_dir: Path) -> None:
|
||||
"""Generate regional sales data for Workflow 2."""
|
||||
print("\n📊 Generating Workflow 2: Regional sales data")
|
||||
|
||||
regions = ["northeast", "southeast", "west"]
|
||||
products = ["Widget A", "Widget B", "Widget C", "Gadget X", "Gadget Y"]
|
||||
|
||||
for region in regions:
|
||||
data = []
|
||||
start_date = datetime(2024, 1, 1)
|
||||
|
||||
# Generate 90 days of sales data
|
||||
for day in range(90):
|
||||
date = start_date + timedelta(days=day)
|
||||
# Random number of sales per day (3-8)
|
||||
for _ in range(random.randint(3, 8)):
|
||||
product = random.choice(products)
|
||||
units_sold = random.randint(1, 20)
|
||||
price_per_unit = random.randint(50, 200)
|
||||
revenue = units_sold * price_per_unit
|
||||
|
||||
data.append(
|
||||
{
|
||||
"Date": date.strftime("%Y-%m-%d"),
|
||||
"Product": product,
|
||||
"Units_Sold": units_sold,
|
||||
"Revenue": revenue,
|
||||
}
|
||||
)
|
||||
|
||||
df = pd.DataFrame(data)
|
||||
|
||||
# Write to Excel
|
||||
output_file = output_dir / f"sales_{region}.xlsx"
|
||||
df.to_excel(output_file, sheet_name="Sales", index=False)
|
||||
print(f" ✅ Created {output_file} ({len(df)} rows)")
|
||||
|
||||
|
||||
def generate_workflow_3_data(output_dir: Path) -> None:
|
||||
"""Generate complex budget spreadsheet with formatting for Workflow 3."""
|
||||
print("\n📊 Generating Workflow 3: company_budget_2024.xlsx")
|
||||
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
ws.title = "Budget"
|
||||
|
||||
# Define departments with colors
|
||||
departments = {
|
||||
"Engineering": "C6E0B4",
|
||||
"Marketing": "FFD966",
|
||||
"Sales": "F4B084",
|
||||
"Operations": "B4C7E7",
|
||||
}
|
||||
|
||||
# Define categories
|
||||
categories = {
|
||||
"Personnel": ["Salaries", "Benefits", "Training"],
|
||||
"Infrastructure": ["Office Rent", "Equipment", "Software Licenses"],
|
||||
"Operations": ["Travel", "Supplies", "Miscellaneous"],
|
||||
}
|
||||
|
||||
# Styles
|
||||
header_font = Font(bold=True, size=12)
|
||||
category_font = Font(bold=True, size=11)
|
||||
|
||||
row = 1
|
||||
|
||||
# Title
|
||||
ws.merge_cells(f"A{row}:E{row}")
|
||||
ws[f"A{row}"] = "2024 Annual Budget"
|
||||
ws[f"A{row}"].font = Font(bold=True, size=14)
|
||||
ws[f"A{row}"].alignment = Alignment(horizontal="center")
|
||||
row += 2
|
||||
|
||||
# Headers
|
||||
ws[f"A{row}"] = "Category"
|
||||
ws[f"B{row}"] = "Item"
|
||||
for i, dept in enumerate(departments.keys()):
|
||||
ws.cell(row, 3 + i, dept)
|
||||
ws.cell(row, 3 + i).font = header_font
|
||||
|
||||
for cell in ws[row]:
|
||||
cell.font = header_font
|
||||
row += 1
|
||||
|
||||
# Data
|
||||
for category, items in categories.items():
|
||||
# Category header (bold)
|
||||
ws[f"A{row}"] = category
|
||||
ws[f"A{row}"].font = category_font
|
||||
row += 1
|
||||
|
||||
# Items with department budgets
|
||||
for item in items:
|
||||
ws[f"A{row}"] = ""
|
||||
ws[f"B{row}"] = item
|
||||
|
||||
# Add budget amounts for each department (with color)
|
||||
for i, (dept, color) in enumerate(departments.items()):
|
||||
amount = random.randint(5000, 50000)
|
||||
cell = ws.cell(row, 3 + i, amount)
|
||||
cell.fill = PatternFill(
|
||||
start_color=color, end_color=color, fill_type="solid"
|
||||
)
|
||||
cell.number_format = "$#,##0"
|
||||
|
||||
row += 1
|
||||
|
||||
row += 1 # Blank row between categories
|
||||
|
||||
# Adjust column widths
|
||||
ws.column_dimensions["A"].width = 20
|
||||
ws.column_dimensions["B"].width = 25
|
||||
for i in range(len(departments)):
|
||||
ws.column_dimensions[chr(67 + i)].width = 15 # C, D, E, F
|
||||
|
||||
output_file = output_dir / "company_budget_2024.xlsx"
|
||||
wb.save(output_file)
|
||||
print(f" ✅ Created {output_file}")
|
||||
print(" • Bold categories, colored departments, merged title cell")
|
||||
|
||||
|
||||
def generate_workflow_4_data(output_dir: Path) -> None:
|
||||
"""Generate weekly sales report for Workflow 4."""
|
||||
print("\n📊 Generating Workflow 4: sales_weekly.xlsx")
|
||||
|
||||
products = [
|
||||
"Product A",
|
||||
"Product B",
|
||||
"Product C",
|
||||
"Product D",
|
||||
"Product E",
|
||||
"Product F",
|
||||
"Product G",
|
||||
"Product H",
|
||||
]
|
||||
|
||||
# Generate one week of data
|
||||
data = []
|
||||
start_date = datetime(2024, 11, 4) # Monday
|
||||
|
||||
for day in range(7):
|
||||
date = start_date + timedelta(days=day)
|
||||
# Each product has 3-10 transactions per day
|
||||
for product in products:
|
||||
for _ in range(random.randint(3, 10)):
|
||||
units = random.randint(1, 15)
|
||||
price = random.randint(20, 150)
|
||||
revenue = units * price
|
||||
|
||||
data.append(
|
||||
{
|
||||
"Date": date.strftime("%Y-%m-%d"),
|
||||
"Product": product,
|
||||
"Units": units,
|
||||
"Revenue": revenue,
|
||||
}
|
||||
)
|
||||
|
||||
df = pd.DataFrame(data)
|
||||
|
||||
# Write to Excel with some formatting
|
||||
output_file = output_dir / "sales_weekly.xlsx"
|
||||
with pd.ExcelWriter(output_file, engine="openpyxl") as writer:
|
||||
df.to_excel(writer, sheet_name="Weekly Sales", index=False)
|
||||
|
||||
# Format header
|
||||
worksheet = writer.sheets["Weekly Sales"]
|
||||
for cell in worksheet[1]:
|
||||
cell.font = Font(bold=True)
|
||||
|
||||
print(f" ✅ Created {output_file} ({len(df)} rows)")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Generate all sample data files."""
|
||||
print("=" * 60)
|
||||
print("Generating Sample Data for LlamaSheets + Coding Agent Workflows")
|
||||
print("=" * 60)
|
||||
|
||||
# Create output directory
|
||||
output_dir = Path("input_data")
|
||||
output_dir.mkdir(exist_ok=True)
|
||||
|
||||
# Generate data for each workflow
|
||||
generate_workflow_1_data(output_dir)
|
||||
generate_workflow_2_data(output_dir)
|
||||
generate_workflow_3_data(output_dir)
|
||||
generate_workflow_4_data(output_dir)
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("✅ All sample data generated!")
|
||||
print("=" * 60)
|
||||
print(f"\nFiles created in {output_dir.absolute()}:")
|
||||
print("\nWorkflow 1 (Understanding a New Spreadsheet):")
|
||||
print(" • financial_report_q1.xlsx")
|
||||
print("\nWorkflow 2 (Generating Analysis Scripts):")
|
||||
print(" • sales_northeast.xlsx")
|
||||
print(" • sales_southeast.xlsx")
|
||||
print(" • sales_west.xlsx")
|
||||
print("\nWorkflow 3 (Using Cell Metadata):")
|
||||
print(" • company_budget_2024.xlsx")
|
||||
print("\nWorkflow 4 (Complete Automation):")
|
||||
print(" • sales_weekly.xlsx")
|
||||
print("\nYou can now use these files with the workflows in the documentation!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,5 @@
|
||||
llama-cloud-services # LlamaSheets SDK
|
||||
pandas>=2.0.0
|
||||
pyarrow>=12.0.0
|
||||
openpyxl>=3.0.0 # For Excel file support
|
||||
matplotlib>=3.7.0 # For visualizations (optional)
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Helper script to extract spreadsheets using LlamaSheets."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import dotenv
|
||||
from pathlib import Path
|
||||
|
||||
from llama_cloud_services.beta.sheets import LlamaSheets
|
||||
from llama_cloud_services.beta.sheets.types import (
|
||||
SpreadsheetParsingConfig,
|
||||
SpreadsheetResultType,
|
||||
)
|
||||
|
||||
dotenv.load_dotenv()
|
||||
|
||||
|
||||
async def extract_spreadsheet(
|
||||
file_path: str, output_dir: str = "data", generate_metadata: bool = True
|
||||
) -> dict:
|
||||
"""Extract a spreadsheet using LlamaSheets."""
|
||||
|
||||
client = LlamaSheets(
|
||||
base_url="https://api.cloud.llamaindex.ai",
|
||||
api_key=os.getenv("LLAMA_CLOUD_API_KEY"),
|
||||
)
|
||||
|
||||
print(f"Extracting {file_path}...")
|
||||
|
||||
# Extract regions
|
||||
config = SpreadsheetParsingConfig(
|
||||
sheet_names=None, # Extract all sheets
|
||||
generate_additional_metadata=generate_metadata,
|
||||
)
|
||||
|
||||
job_result = await client.aextract_regions(file_path, config=config)
|
||||
|
||||
print(f"Extracted {len(job_result.regions)} region(s)")
|
||||
|
||||
# Create output directory
|
||||
output_path = Path(output_dir)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Get base name for files
|
||||
base_name = Path(file_path).stem
|
||||
|
||||
# Save job metadata
|
||||
job_metadata_path = output_path / f"{base_name}_job_metadata.json"
|
||||
with open(job_metadata_path, "w") as f:
|
||||
json.dump(job_result.model_dump(mode="json"), f, indent=2)
|
||||
print(f"Saved job metadata to {job_metadata_path}")
|
||||
|
||||
# Download each region
|
||||
for idx, region in enumerate(job_result.regions, 1):
|
||||
sheet_name = region.sheet_name.replace(" ", "_")
|
||||
|
||||
# Download region data
|
||||
region_bytes = await client.adownload_region_result(
|
||||
job_id=job_result.id,
|
||||
region_id=region.region_id,
|
||||
result_type=region.region_type,
|
||||
)
|
||||
|
||||
region_path = output_path / f"{base_name}_region_{idx}_{sheet_name}.parquet"
|
||||
with open(region_path, "wb") as f:
|
||||
f.write(region_bytes)
|
||||
print(f" Table {idx}: {region_path}")
|
||||
|
||||
# Download metadata
|
||||
metadata_bytes = await client.adownload_region_result(
|
||||
job_id=job_result.id,
|
||||
region_id=region.region_id,
|
||||
result_type=SpreadsheetResultType.CELL_METADATA,
|
||||
)
|
||||
|
||||
metadata_path = output_path / f"{base_name}_metadata_{idx}_{sheet_name}.parquet"
|
||||
with open(metadata_path, "wb") as f:
|
||||
f.write(metadata_bytes)
|
||||
print(f" Metadata {idx}: {metadata_path}")
|
||||
|
||||
print(f"\nAll files saved to {output_path}/")
|
||||
|
||||
return job_result.model_dump(mode="json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python scripts/extract.py <spreadsheet_file>")
|
||||
sys.exit(1)
|
||||
|
||||
file_path = sys.argv[1]
|
||||
|
||||
if not Path(file_path).exists():
|
||||
print(f"❌ File not found: {file_path}")
|
||||
sys.exit(1)
|
||||
|
||||
result = asyncio.run(extract_spreadsheet(file_path))
|
||||
print(f"\n✅ Extraction complete! Job ID: {result['id']}")
|
||||
@@ -0,0 +1,278 @@
|
||||
"""
|
||||
Generate sample spreadsheets for LlamaSheets + LlamaIndex Agent workflows.
|
||||
|
||||
This script creates example Excel files that demonstrate different use cases:
|
||||
1. Simple data table (for Workflow 1)
|
||||
2. Regional sales data (for Workflow 2)
|
||||
3. Complex budget with formatting (for Workflow 3)
|
||||
4. Weekly sales report (for Workflow 4)
|
||||
|
||||
Usage:
|
||||
python generate_sample_data.py
|
||||
"""
|
||||
|
||||
import random
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
from openpyxl import Workbook
|
||||
from openpyxl.styles import Font, PatternFill, Alignment
|
||||
|
||||
|
||||
def generate_workflow_1_data(output_dir: Path) -> None:
|
||||
"""Generate simple financial report for Workflow 1."""
|
||||
print("📊 Generating Workflow 1: financial_report_q1.xlsx")
|
||||
|
||||
# Create sample quarterly data
|
||||
months = ["January", "February", "March"]
|
||||
categories = ["Revenue", "Cost of Goods Sold", "Operating Expenses", "Net Income"]
|
||||
|
||||
data = []
|
||||
for category in categories:
|
||||
row: dict[str, str | int] = {"Category": category}
|
||||
for month in months:
|
||||
if category == "Revenue":
|
||||
value = random.randint(80000, 120000)
|
||||
elif category == "Cost of Goods Sold":
|
||||
value = random.randint(30000, 50000)
|
||||
elif category == "Operating Expenses":
|
||||
value = random.randint(20000, 35000)
|
||||
else: # Net Income
|
||||
value = int(
|
||||
int(row.get("January", 0))
|
||||
+ int(row.get("February", 0))
|
||||
+ int(row.get("March", 0))
|
||||
)
|
||||
value = random.randint(15000, 40000)
|
||||
row[month] = value
|
||||
data.append(row)
|
||||
|
||||
df = pd.DataFrame(data)
|
||||
|
||||
# Write to Excel
|
||||
output_file = output_dir / "financial_report_q1.xlsx"
|
||||
with pd.ExcelWriter(output_file, engine="openpyxl") as writer:
|
||||
df.to_excel(writer, sheet_name="Q1 Summary", index=False)
|
||||
|
||||
# Format it nicely
|
||||
worksheet = writer.sheets["Q1 Summary"]
|
||||
for cell in worksheet[1]: # Header row
|
||||
cell.font = Font(bold=True)
|
||||
cell.fill = PatternFill(
|
||||
start_color="4F81BD", end_color="4F81BD", fill_type="solid"
|
||||
)
|
||||
cell.font = Font(color="FFFFFF", bold=True)
|
||||
|
||||
print(f" ✅ Created {output_file}")
|
||||
|
||||
|
||||
def generate_workflow_2_data(output_dir: Path) -> None:
|
||||
"""Generate regional sales data for Workflow 2."""
|
||||
print("\n📊 Generating Workflow 2: Regional sales data")
|
||||
|
||||
regions = ["northeast", "southeast", "west"]
|
||||
products = ["Widget A", "Widget B", "Widget C", "Gadget X", "Gadget Y"]
|
||||
|
||||
for region in regions:
|
||||
data = []
|
||||
start_date = datetime(2024, 1, 1)
|
||||
|
||||
# Generate 90 days of sales data
|
||||
for day in range(90):
|
||||
date = start_date + timedelta(days=day)
|
||||
# Random number of sales per day (3-8)
|
||||
for _ in range(random.randint(3, 8)):
|
||||
product = random.choice(products)
|
||||
units_sold = random.randint(1, 20)
|
||||
price_per_unit = random.randint(50, 200)
|
||||
revenue = units_sold * price_per_unit
|
||||
|
||||
data.append(
|
||||
{
|
||||
"Date": date.strftime("%Y-%m-%d"),
|
||||
"Product": product,
|
||||
"Units_Sold": units_sold,
|
||||
"Revenue": revenue,
|
||||
}
|
||||
)
|
||||
|
||||
df = pd.DataFrame(data)
|
||||
|
||||
# Write to Excel
|
||||
output_file = output_dir / f"sales_{region}.xlsx"
|
||||
df.to_excel(output_file, sheet_name="Sales", index=False)
|
||||
print(f" ✅ Created {output_file} ({len(df)} rows)")
|
||||
|
||||
|
||||
def generate_workflow_3_data(output_dir: Path) -> None:
|
||||
"""Generate complex budget spreadsheet with formatting for Workflow 3."""
|
||||
print("\n📊 Generating Workflow 3: company_budget_2024.xlsx")
|
||||
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
ws.title = "Budget"
|
||||
|
||||
# Define departments with colors
|
||||
departments = {
|
||||
"Engineering": "C6E0B4",
|
||||
"Marketing": "FFD966",
|
||||
"Sales": "F4B084",
|
||||
"Operations": "B4C7E7",
|
||||
}
|
||||
|
||||
# Define categories
|
||||
categories = {
|
||||
"Personnel": ["Salaries", "Benefits", "Training"],
|
||||
"Infrastructure": ["Office Rent", "Equipment", "Software Licenses"],
|
||||
"Operations": ["Travel", "Supplies", "Miscellaneous"],
|
||||
}
|
||||
|
||||
# Styles
|
||||
header_font = Font(bold=True, size=12)
|
||||
category_font = Font(bold=True, size=11)
|
||||
|
||||
row = 1
|
||||
|
||||
# Title
|
||||
ws.merge_cells(f"A{row}:E{row}")
|
||||
ws[f"A{row}"] = "2024 Annual Budget"
|
||||
ws[f"A{row}"].font = Font(bold=True, size=14)
|
||||
ws[f"A{row}"].alignment = Alignment(horizontal="center")
|
||||
row += 2
|
||||
|
||||
# Headers
|
||||
ws[f"A{row}"] = "Category"
|
||||
ws[f"B{row}"] = "Item"
|
||||
for i, dept in enumerate(departments.keys()):
|
||||
ws.cell(row, 3 + i, dept)
|
||||
ws.cell(row, 3 + i).font = header_font
|
||||
|
||||
for cell in ws[row]:
|
||||
cell.font = header_font
|
||||
row += 1
|
||||
|
||||
# Data
|
||||
for category, items in categories.items():
|
||||
# Category header (bold)
|
||||
ws[f"A{row}"] = category
|
||||
ws[f"A{row}"].font = category_font
|
||||
row += 1
|
||||
|
||||
# Items with department budgets
|
||||
for item in items:
|
||||
ws[f"A{row}"] = ""
|
||||
ws[f"B{row}"] = item
|
||||
|
||||
# Add budget amounts for each department (with color)
|
||||
for i, (dept, color) in enumerate(departments.items()):
|
||||
amount = random.randint(5000, 50000)
|
||||
cell = ws.cell(row, 3 + i, amount)
|
||||
cell.fill = PatternFill(
|
||||
start_color=color, end_color=color, fill_type="solid"
|
||||
)
|
||||
cell.number_format = "$#,##0"
|
||||
|
||||
row += 1
|
||||
|
||||
row += 1 # Blank row between categories
|
||||
|
||||
# Adjust column widths
|
||||
ws.column_dimensions["A"].width = 20
|
||||
ws.column_dimensions["B"].width = 25
|
||||
for i in range(len(departments)):
|
||||
ws.column_dimensions[chr(67 + i)].width = 15 # C, D, E, F
|
||||
|
||||
output_file = output_dir / "company_budget_2024.xlsx"
|
||||
wb.save(output_file)
|
||||
print(f" ✅ Created {output_file}")
|
||||
print(" • Bold categories, colored departments, merged title cell")
|
||||
|
||||
|
||||
def generate_workflow_4_data(output_dir: Path) -> None:
|
||||
"""Generate weekly sales report for Workflow 4."""
|
||||
print("\n📊 Generating Workflow 4: sales_weekly.xlsx")
|
||||
|
||||
products = [
|
||||
"Product A",
|
||||
"Product B",
|
||||
"Product C",
|
||||
"Product D",
|
||||
"Product E",
|
||||
"Product F",
|
||||
"Product G",
|
||||
"Product H",
|
||||
]
|
||||
|
||||
# Generate one week of data
|
||||
data = []
|
||||
start_date = datetime(2024, 11, 4) # Monday
|
||||
|
||||
for day in range(7):
|
||||
date = start_date + timedelta(days=day)
|
||||
# Each product has 3-10 transactions per day
|
||||
for product in products:
|
||||
for _ in range(random.randint(3, 10)):
|
||||
units = random.randint(1, 15)
|
||||
price = random.randint(20, 150)
|
||||
revenue = units * price
|
||||
|
||||
data.append(
|
||||
{
|
||||
"Date": date.strftime("%Y-%m-%d"),
|
||||
"Product": product,
|
||||
"Units": units,
|
||||
"Revenue": revenue,
|
||||
}
|
||||
)
|
||||
|
||||
df = pd.DataFrame(data)
|
||||
|
||||
# Write to Excel with some formatting
|
||||
output_file = output_dir / "sales_weekly.xlsx"
|
||||
with pd.ExcelWriter(output_file, engine="openpyxl") as writer:
|
||||
df.to_excel(writer, sheet_name="Weekly Sales", index=False)
|
||||
|
||||
# Format header
|
||||
worksheet = writer.sheets["Weekly Sales"]
|
||||
for cell in worksheet[1]:
|
||||
cell.font = Font(bold=True)
|
||||
|
||||
print(f" ✅ Created {output_file} ({len(df)} rows)")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Generate all sample data files."""
|
||||
print("=" * 60)
|
||||
print("Generating Sample Data for LlamaSheets + Coding Agent Workflows")
|
||||
print("=" * 60)
|
||||
|
||||
# Create output directory
|
||||
output_dir = Path("input_data")
|
||||
output_dir.mkdir(exist_ok=True)
|
||||
|
||||
# Generate data for each workflow
|
||||
generate_workflow_1_data(output_dir)
|
||||
generate_workflow_2_data(output_dir)
|
||||
generate_workflow_3_data(output_dir)
|
||||
generate_workflow_4_data(output_dir)
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("✅ All sample data generated!")
|
||||
print("=" * 60)
|
||||
print(f"\nFiles created in {output_dir.absolute()}:")
|
||||
print("\nWorkflow 1 (Understanding a New Spreadsheet):")
|
||||
print(" • financial_report_q1.xlsx")
|
||||
print("\nWorkflow 2 (Generating Analysis Scripts):")
|
||||
print(" • sales_northeast.xlsx")
|
||||
print(" • sales_southeast.xlsx")
|
||||
print(" • sales_west.xlsx")
|
||||
print("\nWorkflow 3 (Using Cell Metadata):")
|
||||
print(" • company_budget_2024.xlsx")
|
||||
print("\nWorkflow 4 (Complete Automation):")
|
||||
print(" • sales_weekly.xlsx")
|
||||
print("\nYou can now use these files with the workflows in the documentation!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,308 @@
|
||||
"""
|
||||
LlamaSheets Agent with LlamaIndex
|
||||
|
||||
This example shows how to build an agent that can work with spreadsheet data
|
||||
extracted by LlamaSheets using Python code execution.
|
||||
|
||||
The agent has minimal tools but maximum flexibility - it can execute arbitrary
|
||||
pandas code against the extracted data, similar to a coding agent.
|
||||
|
||||
NOTE: Code execution should be handled safely in a sandboxed environment for security.
|
||||
"""
|
||||
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import dotenv
|
||||
import pandas as pd
|
||||
from llama_index.core.agent import FunctionAgent, ToolCall, ToolCallResult, AgentStream
|
||||
from llama_index.llms.openai import OpenAI
|
||||
from workflows import Context
|
||||
|
||||
dotenv.load_dotenv()
|
||||
|
||||
# Global context for loaded dataframes
|
||||
_dataframe_context: Dict[str, Any] = {}
|
||||
|
||||
|
||||
# Helper function for initial agent context
|
||||
def list_extracted_data(data_dir: str = "data") -> str:
|
||||
"""
|
||||
List all regions and metadata files extracted by LlamaSheets.
|
||||
|
||||
This helps discover what data is available to work with.
|
||||
|
||||
Args:
|
||||
data_dir: Directory containing extracted parquet files (default: "data")
|
||||
|
||||
Returns:
|
||||
JSON string with information about available files
|
||||
"""
|
||||
data_path = Path(data_dir)
|
||||
|
||||
if not data_path.exists():
|
||||
return json.dumps({"error": f"Data directory '{data_dir}' not found"})
|
||||
|
||||
# Find all parquet and metadata files
|
||||
region_files = list(data_path.glob("*_region_*.parquet"))
|
||||
job_metadata_files = list(data_path.glob("*_job_metadata.json"))
|
||||
|
||||
regions = []
|
||||
for region_file in region_files:
|
||||
# Quick peek at dimensions
|
||||
df = pd.read_parquet(region_file)
|
||||
|
||||
# Find corresponding metadata file
|
||||
base_name = region_file.stem.replace("_region_", "_metadata_")
|
||||
metadata_path = region_file.parent / f"{base_name}.parquet"
|
||||
|
||||
regions.append(
|
||||
{
|
||||
"region_file": str(region_file),
|
||||
"metadata_file": str(metadata_path) if metadata_path.exists() else None,
|
||||
"shape": {"rows": len(df), "columns": len(df.columns)},
|
||||
"columns": list(df.columns),
|
||||
}
|
||||
)
|
||||
|
||||
result = {
|
||||
"data_directory": str(data_path.absolute()),
|
||||
"num_regions": len(regions),
|
||||
"regions": regions,
|
||||
"job_metadata_files": [str(f) for f in job_metadata_files],
|
||||
}
|
||||
|
||||
return json.dumps(result, indent=2)
|
||||
|
||||
|
||||
# Agent tool for code execution against dataframes
|
||||
def execute_dataframe_code(
|
||||
code: str, load_files: Optional[Dict[str, str]] = None
|
||||
) -> str:
|
||||
"""
|
||||
Execute Python pandas code against LlamaSheets extracted data.
|
||||
|
||||
This tool allows flexible data analysis by executing arbitrary pandas code.
|
||||
You can load parquet files, manipulate dataframes, and return results.
|
||||
|
||||
The code executes in a context where:
|
||||
- pandas is available as 'pd'
|
||||
- json is available for formatting output
|
||||
- Previously loaded dataframes are accessible by their variable names
|
||||
|
||||
Args:
|
||||
code: Python code to execute. Any print() statements or stdout/stderr
|
||||
will be captured and returned. Optionally set a 'result' variable
|
||||
for structured output.
|
||||
load_files: Optional dict mapping variable names to file paths to load
|
||||
Example: {"df": "data/sales_region_1.parquet",
|
||||
"meta": "data/sales_metadata_1.parquet"}
|
||||
|
||||
Returns:
|
||||
String containing:
|
||||
- Any stdout/stderr output from the code execution
|
||||
- The 'result' variable if it was set (formatted appropriately)
|
||||
- Error message if execution failed
|
||||
|
||||
Example usage:
|
||||
code = '''
|
||||
# Load and inspect data
|
||||
df = pd.read_parquet("data/sales_region_1.parquet")
|
||||
print(f"Loaded {len(df)} rows")
|
||||
|
||||
result = {
|
||||
"shape": df.shape,
|
||||
"columns": list(df.columns),
|
||||
"sample": df.head(3).to_dict(orient="records")
|
||||
}
|
||||
'''
|
||||
"""
|
||||
global _dataframe_context
|
||||
|
||||
# Capture stdout and stderr
|
||||
stdout_capture = io.StringIO()
|
||||
stderr_capture = io.StringIO()
|
||||
old_stdout = sys.stdout
|
||||
old_stderr = sys.stderr
|
||||
|
||||
try:
|
||||
# Redirect stdout/stderr
|
||||
sys.stdout = stdout_capture
|
||||
sys.stderr = stderr_capture
|
||||
|
||||
# Create execution context with pandas, json, and previously loaded dfs
|
||||
exec_context = {
|
||||
"pd": pd,
|
||||
"json": json,
|
||||
"Path": Path,
|
||||
**_dataframe_context, # Include previously loaded dataframes
|
||||
}
|
||||
|
||||
# Load any requested files into context
|
||||
if load_files:
|
||||
for var_name, file_path in load_files.items():
|
||||
if file_path.endswith(".parquet"):
|
||||
exec_context[var_name] = pd.read_parquet(file_path)
|
||||
# Also save to global context for future calls
|
||||
_dataframe_context[var_name] = exec_context[var_name]
|
||||
elif file_path.endswith(".json"):
|
||||
with open(file_path, "r") as f:
|
||||
exec_context[var_name] = json.load(f)
|
||||
_dataframe_context[var_name] = exec_context[var_name]
|
||||
|
||||
# Execute the code
|
||||
exec(code, exec_context)
|
||||
|
||||
# Restore stdout/stderr
|
||||
sys.stdout = old_stdout
|
||||
sys.stderr = old_stderr
|
||||
|
||||
# Collect output
|
||||
stdout_output = stdout_capture.getvalue()
|
||||
stderr_output = stderr_capture.getvalue()
|
||||
|
||||
output_parts = []
|
||||
|
||||
# Add stdout if any
|
||||
if stdout_output:
|
||||
output_parts.append(f"<stdout>{stdout_output}</stdout>")
|
||||
|
||||
# Add stderr if any
|
||||
if stderr_output:
|
||||
output_parts.append(f"<stderr>{stderr_output}</stderr>")
|
||||
|
||||
# Try to get a result (if code set a 'result' variable)
|
||||
if "result" in exec_context:
|
||||
result = exec_context["result"]
|
||||
result_str = None
|
||||
|
||||
if isinstance(result, pd.DataFrame):
|
||||
# Convert DataFrame to readable format
|
||||
result_str = result.to_string()
|
||||
elif isinstance(result, (dict, list)):
|
||||
result_str = json.dumps(result, indent=2, default=str)
|
||||
else:
|
||||
result_str = str(result)
|
||||
|
||||
if result_str:
|
||||
output_parts.append(f"<result_var>{result_str}</result_var>")
|
||||
|
||||
# Return combined output or success message
|
||||
if output_parts:
|
||||
return "\n\n".join(output_parts)
|
||||
else:
|
||||
return "Code executed successfully (no output or result)"
|
||||
|
||||
except Exception as e:
|
||||
# Restore stdout/stderr in case of error
|
||||
sys.stdout = old_stdout
|
||||
sys.stderr = old_stderr
|
||||
|
||||
# Get any partial output
|
||||
stdout_output = stdout_capture.getvalue()
|
||||
stderr_output = stderr_capture.getvalue()
|
||||
|
||||
error_parts = []
|
||||
if stdout_output:
|
||||
error_parts.append(f"=== STDOUT (before error) ===\n{stdout_output}")
|
||||
if stderr_output:
|
||||
error_parts.append(f"=== STDERR (before error) ===\n{stderr_output}")
|
||||
|
||||
error_parts.append(f"=== ERROR ===\n{str(e)}")
|
||||
error_parts.append(f"\n=== CODE ===\n{code}")
|
||||
|
||||
return "\n\n".join(error_parts)
|
||||
|
||||
|
||||
def create_llamasheets_agent(
|
||||
llm_model: str = "gpt-4.1", api_key: Optional[str] = None
|
||||
) -> FunctionAgent:
|
||||
# Initialize LLM
|
||||
llm = OpenAI(model=llm_model, api_key=api_key)
|
||||
|
||||
# Create tools - just 4 simple but powerful tools
|
||||
tools = [execute_dataframe_code]
|
||||
|
||||
# System prompt to guide the agent
|
||||
available_regions = list_extracted_data()
|
||||
system_prompt = f"""You are an AI assistant that helps analyze spreadsheet data extracted by LlamaSheets.
|
||||
|
||||
LlamaSheets extracts messy spreadsheets into clean parquet files with two types of outputs:
|
||||
1. Region files (*_region_*.parquet) - The actual data with columns and rows
|
||||
2. Metadata files (*_metadata_*.parquet) - Rich cell-level metadata including:
|
||||
- Formatting: font_bold, font_italic, font_size, background_color_rgb
|
||||
- Position: row_number, column_number, coordinate
|
||||
- Type detection: data_type, is_date_like, is_percentage, is_currency
|
||||
- Layout: is_in_first_row, is_merged_cell, horizontal_alignment
|
||||
|
||||
Your approach:
|
||||
1. Use list_extracted_data() to discover available files
|
||||
2. Use execute_dataframe_code() to load and analyze data with pandas
|
||||
3. Use metadata to understand structure (bold = headers, colors = groups)
|
||||
4. Use save_dataframe() to export results
|
||||
|
||||
Key tips:
|
||||
- Bold cells in metadata often indicate headers
|
||||
- Background colors often indicate groupings or departments
|
||||
- Load both region and metadata files for complete analysis
|
||||
- Write clear pandas code - you have full pandas functionality available
|
||||
- Store results in variables for reuse across multiple code executions
|
||||
|
||||
Existing Processed Regions:
|
||||
{available_regions}
|
||||
"""
|
||||
|
||||
# Configure agent
|
||||
return FunctionAgent(tools=tools, llm=llm, system_prompt=system_prompt)
|
||||
|
||||
|
||||
async def main():
|
||||
"""Example of using the LlamaSheets agent."""
|
||||
|
||||
# Create the agent
|
||||
agent = create_llamasheets_agent()
|
||||
ctx = Context(agent)
|
||||
|
||||
# Example queries the agent can handle:
|
||||
queries = [
|
||||
# Discovery
|
||||
"What spreadsheet data is available?",
|
||||
# Simple analysis
|
||||
"Load the sales data and show me the first few rows with column info",
|
||||
# Using metadata
|
||||
"Find all bold cells in the metadata - these are likely headers",
|
||||
]
|
||||
|
||||
# Example: Run a query
|
||||
for query in queries:
|
||||
print(f"\n=== Query: {query} ===")
|
||||
handler = agent.run(query, ctx=ctx)
|
||||
async for ev in handler.stream_events():
|
||||
if isinstance(ev, ToolCall):
|
||||
tool_kwargs_str = (
|
||||
str(ev.tool_kwargs)[:500] + " ..."
|
||||
if len(str(ev.tool_kwargs)) > 500
|
||||
else str(ev.tool_kwargs)
|
||||
)
|
||||
print(f"\n[Tool Call] {ev.tool_name} with args:\n{tool_kwargs_str}\n\n")
|
||||
elif isinstance(ev, ToolCallResult):
|
||||
result_str = (
|
||||
str(ev.tool_output)[:500] + " ..."
|
||||
if len(str(ev.tool_output)) > 500
|
||||
else str(ev.tool_output)
|
||||
)
|
||||
print(f"\n[Tool Result] {ev.tool_name}:\n{result_str}\n\n")
|
||||
elif isinstance(ev, AgentStream):
|
||||
print(ev.delta, end="", flush=True)
|
||||
|
||||
_ = await handler
|
||||
print("=== End Query ===\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import asyncio
|
||||
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,7 @@
|
||||
llama-cloud-services # LlamaSheets SDK
|
||||
llama-index-core
|
||||
llama-index-llms-openai
|
||||
pandas>=2.0.0
|
||||
pyarrow>=12.0.0
|
||||
openpyxl>=3.0.0 # For Excel file support
|
||||
matplotlib>=3.7.0 # For visualizations (optional)
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Helper script to extract spreadsheets using LlamaSheets."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import dotenv
|
||||
from pathlib import Path
|
||||
|
||||
from llama_cloud_services.beta.sheets import LlamaSheets
|
||||
from llama_cloud_services.beta.sheets.types import (
|
||||
SpreadsheetParsingConfig,
|
||||
SpreadsheetResultType,
|
||||
)
|
||||
|
||||
dotenv.load_dotenv()
|
||||
|
||||
|
||||
async def extract_spreadsheet(
|
||||
file_path: str, output_dir: str = "data", generate_metadata: bool = True
|
||||
) -> dict:
|
||||
"""Extract a spreadsheet using LlamaSheets."""
|
||||
|
||||
client = LlamaSheets(
|
||||
base_url="https://api.cloud.llamaindex.ai",
|
||||
api_key=os.getenv("LLAMA_CLOUD_API_KEY"),
|
||||
)
|
||||
|
||||
print(f"Extracting {file_path}...")
|
||||
|
||||
# Extract regions
|
||||
config = SpreadsheetParsingConfig(
|
||||
sheet_names=None, # Extract all sheets
|
||||
generate_additional_metadata=generate_metadata,
|
||||
)
|
||||
|
||||
job_result = await client.aextract_regions(file_path, config=config)
|
||||
|
||||
print(f"Extracted {len(job_result.regions)} region(s)")
|
||||
|
||||
# Create output directory
|
||||
output_path = Path(output_dir)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Get base name for files
|
||||
base_name = Path(file_path).stem
|
||||
|
||||
# Save job metadata
|
||||
job_metadata_path = output_path / f"{base_name}_job_metadata.json"
|
||||
with open(job_metadata_path, "w") as f:
|
||||
json.dump(job_result.model_dump(mode="json"), f, indent=2)
|
||||
print(f"Saved job metadata to {job_metadata_path}")
|
||||
|
||||
# Download each region
|
||||
for idx, region in enumerate(job_result.regions, 1):
|
||||
sheet_name = region.sheet_name.replace(" ", "_")
|
||||
|
||||
# Download region data
|
||||
region_bytes = await client.adownload_region_result(
|
||||
job_id=job_result.id,
|
||||
region_id=region.region_id,
|
||||
result_type=region.region_type,
|
||||
)
|
||||
|
||||
region_path = output_path / f"{base_name}_region_{idx}_{sheet_name}.parquet"
|
||||
with open(region_path, "wb") as f:
|
||||
f.write(region_bytes)
|
||||
print(f" Table {idx}: {region_path}")
|
||||
|
||||
# Download metadata
|
||||
metadata_bytes = await client.adownload_region_result(
|
||||
job_id=job_result.id,
|
||||
region_id=region.region_id,
|
||||
result_type=SpreadsheetResultType.CELL_METADATA,
|
||||
)
|
||||
|
||||
metadata_path = output_path / f"{base_name}_metadata_{idx}_{sheet_name}.parquet"
|
||||
with open(metadata_path, "wb") as f:
|
||||
f.write(metadata_bytes)
|
||||
print(f" Metadata {idx}: {metadata_path}")
|
||||
|
||||
print(f"\nAll files saved to {output_path}/")
|
||||
|
||||
return job_result.model_dump(mode="json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: python scripts/extract.py <spreadsheet_file>")
|
||||
sys.exit(1)
|
||||
|
||||
file_path = sys.argv[1]
|
||||
|
||||
if not Path(file_path).exists():
|
||||
print(f"❌ File not found: {file_path}")
|
||||
sys.exit(1)
|
||||
|
||||
result = asyncio.run(extract_spreadsheet(file_path))
|
||||
print(f"\n✅ Extraction complete! Job ID: {result['id']}")
|
||||
+1
-1
@@ -19,7 +19,7 @@
|
||||
"lint-staged": {
|
||||
"ts/llama_cloud_services/src/**/*.{ts,tsx,js,jsx}": [
|
||||
"pnpm --filter llama-cloud-services exec eslint --fix",
|
||||
"pnpm --filter llama-cloud-services exec prettier --write"
|
||||
"pnpm --filter llama-cloud-services exec prettier --write src/ tests/"
|
||||
]
|
||||
},
|
||||
"packageManager": "pnpm@10.11.1+sha512.e519b9f7639869dc8d5c3c5dfef73b3f091094b0a006d7317353c72b124e80e1afd429732e28705ad6bfa1ee879c1fce46c128ccebd3192101f43dd67c667912"
|
||||
|
||||
Generated
+3368
-19
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,17 @@
|
||||
# llama-cloud-services-py
|
||||
|
||||
## 0.6.80
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 0506c88: Moved ClassifyClient to LlamaClassify (backward compatible)
|
||||
|
||||
## 0.6.79
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- e020e3e: Remove unneeded organization_id param from beta classifier client
|
||||
|
||||
## 0.6.78
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
from llama_cloud_services.beta.classifier.client import ClassifyClient
|
||||
from llama_cloud_services.beta.classifier.client import LlamaClassify, ClassifyClient
|
||||
from llama_cloud_services.beta.classifier.types import ClassifyJobResultsWithFiles
|
||||
from llama_cloud_services.utils import SourceText, FileInput
|
||||
|
||||
__all__ = [
|
||||
"LlamaClassify",
|
||||
"ClassifyClient",
|
||||
"ClassifyJobResultsWithFiles",
|
||||
"SourceText",
|
||||
|
||||
@@ -31,7 +31,7 @@ class ClassificationOutput(BaseModel):
|
||||
classification: str
|
||||
|
||||
|
||||
class ClassifyClient:
|
||||
class LlamaClassify:
|
||||
"""
|
||||
Experimental - Client for interacting with the LlamaCloud Classifier API.
|
||||
The Classification API is currently in beta and may change in the future without notice.
|
||||
@@ -39,7 +39,6 @@ class ClassifyClient:
|
||||
Args:
|
||||
client: The LlamaCloud client to use.
|
||||
project_id: The project ID to use.
|
||||
organization_id: The organization ID to use.
|
||||
polling_interval: The interval to poll for job completion in seconds.
|
||||
polling_timeout: The timeout for the job to complete in seconds.
|
||||
"""
|
||||
@@ -48,15 +47,13 @@ class ClassifyClient:
|
||||
self,
|
||||
client: AsyncLlamaCloud,
|
||||
project_id: Optional[str] = None,
|
||||
organization_id: Optional[str] = None,
|
||||
polling_interval: float = 1.0,
|
||||
polling_timeout: float = POLLING_TIMEOUT_SECONDS,
|
||||
):
|
||||
self.client = client
|
||||
self.project_id = project_id
|
||||
self.organization_id = organization_id
|
||||
self.polling_interval = polling_interval
|
||||
self.file_client = FileClient(client, project_id, organization_id)
|
||||
self.file_client = FileClient(client, project_id)
|
||||
self.polling_timeout = polling_timeout
|
||||
|
||||
@classmethod
|
||||
@@ -64,7 +61,6 @@ class ClassifyClient:
|
||||
cls,
|
||||
api_key: str,
|
||||
project_id: Optional[str] = None,
|
||||
organization_id: Optional[str] = None,
|
||||
base_url: Optional[str] = None,
|
||||
) -> "ClassifyClient":
|
||||
"""
|
||||
@@ -74,7 +70,6 @@ class ClassifyClient:
|
||||
return cls(
|
||||
client,
|
||||
project_id,
|
||||
organization_id,
|
||||
)
|
||||
|
||||
async def acreate_classify_job(
|
||||
@@ -101,7 +96,6 @@ class ClassifyClient:
|
||||
file_ids=file_ids,
|
||||
parsing_configuration=parsing_configuration or OMIT,
|
||||
project_id=self.project_id,
|
||||
organization_id=self.organization_id,
|
||||
)
|
||||
|
||||
def create_classify_job(
|
||||
@@ -152,7 +146,6 @@ class ClassifyClient:
|
||||
results = await self.client.classifier.get_classification_job_results(
|
||||
classify_job_with_status.id,
|
||||
project_id=self.project_id,
|
||||
organization_id=self.organization_id,
|
||||
)
|
||||
|
||||
return results
|
||||
@@ -359,7 +352,7 @@ class ClassifyClient:
|
||||
The classify job with status.
|
||||
"""
|
||||
job = await self.client.classifier.get_classify_job(
|
||||
job_id, project_id=self.project_id, organization_id=self.organization_id
|
||||
job_id, project_id=self.project_id
|
||||
)
|
||||
start_time = time.time()
|
||||
while not is_terminal_status(job.status):
|
||||
@@ -370,6 +363,9 @@ class ClassifyClient:
|
||||
)
|
||||
await asyncio.sleep(self.polling_interval)
|
||||
job = await self.client.classifier.get_classify_job(
|
||||
job_id, project_id=self.project_id, organization_id=self.organization_id
|
||||
job_id, project_id=self.project_id
|
||||
)
|
||||
return job
|
||||
|
||||
|
||||
ClassifyClient = LlamaClassify
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
"""LlamaCloud Spreadsheet API SDK
|
||||
|
||||
This module provides a Python SDK for the LlamaCloud Spreadsheet API.
|
||||
"""
|
||||
|
||||
from llama_cloud_services.beta.sheets.client import (
|
||||
LlamaSheets,
|
||||
SpreadsheetAPIError,
|
||||
SpreadsheetJobError,
|
||||
SpreadsheetTimeoutError,
|
||||
)
|
||||
from llama_cloud_services.beta.sheets.types import (
|
||||
ExtractedRegionSummary,
|
||||
FileUploadResponse,
|
||||
JobStatus,
|
||||
PresignedUrlResponse,
|
||||
SpreadsheetJob,
|
||||
SpreadsheetJobResult,
|
||||
SpreadsheetParseResult,
|
||||
SpreadsheetParsingConfig,
|
||||
SpreadsheetResultType,
|
||||
WorksheetMetadata,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
# Client
|
||||
"LlamaSheets",
|
||||
# Exceptions
|
||||
"SpreadsheetAPIError",
|
||||
"SpreadsheetJobError",
|
||||
"SpreadsheetTimeoutError",
|
||||
# Types
|
||||
"ExtractedRegionSummary",
|
||||
"FileUploadResponse",
|
||||
"JobStatus",
|
||||
"PresignedUrlResponse",
|
||||
"SpreadsheetJob",
|
||||
"SpreadsheetJobResult",
|
||||
"SpreadsheetParseResult",
|
||||
"SpreadsheetParsingConfig",
|
||||
"SpreadsheetResultType",
|
||||
"WorksheetMetadata",
|
||||
]
|
||||
@@ -0,0 +1,518 @@
|
||||
import asyncio
|
||||
import io
|
||||
import os
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import httpx
|
||||
from llama_cloud.client import AsyncLlamaCloud
|
||||
from tenacity import (
|
||||
AsyncRetrying,
|
||||
retry_if_exception,
|
||||
stop_after_attempt,
|
||||
wait_exponential,
|
||||
)
|
||||
|
||||
from llama_cloud_services.beta.sheets.types import (
|
||||
FileUploadResponse,
|
||||
JobStatus,
|
||||
PresignedUrlResponse,
|
||||
SpreadsheetJob,
|
||||
SpreadsheetJobResult,
|
||||
SpreadsheetParsingConfig,
|
||||
SpreadsheetResultType,
|
||||
)
|
||||
from llama_cloud_services.constants import BASE_URL
|
||||
from llama_cloud_services.files.client import FileClient
|
||||
from llama_cloud_services.utils import (
|
||||
augment_async_errors,
|
||||
FileInput,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def _should_retry_exception(exception: BaseException) -> bool:
|
||||
"""Determine if an exception should be retried."""
|
||||
if isinstance(exception, httpx.HTTPStatusError):
|
||||
return exception.response.status_code in (429, 500, 502, 503, 504)
|
||||
return False
|
||||
|
||||
|
||||
class SpreadsheetAPIError(Exception):
|
||||
"""Base exception for spreadsheet API errors"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class SpreadsheetJobError(SpreadsheetAPIError):
|
||||
"""Exception raised when a spreadsheet job fails"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class SpreadsheetTimeoutError(SpreadsheetAPIError):
|
||||
"""Exception raised when a job times out"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class LlamaSheets:
|
||||
"""Client for the LlamaCloud Spreadsheet API"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str | None = None,
|
||||
base_url: str | None = None,
|
||||
max_timeout: int = 300,
|
||||
poll_interval: int = 5,
|
||||
max_retries: int = 3,
|
||||
async_httpx_client: httpx.AsyncClient | None = None,
|
||||
) -> None:
|
||||
"""Initialize the LlamaSheets client.
|
||||
|
||||
Args:
|
||||
api_key: API key for authentication. If not provided, will use LLAMA_CLOUD_API_KEY env var
|
||||
base_url: Base URL for the API
|
||||
max_timeout: Maximum time to wait for job completion in seconds
|
||||
poll_interval: Interval between status checks in seconds
|
||||
max_retries: Maximum number of retries for failed requests
|
||||
async_httpx_client: Optional custom async httpx client
|
||||
"""
|
||||
self.api_key = api_key or os.environ.get("LLAMA_CLOUD_API_KEY")
|
||||
if not self.api_key:
|
||||
raise ValueError(
|
||||
"An API key must be provided either as an argument or via the LLAMA_CLOUD_API_KEY environment variable."
|
||||
)
|
||||
|
||||
base_url = base_url or os.environ.get("LLAMA_CLOUD_BASE_URL", BASE_URL)
|
||||
self.base_url = str(base_url).rstrip("/")
|
||||
|
||||
self.max_timeout = max_timeout
|
||||
self.poll_interval = poll_interval
|
||||
self.max_retries = max_retries
|
||||
|
||||
self._async_client: httpx.AsyncClient | None = async_httpx_client
|
||||
self._files_client = FileClient(
|
||||
AsyncLlamaCloud(
|
||||
token=self.api_key,
|
||||
base_url=self.base_url,
|
||||
httpx_client=async_httpx_client,
|
||||
)
|
||||
)
|
||||
|
||||
def _get_async_client(self) -> httpx.AsyncClient:
|
||||
"""Get or create the async httpx client"""
|
||||
if self._async_client is None:
|
||||
self._async_client = httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(60.0),
|
||||
follow_redirects=True,
|
||||
)
|
||||
return self._async_client
|
||||
|
||||
def _get_headers(self) -> dict[str, str]:
|
||||
"""Get common headers for API requests"""
|
||||
return {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
# Sync methods
|
||||
|
||||
def upload_file(
|
||||
self, file_obj: FileInput, file_name: str | None = None
|
||||
) -> FileUploadResponse:
|
||||
"""Upload a file to the Files API.
|
||||
|
||||
Args:
|
||||
file_obj: File to upload (path, bytes, or file-like object)
|
||||
file_name: Optional name for the uploaded filename
|
||||
|
||||
Returns:
|
||||
FileUploadResponse with the uploaded file ID
|
||||
"""
|
||||
with augment_async_errors():
|
||||
return asyncio.run(self.aupload_file(file_obj))
|
||||
|
||||
def create_job(
|
||||
self,
|
||||
file_id: str,
|
||||
config: dict | SpreadsheetParsingConfig | None = None,
|
||||
) -> SpreadsheetJob:
|
||||
"""Create a new spreadsheet parsing job.
|
||||
|
||||
Args:
|
||||
file_id: ID of the uploaded file
|
||||
config: Parsing configuration
|
||||
|
||||
Returns:
|
||||
SpreadsheetJob with job details
|
||||
"""
|
||||
with augment_async_errors():
|
||||
return asyncio.run(self.acreate_job(file_id, config))
|
||||
|
||||
def get_job(
|
||||
self, job_id: str, include_results_metadata: bool = True
|
||||
) -> SpreadsheetJobResult:
|
||||
"""Get the status of a spreadsheet parsing job.
|
||||
|
||||
Args:
|
||||
job_id: ID of the job
|
||||
include_results_metadata: Whether to include results metadata in the response
|
||||
|
||||
Returns:
|
||||
SpreadsheetJobResult with job status and optionally results
|
||||
"""
|
||||
with augment_async_errors():
|
||||
return asyncio.run(self.aget_job(job_id, include_results_metadata))
|
||||
|
||||
def wait_for_completion(self, job_id: str) -> SpreadsheetJobResult:
|
||||
"""Wait for a job to complete by polling.
|
||||
|
||||
Args:
|
||||
job_id: ID of the job to wait for
|
||||
|
||||
Returns:
|
||||
SpreadsheetJobResult when job is complete
|
||||
|
||||
Raises:
|
||||
SpreadsheetTimeoutError: If job doesn't complete within max_timeout
|
||||
SpreadsheetJobError: If job fails
|
||||
"""
|
||||
with augment_async_errors():
|
||||
return asyncio.run(self.await_for_completion(job_id))
|
||||
|
||||
def download_region_result(
|
||||
self,
|
||||
job_id: str,
|
||||
region_id: str,
|
||||
result_type: SpreadsheetResultType = SpreadsheetResultType.TABLE,
|
||||
) -> bytes:
|
||||
"""Download a region result (either region data or cell metadata).
|
||||
|
||||
Args:
|
||||
job_id: ID of the job
|
||||
region_id: ID of the region
|
||||
result_type: Type of result to download (region or cell_metadata)
|
||||
|
||||
Returns:
|
||||
Raw bytes of the parquet file
|
||||
"""
|
||||
with augment_async_errors():
|
||||
return asyncio.run(
|
||||
self.adownload_region_result(job_id, region_id, result_type)
|
||||
)
|
||||
|
||||
def download_region_as_dataframe(
|
||||
self,
|
||||
job_id: str,
|
||||
region_id: str,
|
||||
result_type: SpreadsheetResultType = SpreadsheetResultType.TABLE,
|
||||
) -> "pd.DataFrame":
|
||||
"""Download a region result as a pandas DataFrame.
|
||||
|
||||
Args:
|
||||
job_id: ID of the job
|
||||
region_id: ID of the region
|
||||
result_type: Type of result to download (region or cell_metadata)
|
||||
|
||||
Returns:
|
||||
pandas DataFrame
|
||||
"""
|
||||
with augment_async_errors():
|
||||
return asyncio.run(
|
||||
self.adownload_region_as_dataframe(job_id, region_id, result_type)
|
||||
)
|
||||
|
||||
def extract_regions(
|
||||
self,
|
||||
file_obj: FileInput,
|
||||
config: dict | SpreadsheetParsingConfig | None = None,
|
||||
) -> SpreadsheetJobResult:
|
||||
"""High-level method to parse a spreadsheet file.
|
||||
|
||||
This method handles the entire workflow:
|
||||
1. Upload the file
|
||||
2. Create a parsing job
|
||||
3. Wait for completion
|
||||
4. Return results
|
||||
|
||||
Args:
|
||||
file_obj: File to parse (path, bytes, or file-like object)
|
||||
config: Parsing configuration
|
||||
|
||||
Returns:
|
||||
SpreadsheetJobResult with parsing results
|
||||
"""
|
||||
with augment_async_errors():
|
||||
return asyncio.run(self.aextract_regions(file_obj, config))
|
||||
|
||||
# Async methods
|
||||
|
||||
async def aupload_file(
|
||||
self, file_obj: FileInput, file_name: str | None = None
|
||||
) -> FileUploadResponse:
|
||||
"""Upload a file to the Files API.
|
||||
|
||||
Args:
|
||||
file_obj: File to upload (path, bytes, or file-like object)
|
||||
file_name: Optional name for the uploaded filename
|
||||
|
||||
Returns:
|
||||
FileUploadResponse with the uploaded file ID
|
||||
"""
|
||||
try:
|
||||
async for attempt in AsyncRetrying(
|
||||
stop=stop_after_attempt(self.max_retries),
|
||||
wait=wait_exponential(multiplier=1, min=1, max=32),
|
||||
retry=retry_if_exception(_should_retry_exception),
|
||||
reraise=True,
|
||||
):
|
||||
with attempt:
|
||||
return await self._files_client.upload_content(
|
||||
file_obj, external_file_id=file_name
|
||||
)
|
||||
except Exception as e:
|
||||
raise SpreadsheetAPIError(f"Failed to upload file: {e}") from e
|
||||
raise RuntimeError("Tenacity did not execute")
|
||||
|
||||
async def acreate_job(
|
||||
self,
|
||||
file_id: str,
|
||||
config: dict | SpreadsheetParsingConfig | None = None,
|
||||
) -> SpreadsheetJob:
|
||||
"""Create a new spreadsheet parsing job.
|
||||
|
||||
Args:
|
||||
file_id: ID of the uploaded file
|
||||
config: Parsing configuration
|
||||
|
||||
Returns:
|
||||
SpreadsheetJob with job details
|
||||
"""
|
||||
if config is None:
|
||||
config = SpreadsheetParsingConfig()
|
||||
elif isinstance(config, dict):
|
||||
config = SpreadsheetParsingConfig.model_validate(config)
|
||||
|
||||
if not isinstance(config, SpreadsheetParsingConfig):
|
||||
raise ValueError(
|
||||
"config must be a dict or SpreadsheetParsingConfig instance"
|
||||
)
|
||||
|
||||
payload = {
|
||||
"file_id": file_id,
|
||||
"config": config.model_dump(mode="json", exclude_none=True),
|
||||
}
|
||||
|
||||
try:
|
||||
async for attempt in AsyncRetrying(
|
||||
stop=stop_after_attempt(self.max_retries),
|
||||
wait=wait_exponential(multiplier=1, min=1, max=32),
|
||||
retry=retry_if_exception(_should_retry_exception),
|
||||
reraise=True,
|
||||
):
|
||||
with attempt:
|
||||
client = self._get_async_client()
|
||||
response = await client.post(
|
||||
f"{self.base_url}/api/v1/beta/sheets/jobs",
|
||||
headers=self._get_headers(),
|
||||
json=payload,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return SpreadsheetJob.model_validate(response.json())
|
||||
except Exception as e:
|
||||
raise SpreadsheetAPIError(f"Failed to create job: {e}") from e
|
||||
raise RuntimeError("Tenacity did not execute")
|
||||
|
||||
async def aget_job(
|
||||
self, job_id: str, include_results_metadata: bool = True
|
||||
) -> SpreadsheetJobResult:
|
||||
"""Get the status of a spreadsheet parsing job.
|
||||
|
||||
Args:
|
||||
job_id: ID of the job
|
||||
include_results_metadata: Whether to include results in the response
|
||||
|
||||
Returns:
|
||||
SpreadsheetJobResult with job status and optionally results
|
||||
"""
|
||||
try:
|
||||
async for attempt in AsyncRetrying(
|
||||
stop=stop_after_attempt(self.max_retries),
|
||||
wait=wait_exponential(multiplier=1, min=1, max=32),
|
||||
retry=retry_if_exception(_should_retry_exception),
|
||||
reraise=True,
|
||||
):
|
||||
with attempt:
|
||||
client = self._get_async_client()
|
||||
response = await client.get(
|
||||
f"{self.base_url}/api/v1/beta/sheets/jobs/{job_id}",
|
||||
headers=self._get_headers(),
|
||||
params={"include_results": include_results_metadata},
|
||||
)
|
||||
response.raise_for_status()
|
||||
return SpreadsheetJobResult.model_validate(response.json())
|
||||
except Exception as e:
|
||||
raise SpreadsheetAPIError(f"Failed to get job status: {e}") from e
|
||||
raise RuntimeError("Tenacity did not execute")
|
||||
|
||||
async def await_for_completion(self, job_id: str) -> SpreadsheetJobResult:
|
||||
"""Wait for a job to complete by polling.
|
||||
|
||||
Args:
|
||||
job_id: ID of the job to wait for
|
||||
|
||||
Returns:
|
||||
SpreadsheetJobResult when job is complete
|
||||
|
||||
Raises:
|
||||
SpreadsheetTimeoutError: If job doesn't complete within max_timeout
|
||||
SpreadsheetJobError: If job fails
|
||||
"""
|
||||
start_time = time.time()
|
||||
|
||||
while (time.time() - start_time) < self.max_timeout:
|
||||
job_result = await self.aget_job(job_id, include_results_metadata=True)
|
||||
|
||||
if job_result.status in (
|
||||
JobStatus.SUCCESS,
|
||||
JobStatus.PARTIAL_SUCCESS,
|
||||
JobStatus.ERROR,
|
||||
JobStatus.FAILURE,
|
||||
):
|
||||
if job_result.status in (JobStatus.SUCCESS, JobStatus.PARTIAL_SUCCESS):
|
||||
return job_result
|
||||
else:
|
||||
error_msg = f"Job failed with status: {job_result.status}"
|
||||
if job_result.errors:
|
||||
error_msg += f"\nErrors: {', '.join(job_result.errors)}"
|
||||
raise SpreadsheetJobError(error_msg)
|
||||
|
||||
await asyncio.sleep(self.poll_interval)
|
||||
|
||||
raise SpreadsheetTimeoutError(
|
||||
f"Job did not complete within {self.max_timeout} seconds"
|
||||
)
|
||||
|
||||
async def adownload_region_result(
|
||||
self,
|
||||
job_id: str,
|
||||
region_id: str,
|
||||
result_type: SpreadsheetResultType = SpreadsheetResultType.TABLE,
|
||||
) -> bytes:
|
||||
"""Download a region result (either region data or cell metadata).
|
||||
|
||||
Args:
|
||||
job_id: ID of the job
|
||||
region_id: ID of the region
|
||||
result_type: Type of result to download (region or cell_metadata)
|
||||
|
||||
Returns:
|
||||
Raw bytes of the parquet file
|
||||
"""
|
||||
# Get presigned URL
|
||||
presigned_response = None
|
||||
result_type_str = str(result_type)
|
||||
try:
|
||||
async for attempt in AsyncRetrying(
|
||||
stop=stop_after_attempt(self.max_retries),
|
||||
wait=wait_exponential(multiplier=1, min=1, max=32),
|
||||
retry=retry_if_exception(_should_retry_exception),
|
||||
reraise=True,
|
||||
):
|
||||
with attempt:
|
||||
client = self._get_async_client()
|
||||
response = await client.get(
|
||||
f"{self.base_url}/api/v1/beta/sheets/jobs/{job_id}/regions/{region_id}/result/{result_type_str}",
|
||||
headers=self._get_headers(),
|
||||
)
|
||||
response.raise_for_status()
|
||||
presigned_response = PresignedUrlResponse.model_validate(
|
||||
response.json()
|
||||
)
|
||||
except Exception as e:
|
||||
raise SpreadsheetAPIError(f"Failed to get presigned URL: {e}") from e
|
||||
|
||||
# Download using presigned URL
|
||||
if presigned_response is None:
|
||||
raise SpreadsheetAPIError("Failed to obtain presigned URL.")
|
||||
|
||||
try:
|
||||
async for attempt in AsyncRetrying(
|
||||
stop=stop_after_attempt(self.max_retries),
|
||||
wait=wait_exponential(multiplier=1, min=1, max=32),
|
||||
retry=retry_if_exception(_should_retry_exception),
|
||||
reraise=True,
|
||||
):
|
||||
with attempt:
|
||||
download_response = await client.get(presigned_response.url)
|
||||
download_response.raise_for_status()
|
||||
return download_response.content
|
||||
except Exception as e:
|
||||
raise SpreadsheetAPIError(f"Failed to download result: {e}") from e
|
||||
raise RuntimeError("Tenacity did not execute")
|
||||
|
||||
async def adownload_region_as_dataframe(
|
||||
self,
|
||||
job_id: str,
|
||||
region_id: str,
|
||||
result_type: SpreadsheetResultType = SpreadsheetResultType.TABLE,
|
||||
) -> "pd.DataFrame":
|
||||
"""Download a region result as a pandas DataFrame.
|
||||
|
||||
Args:
|
||||
job_id: ID of the job
|
||||
region_id: ID of the region
|
||||
result_type: Type of result to download (region or cell_metadata)
|
||||
|
||||
Returns:
|
||||
pandas DataFrame
|
||||
"""
|
||||
import pandas as pd
|
||||
|
||||
parquet_bytes = await self.adownload_region_result(
|
||||
job_id, region_id, result_type
|
||||
)
|
||||
return pd.read_parquet(io.BytesIO(parquet_bytes))
|
||||
|
||||
async def aextract_regions(
|
||||
self,
|
||||
file_obj: FileInput,
|
||||
config: dict | SpreadsheetParsingConfig | None = None,
|
||||
) -> SpreadsheetJobResult:
|
||||
"""High-level method to parse a spreadsheet file.
|
||||
|
||||
This method handles the entire workflow:
|
||||
1. Upload the file
|
||||
2. Create a parsing job
|
||||
3. Wait for completion
|
||||
4. Return results
|
||||
|
||||
Args:
|
||||
file_obj: File to parse (path, bytes, or file-like object)
|
||||
config: Parsing configuration
|
||||
|
||||
Returns:
|
||||
SpreadsheetJobResult with parsing results
|
||||
"""
|
||||
# Upload file
|
||||
file_response = await self.aupload_file(file_obj)
|
||||
|
||||
# Create job
|
||||
job = await self.acreate_job(file_response.id, config)
|
||||
|
||||
# Wait for completion
|
||||
return await self.await_for_completion(job.id)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
"""Close all HTTP clients (async)"""
|
||||
if self._async_client:
|
||||
await self._async_client.aclose()
|
||||
|
||||
async def __aenter__(self) -> "LlamaSheets":
|
||||
return self
|
||||
|
||||
async def __aexit__(self, _exc_type, _exc_val, _exc_tb) -> None: # type: ignore
|
||||
await self.aclose()
|
||||
@@ -0,0 +1,156 @@
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
|
||||
class SpreadsheetResultType(str, Enum):
|
||||
TABLE = "table"
|
||||
EXTRA = "extra"
|
||||
CELL_METADATA = "cell_metadata"
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.value
|
||||
|
||||
|
||||
class ExtractedRegionSummary(BaseModel):
|
||||
"""A summary of a single extracted region from a spreadsheet"""
|
||||
|
||||
region_id: str = Field(
|
||||
...,
|
||||
description="Unique identifier for this region within the file",
|
||||
)
|
||||
sheet_name: str = Field(..., description="Worksheet name where region was found")
|
||||
location: str = Field(..., description="Location of the region in the spreadsheet")
|
||||
title: str | None = Field(None, description="Generated title for the region")
|
||||
description: str | None = Field(
|
||||
None, description="Generated description of the region"
|
||||
)
|
||||
region_type: SpreadsheetResultType = Field(
|
||||
..., description="Type of the extracted region"
|
||||
)
|
||||
|
||||
|
||||
class WorksheetMetadata(BaseModel):
|
||||
"""Metadata about a worksheet in a spreadsheet"""
|
||||
|
||||
sheet_name: str = Field(..., description="Name of the worksheet")
|
||||
title: str | None = Field(None, description="Generated title for the worksheet")
|
||||
description: str | None = Field(
|
||||
None, description="Generated description of the worksheet"
|
||||
)
|
||||
|
||||
|
||||
class SpreadsheetParseResult(BaseModel):
|
||||
"""Result of parsing a single spreadsheet file"""
|
||||
|
||||
success: bool = Field(..., description="Whether parsing was successful")
|
||||
file_name: str = Field(..., description="Original filename")
|
||||
|
||||
regions: list[ExtractedRegionSummary] = Field(
|
||||
default_factory=list, description="All successfully extracted regions"
|
||||
)
|
||||
worksheet_metadata: list[WorksheetMetadata] = Field(
|
||||
default_factory=list, description="Metadata for each processed worksheet"
|
||||
)
|
||||
|
||||
# Error information
|
||||
errors: list[str] = Field(
|
||||
default_factory=list, description="Any errors encountered during parsing"
|
||||
)
|
||||
|
||||
|
||||
class SpreadsheetParsingConfig(BaseModel):
|
||||
"""Configuration for spreadsheet parsing and region extraction"""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
sheet_names: list[str] | None = Field(
|
||||
default=None,
|
||||
description="The names of the sheets to extract regions from. If empty, the default sheet is extracted.",
|
||||
)
|
||||
include_hidden_cells: bool = Field(
|
||||
default=True,
|
||||
description="Whether to include hidden cells when extracting regions from the spreadsheet.",
|
||||
)
|
||||
extraction_range: str | None = Field(
|
||||
default=None,
|
||||
description="A1 notation of the range to extract a single region from. If None, the entire sheet is used.",
|
||||
)
|
||||
generate_additional_metadata: bool = Field(
|
||||
default=True,
|
||||
description="Whether to generate additional metadata (title, description) for each extracted region.",
|
||||
)
|
||||
use_experimental_processing: bool = Field(
|
||||
default=False,
|
||||
description="Enables experimental processing. Accuracy may be impacted.",
|
||||
)
|
||||
|
||||
|
||||
class SpreadsheetJob(BaseModel):
|
||||
"""A spreadsheet parsing job"""
|
||||
|
||||
id: str = Field(..., description="The ID of the job")
|
||||
user_id: str = Field(..., description="The ID of the user")
|
||||
project_id: str = Field(..., description="The ID of the project")
|
||||
file: dict = Field(..., description="The file object being parsed")
|
||||
config: SpreadsheetParsingConfig = Field(
|
||||
..., description="Configuration for the parsing job"
|
||||
)
|
||||
status: str = Field(..., description="The status of the parsing job")
|
||||
created_at: str = Field(..., description="When the job was created")
|
||||
updated_at: str = Field(..., description="When the job was last updated")
|
||||
|
||||
@field_validator("created_at", "updated_at", mode="before")
|
||||
def validate_dates(cls, v: str) -> str:
|
||||
"""Validate that the dates are in the correct format"""
|
||||
if isinstance(v, datetime):
|
||||
return v.isoformat()
|
||||
else:
|
||||
return v
|
||||
|
||||
|
||||
class SpreadsheetJobResult(SpreadsheetJob):
|
||||
"""A spreadsheet parsing job result."""
|
||||
|
||||
# Results are included when the job is complete
|
||||
success: bool | None = Field(
|
||||
None, description="Whether the job completed successfully"
|
||||
)
|
||||
regions: list[ExtractedRegionSummary] = Field(
|
||||
default_factory=list,
|
||||
description="All extracted regions (populated when job is complete)",
|
||||
)
|
||||
worksheet_metadata: list[WorksheetMetadata] = Field(
|
||||
default_factory=list,
|
||||
description="Metadata for each processed worksheet (populated when job is complete)",
|
||||
)
|
||||
errors: list[str] = Field(
|
||||
default_factory=list, description="Any errors encountered"
|
||||
)
|
||||
|
||||
|
||||
class JobStatus(str, Enum):
|
||||
"""Status of a spreadsheet parsing job"""
|
||||
|
||||
PENDING = "PENDING"
|
||||
IN_PROGRESS = "IN_PROGRESS"
|
||||
SUCCESS = "SUCCESS"
|
||||
PARTIAL_SUCCESS = "PARTIAL_SUCCESS"
|
||||
ERROR = "ERROR"
|
||||
FAILURE = "FAILURE"
|
||||
|
||||
|
||||
class PresignedUrlResponse(BaseModel):
|
||||
"""Response containing a presigned URL for downloading results"""
|
||||
|
||||
url: str = Field(..., description="The presigned URL for downloading")
|
||||
|
||||
|
||||
class FileUploadResponse(BaseModel):
|
||||
"""Response from uploading a file"""
|
||||
|
||||
id: str = Field(..., description="The ID of the uploaded file")
|
||||
name: str = Field(..., description="The name of the file")
|
||||
project_id: str = Field(..., description="The project ID")
|
||||
user_id: str = Field(..., description="The user ID")
|
||||
@@ -1,2 +1,3 @@
|
||||
BASE_URL = "https://api.cloud.llamaindex.ai"
|
||||
EU_BASE_URL = "https://api.cloud.eu.llamaindex.ai"
|
||||
POLLING_TIMEOUT_SECONDS = 300.0
|
||||
|
||||
@@ -19,6 +19,7 @@ from llama_cloud import (
|
||||
PipelineCreate,
|
||||
PipelineCreateEmbeddingConfig,
|
||||
PipelineCreateTransformConfig,
|
||||
PipelineFileCreateCustomMetadataValue,
|
||||
PipelineType,
|
||||
ProjectCreate,
|
||||
ManagedIngestionStatus,
|
||||
@@ -905,6 +906,9 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
def upload_file(
|
||||
self,
|
||||
file_path: str,
|
||||
custom_metadata: Optional[
|
||||
dict[str, Optional[PipelineFileCreateCustomMetadataValue]]
|
||||
] = None,
|
||||
verbose: bool = False,
|
||||
wait_for_ingestion: bool = True,
|
||||
raise_on_error: bool = False,
|
||||
@@ -918,7 +922,9 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
print(f"Uploaded file {file.id} with name {file.name}")
|
||||
|
||||
# Add file to pipeline
|
||||
pipeline_file_create = PipelineFileCreate(file_id=file.id)
|
||||
pipeline_file_create = PipelineFileCreate(
|
||||
file_id=file.id, custom_metadata=custom_metadata
|
||||
)
|
||||
self._client.pipeline_files.add_files_to_pipeline_api(
|
||||
pipeline_id=self.pipeline.id, request=[pipeline_file_create]
|
||||
)
|
||||
@@ -932,6 +938,9 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
async def aupload_file(
|
||||
self,
|
||||
file_path: str,
|
||||
custom_metadata: Optional[
|
||||
dict[str, Optional[PipelineFileCreateCustomMetadataValue]]
|
||||
] = None,
|
||||
verbose: bool = False,
|
||||
wait_for_ingestion: bool = True,
|
||||
raise_on_error: bool = False,
|
||||
@@ -945,7 +954,9 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
print(f"Uploaded file {file.id} with name {file.name}")
|
||||
|
||||
# Add file to pipeline
|
||||
pipeline_file_create = PipelineFileCreate(file_id=file.id)
|
||||
pipeline_file_create = PipelineFileCreate(
|
||||
file_id=file.id, custom_metadata=custom_metadata
|
||||
)
|
||||
await self._aclient.pipeline_files.add_files_to_pipeline_api(
|
||||
pipeline_id=self.pipeline.id, request=[pipeline_file_create]
|
||||
)
|
||||
@@ -961,6 +972,9 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
self,
|
||||
file_name: str,
|
||||
url: str,
|
||||
custom_metadata: Optional[
|
||||
dict[str, Optional[PipelineFileCreateCustomMetadataValue]]
|
||||
] = None,
|
||||
proxy_url: Optional[str] = None,
|
||||
request_headers: Optional[Dict[str, str]] = None,
|
||||
verify_ssl: bool = True,
|
||||
@@ -983,7 +997,9 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
print(f"Uploaded file {file.id} with ID {file.id}")
|
||||
|
||||
# Add file to pipeline
|
||||
pipeline_file_create = PipelineFileCreate(file_id=file.id)
|
||||
pipeline_file_create = PipelineFileCreate(
|
||||
file_id=file.id, custom_metadata=custom_metadata
|
||||
)
|
||||
self._client.pipeline_files.add_files_to_pipeline_api(
|
||||
pipeline_id=self.pipeline.id, request=[pipeline_file_create]
|
||||
)
|
||||
@@ -998,6 +1014,9 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
self,
|
||||
file_name: str,
|
||||
url: str,
|
||||
custom_metadata: Optional[
|
||||
dict[str, Optional[PipelineFileCreateCustomMetadataValue]]
|
||||
] = None,
|
||||
proxy_url: Optional[str] = None,
|
||||
request_headers: Optional[Dict[str, str]] = None,
|
||||
verify_ssl: bool = True,
|
||||
@@ -1020,7 +1039,9 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
print(f"Uploaded file {file.id} with ID {file.id}")
|
||||
|
||||
# Add file to pipeline
|
||||
pipeline_file_create = PipelineFileCreate(file_id=file.id)
|
||||
pipeline_file_create = PipelineFileCreate(
|
||||
file_id=file.id, custom_metadata=custom_metadata
|
||||
)
|
||||
await self._aclient.pipeline_files.add_files_to_pipeline_api(
|
||||
pipeline_id=self.pipeline.id, request=[pipeline_file_create]
|
||||
)
|
||||
|
||||
@@ -1,5 +1,19 @@
|
||||
# llama_parse
|
||||
|
||||
## 0.6.80
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [0506c88]
|
||||
- llama-cloud-services-py@0.6.80
|
||||
|
||||
## 0.6.79
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [e020e3e]
|
||||
- llama-cloud-services-py@0.6.79
|
||||
|
||||
## 0.6.78
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "llama_parse",
|
||||
"version": "0.6.78",
|
||||
"version": "0.6.80",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"private": false,
|
||||
|
||||
@@ -11,13 +11,13 @@ dev = [
|
||||
|
||||
[project]
|
||||
name = "llama-parse"
|
||||
version = "0.6.78"
|
||||
version = "0.6.80"
|
||||
description = "Parse files into RAG-Optimized formats."
|
||||
authors = [{name = "Logan Markewich", email = "logan@llamaindex.ai"}]
|
||||
requires-python = ">=3.9,<4.0"
|
||||
readme = "README.md"
|
||||
license = "MIT"
|
||||
dependencies = ["llama-cloud-services>=0.6.78"]
|
||||
dependencies = ["llama-cloud-services>=0.6.80"]
|
||||
|
||||
[project.scripts]
|
||||
llama-parse = "llama_parse.cli.main:parse"
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "llama-cloud-services-py",
|
||||
"version": "0.6.78",
|
||||
"version": "0.6.80",
|
||||
"private": false,
|
||||
"license": "MIT",
|
||||
"scripts": {},
|
||||
|
||||
+5
-2
@@ -14,12 +14,15 @@ dev = [
|
||||
"ipython>=8.12.3,<9",
|
||||
"jupyter>=1.1.1,<2",
|
||||
"mypy>=1.14.1,<2",
|
||||
"pydantic-settings>=2.10.1"
|
||||
"pydantic-settings>=2.10.1",
|
||||
"pandas",
|
||||
"openpyxl",
|
||||
"pyarrow"
|
||||
]
|
||||
|
||||
[project]
|
||||
name = "llama-cloud-services"
|
||||
version = "0.6.78"
|
||||
version = "0.6.80"
|
||||
description = "Tailored SDK clients for LlamaCloud services."
|
||||
authors = [{name = "Logan Markewich", email = "logan@runllama.ai"}]
|
||||
requires-python = ">=3.9,<4.0"
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
import os
|
||||
import tempfile
|
||||
import pytest
|
||||
import pandas as pd
|
||||
|
||||
from llama_cloud_services.beta.sheets import LlamaSheets
|
||||
from llama_cloud_services.beta.sheets.types import SpreadsheetParsingConfig
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sheets_client():
|
||||
"""Create a LlamaSheets client for testing."""
|
||||
api_key = os.getenv("LLAMA_CLOUD_API_KEY")
|
||||
base_url = os.getenv("LLAMA_CLOUD_BASE_URL", "https://api.cloud.llamaindex.ai")
|
||||
|
||||
client = LlamaSheets(
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
max_timeout=300,
|
||||
poll_interval=2,
|
||||
)
|
||||
return client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_excel_file():
|
||||
"""Create a temporary Excel file with sample data."""
|
||||
# Create a simple dataframe with various data types
|
||||
data = {
|
||||
"Name": ["Alice", "Bob", "Charlie", "David", "Eve"],
|
||||
"Age": [25, 30, 35, 40, 45],
|
||||
"City": ["New York", "Los Angeles", "Chicago", "Houston", "Phoenix"],
|
||||
"Salary": [50000.50, 75000.75, 100000.00, 125000.25, 150000.50],
|
||||
}
|
||||
df = pd.DataFrame(data)
|
||||
|
||||
# Create a temporary file
|
||||
with tempfile.NamedTemporaryFile(suffix=".xlsx", delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
df.to_excel(tmp_path, index=False, sheet_name="TestSheet")
|
||||
|
||||
yield tmp_path
|
||||
|
||||
# Cleanup
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
|
||||
reason="LLAMA_CLOUD_API_KEY not set",
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_spreadsheet_extraction_e2e(
|
||||
sheets_client: LlamaSheets, sample_excel_file: str
|
||||
):
|
||||
"""End-to-end test for spreadsheet extraction.
|
||||
|
||||
This test:
|
||||
1. Creates a temporary Excel file with sample data
|
||||
2. Uploads and extracts tables from the file
|
||||
3. Downloads the extracted table as a DataFrame
|
||||
4. Verifies the extracted data matches the original data
|
||||
"""
|
||||
# Extract tables from the spreadsheet
|
||||
result = await sheets_client.aextract_tables(sample_excel_file)
|
||||
|
||||
# Verify job completed successfully
|
||||
assert result.status in ("SUCCESS", "PARTIAL_SUCCESS")
|
||||
assert result.success is True
|
||||
|
||||
# Verify we extracted at least one table
|
||||
assert len(result.tables) > 0, "Expected at least one table to be extracted"
|
||||
|
||||
# Get the first table
|
||||
first_table = result.tables[0]
|
||||
assert first_table.sheet_name == "TestSheet"
|
||||
|
||||
# Download the table as a DataFrame
|
||||
extracted_df = await sheets_client.adownload_table_as_dataframe(
|
||||
job_id=result.id,
|
||||
table_id=first_table.table_id,
|
||||
)
|
||||
|
||||
# Load the original dataframe for comparison
|
||||
original_df = pd.read_excel(sample_excel_file)
|
||||
|
||||
# Verify the extracted DataFrame has the expected shape
|
||||
breakpoint()
|
||||
assert extracted_df.shape[0] == original_df.shape[0], (
|
||||
f"Row count mismatch: extracted {extracted_df.shape[0]}, "
|
||||
f"original {original_df.shape[0]}"
|
||||
)
|
||||
assert extracted_df.shape[1] == original_df.shape[1], (
|
||||
f"Column count mismatch: extracted {extracted_df.shape[1]}, "
|
||||
f"original {original_df.shape[1]}"
|
||||
)
|
||||
|
||||
# Verify column names match
|
||||
assert list(extracted_df.columns) == list(original_df.columns), (
|
||||
f"Column names mismatch: extracted {list(extracted_df.columns)}, "
|
||||
f"original {list(original_df.columns)}"
|
||||
)
|
||||
|
||||
# Verify data types are preserved (at least numerically)
|
||||
for col in original_df.columns:
|
||||
if original_df[col].dtype in ["int64", "float64"]:
|
||||
assert extracted_df[col].dtype in ["int64", "float64"], (
|
||||
f"Column {col} type mismatch: extracted {extracted_df[col].dtype}, "
|
||||
f"original {original_df[col].dtype}"
|
||||
)
|
||||
|
||||
# Verify the data values match (allowing for minor type conversions)
|
||||
for col in original_df.columns:
|
||||
original_values = original_df[col].tolist()
|
||||
extracted_values = extracted_df[col].tolist()
|
||||
|
||||
# Convert both to strings for comparison to handle type differences
|
||||
original_str = [str(v) for v in original_values]
|
||||
extracted_str = [str(v) for v in extracted_values]
|
||||
|
||||
assert original_str == extracted_str, (
|
||||
f"Column {col} values mismatch:\n"
|
||||
f"Original: {original_str}\n"
|
||||
f"Extracted: {extracted_str}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
os.environ.get("LLAMA_CLOUD_API_KEY", "") == "",
|
||||
reason="LLAMA_CLOUD_API_KEY not set",
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_spreadsheet_extraction_with_config(
|
||||
sheets_client: LlamaSheets, sample_excel_file: str
|
||||
):
|
||||
"""Test spreadsheet extraction with custom configuration."""
|
||||
# Create a config with specific settings
|
||||
config = SpreadsheetParsingConfig(
|
||||
sheet_names=["TestSheet"],
|
||||
include_hidden_cells=True,
|
||||
generate_additional_metadata=True,
|
||||
)
|
||||
|
||||
# Extract tables with the config
|
||||
result = await sheets_client.aextract_tables(sample_excel_file, config=config)
|
||||
|
||||
# Verify job completed successfully
|
||||
assert result.status in ("SUCCESS", "PARTIAL_SUCCESS")
|
||||
assert result.success is True
|
||||
|
||||
# Verify that additional metadata was generated
|
||||
assert len(result.worksheet_metadata) > 0
|
||||
assert result.worksheet_metadata[0].title is not None
|
||||
assert result.worksheet_metadata[0].description is not None
|
||||
|
||||
# Verify we extracted at least one table
|
||||
assert len(result.tables) > 0
|
||||
|
||||
# Verify the sheet name matches
|
||||
assert result.tables[0].sheet_name == "TestSheet"
|
||||
@@ -44,7 +44,6 @@ def classify_client(
|
||||
return ClassifyClient(
|
||||
async_llama_cloud_client,
|
||||
project_id=project.id,
|
||||
organization_id=project.organization_id,
|
||||
polling_interval=1,
|
||||
)
|
||||
|
||||
@@ -56,7 +55,6 @@ def file_client(
|
||||
return FileClient(
|
||||
async_llama_cloud_client,
|
||||
project_id=project.id,
|
||||
organization_id=project.organization_id,
|
||||
use_presigned_url=False,
|
||||
)
|
||||
|
||||
@@ -148,7 +146,6 @@ async def test_classify_file_ids_from_api_key(
|
||||
api_key=e2e_test_settings.LLAMA_CLOUD_API_KEY.get_secret_value(),
|
||||
base_url=e2e_test_settings.LLAMA_CLOUD_BASE_URL,
|
||||
project_id=pdf_file.project_id,
|
||||
organization_id=e2e_test_settings.LLAMA_CLOUD_ORGANIZATION_ID,
|
||||
)
|
||||
|
||||
# Classify the uploaded files
|
||||
|
||||
@@ -170,6 +170,43 @@ def test_upload_file(index_name: str):
|
||||
os.remove(temp_file_path)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not base_url or not api_key, reason="No platform base url or api key set"
|
||||
)
|
||||
def test_upload_file_with_custom_metadata(index_name: str):
|
||||
index = LlamaCloudIndex.create_index(
|
||||
name=index_name,
|
||||
project_name=project_name,
|
||||
organization_id=organization_id,
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
)
|
||||
|
||||
# Create a temporary file to upload
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as temp_file:
|
||||
temp_file.write(b"Sample content for testing upload.")
|
||||
temp_file_path = temp_file.name
|
||||
custom_metadata = {"foo": "bar"}
|
||||
|
||||
try:
|
||||
# Upload the file
|
||||
file_id = index.upload_file(
|
||||
temp_file_path, custom_metadata=custom_metadata, verbose=True
|
||||
)
|
||||
assert file_id is not None
|
||||
|
||||
# Verify the file is part of the index
|
||||
docs = index.ref_doc_info
|
||||
temp_file_name = os.path.basename(temp_file_path)
|
||||
assert any(
|
||||
temp_file_name == doc.metadata.get("file_name") for doc in docs.values()
|
||||
)
|
||||
|
||||
finally:
|
||||
# Clean up the temporary file
|
||||
os.remove(temp_file_path)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not base_url or not api_key, reason="No platform base url or api key set"
|
||||
)
|
||||
@@ -196,6 +233,38 @@ def test_upload_file_from_url(remote_file: Tuple[str, str], index_name: str):
|
||||
assert any(test_file_name == doc.metadata.get("file_name") for doc in docs.values())
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not base_url or not api_key, reason="No platform base url or api key set"
|
||||
)
|
||||
def test_upload_file_from_url_with_custom_metadata(
|
||||
remote_file: Tuple[str, str], index_name: str
|
||||
):
|
||||
index = LlamaCloudIndex.create_index(
|
||||
name=index_name,
|
||||
project_name=project_name,
|
||||
organization_id=organization_id,
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
)
|
||||
|
||||
# Define a URL to a file for testing
|
||||
custom_metadata = {"foo": "bar"}
|
||||
test_file_url, test_file_name = remote_file
|
||||
|
||||
# Upload the file from the URL
|
||||
file_id = index.upload_file_from_url(
|
||||
file_name=test_file_name,
|
||||
url=test_file_url,
|
||||
custom_metadata=custom_metadata,
|
||||
verbose=True,
|
||||
)
|
||||
assert file_id is not None
|
||||
|
||||
# Verify the file is part of the index
|
||||
docs = index.ref_doc_info
|
||||
assert any(test_file_name == doc.metadata.get("file_name") for doc in docs.values())
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not base_url or not api_key, reason="No platform base url or api key set"
|
||||
)
|
||||
@@ -507,6 +576,33 @@ async def test_async_upload_file_from_url(
|
||||
await index.await_for_completion()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not base_url or not api_key, reason="No platform base url or api key set"
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_upload_file_from_url_with_custom_metadata(
|
||||
remote_file: Tuple[str, str], index_name: str
|
||||
):
|
||||
index = await LlamaCloudIndex.acreate_index(
|
||||
name=index_name,
|
||||
project_name=project_name,
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
)
|
||||
|
||||
custom_metadata = {"foo": "bar"}
|
||||
test_file_url, test_file_name = remote_file
|
||||
file_id = await index.aupload_file_from_url(
|
||||
file_name=test_file_name,
|
||||
url=test_file_url,
|
||||
custom_metadata=custom_metadata,
|
||||
verbose=True,
|
||||
)
|
||||
assert file_id is not None
|
||||
|
||||
await index.await_for_completion()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not base_url or not api_key, reason="No platform base url or api key set"
|
||||
)
|
||||
@@ -525,6 +621,29 @@ async def test_async_index_from_file(index_name: str, local_file: str):
|
||||
await index.await_for_completion()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not base_url or not api_key, reason="No platform base url or api key set"
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_index_from_file_with_custom_metadata(
|
||||
index_name: str, local_file: str
|
||||
):
|
||||
index = await LlamaCloudIndex.acreate_index(
|
||||
name=index_name,
|
||||
project_name=project_name,
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
)
|
||||
|
||||
custom_metadata = {"foo": "bar"}
|
||||
file_id = await index.aupload_file(
|
||||
file_path=local_file, custom_metadata=custom_metadata, verbose=True
|
||||
)
|
||||
assert file_id is not None
|
||||
|
||||
await index.await_for_completion()
|
||||
|
||||
|
||||
class DummySchema(BaseModel):
|
||||
source: str
|
||||
|
||||
|
||||
Generated
+1
-1
@@ -1596,7 +1596,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "llama-cloud-services"
|
||||
version = "0.6.77"
|
||||
version = "0.6.79"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
# llama-cloud-services
|
||||
|
||||
## 0.4.0
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- f293547: Switch to keyword arguments rather than positional args
|
||||
|
||||
## 0.3.10
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "llama-cloud-services",
|
||||
"version": "0.3.10",
|
||||
"version": "0.4.0",
|
||||
"type": "module",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
@@ -9,8 +9,8 @@
|
||||
"build": "pnpm run generate && bunchee",
|
||||
"dev": "bunchee --watch",
|
||||
"lint": "eslint src/ --ignore-pattern client/*.ts --no-warn-ignored",
|
||||
"format": "prettier --write ./src/",
|
||||
"format:check": "prettier --check ./src/",
|
||||
"format": "prettier --write ./src/ tests/",
|
||||
"format:check": "prettier --check ./src/ tests/",
|
||||
"test": "vitest run --testTimeout=60000",
|
||||
"test:watch": "vitest --watch",
|
||||
"test:ui": "vitest --ui",
|
||||
|
||||
@@ -36,34 +36,40 @@ export class LlamaClassify {
|
||||
|
||||
async classify(
|
||||
rules: ClassifierRule[],
|
||||
parsingConfiguration: ClassifyParsingConfiguration,
|
||||
fileContents:
|
||||
| Buffer<ArrayBufferLike>[]
|
||||
| File[]
|
||||
| Uint8Array<ArrayBuffer>[]
|
||||
| string[]
|
||||
| undefined = undefined,
|
||||
filePaths: string[] | undefined = undefined,
|
||||
projectId: string | null = null,
|
||||
organizationId: string | null = null,
|
||||
pollingInterval: number = 1,
|
||||
maxPollingIterations: number = 1800,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<ClassifyJobResults> {
|
||||
const result = await classify(
|
||||
rules,
|
||||
parsingConfiguration,
|
||||
configuration: ClassifyParsingConfiguration,
|
||||
{
|
||||
fileContents,
|
||||
filePaths,
|
||||
projectId,
|
||||
organizationId,
|
||||
this.client,
|
||||
pollingInterval = 1,
|
||||
maxPollingIterations = 1800,
|
||||
maxRetriesOnError = 10,
|
||||
retryInterval = 0.5,
|
||||
}: {
|
||||
fileContents?:
|
||||
| Buffer<ArrayBufferLike>[]
|
||||
| File[]
|
||||
| Uint8Array<ArrayBuffer>[]
|
||||
| string[]
|
||||
| undefined;
|
||||
filePaths?: string[] | undefined;
|
||||
projectId?: string;
|
||||
pollingInterval?: number;
|
||||
maxPollingIterations?: number;
|
||||
maxRetriesOnError?: number;
|
||||
retryInterval?: number;
|
||||
},
|
||||
): Promise<ClassifyJobResults> {
|
||||
const result = await classify(rules, configuration, {
|
||||
fileContents,
|
||||
filePaths,
|
||||
projectId: projectId ?? undefined,
|
||||
client: this.client,
|
||||
pollingInterval,
|
||||
maxPollingIterations,
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
);
|
||||
});
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,16 +19,23 @@ import { sleep } from "./utils";
|
||||
import { uploadFile } from "./fileUpload";
|
||||
import { File } from "buffer";
|
||||
|
||||
async function createClassifyJob(
|
||||
fileIds: string[],
|
||||
rules: ClassifierRule[],
|
||||
parsingConfiguration: ClassifyParsingConfiguration,
|
||||
organizationId: null | string,
|
||||
projectId: null | string,
|
||||
client: Client | undefined,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<string> {
|
||||
async function createClassifyJob({
|
||||
fileIds,
|
||||
rules,
|
||||
parsingConfiguration,
|
||||
projectId,
|
||||
client,
|
||||
maxRetriesOnError = 10,
|
||||
retryInterval = 0.5,
|
||||
}: {
|
||||
fileIds: string[];
|
||||
rules: ClassifierRule[];
|
||||
parsingConfiguration: ClassifyParsingConfiguration;
|
||||
projectId?: string | undefined;
|
||||
client?: Client | undefined;
|
||||
maxRetriesOnError?: number;
|
||||
retryInterval?: number;
|
||||
}): Promise<string> {
|
||||
const rawData = {
|
||||
file_ids: fileIds,
|
||||
rules: rules,
|
||||
@@ -38,7 +45,6 @@ async function createClassifyJob(
|
||||
body: rawData,
|
||||
query: {
|
||||
project_id: projectId,
|
||||
organization_id: organizationId,
|
||||
},
|
||||
} as CreateClassifyJobApiV1ClassifierJobsPostData;
|
||||
const options = data as Options<CreateClassifyJobApiV1ClassifierJobsPostData>;
|
||||
@@ -75,12 +81,17 @@ async function createClassifyJob(
|
||||
}
|
||||
}
|
||||
|
||||
async function pollForJobCompletion(
|
||||
jobId: string,
|
||||
interval: number = 1,
|
||||
maxIterations: number = 1800,
|
||||
client: Client | undefined = undefined,
|
||||
): Promise<boolean> {
|
||||
async function pollForJobCompletion({
|
||||
jobId,
|
||||
interval = 1,
|
||||
maxIterations = 1800,
|
||||
client,
|
||||
}: {
|
||||
jobId: string;
|
||||
interval?: number;
|
||||
maxIterations?: number;
|
||||
client?: Client | undefined;
|
||||
}): Promise<boolean> {
|
||||
let status: StatusEnum | undefined = undefined;
|
||||
const jobData = {
|
||||
path: { classify_job_id: jobId },
|
||||
@@ -114,17 +125,22 @@ async function pollForJobCompletion(
|
||||
}
|
||||
}
|
||||
|
||||
async function getJobResult(
|
||||
jobId: string,
|
||||
client: Client | undefined = undefined,
|
||||
projectId: string | null = null,
|
||||
organizationId: string | null = null,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<ClassifyJobResults> {
|
||||
async function getJobResult({
|
||||
jobId,
|
||||
client,
|
||||
projectId,
|
||||
maxRetriesOnError = 10,
|
||||
retryInterval = 0.5,
|
||||
}: {
|
||||
jobId: string;
|
||||
client?: Client | undefined;
|
||||
projectId?: string | undefined;
|
||||
maxRetriesOnError?: number;
|
||||
retryInterval?: number;
|
||||
}): Promise<ClassifyJobResults> {
|
||||
const jobData = {
|
||||
path: { classify_job_id: jobId },
|
||||
query: { organization_id: organizationId, project_id: projectId },
|
||||
query: { project_id: projectId },
|
||||
} as GetClassificationJobResultsApiV1ClassifierJobsClassifyJobIdResultsGetData;
|
||||
const jobOptions =
|
||||
jobData as Options<GetClassificationJobResultsApiV1ClassifierJobsClassifyJobIdResultsGetData>;
|
||||
@@ -166,20 +182,30 @@ async function getJobResult(
|
||||
export async function classify(
|
||||
rules: ClassifierRule[],
|
||||
parsingConfiguration: ClassifyParsingConfiguration,
|
||||
fileContents:
|
||||
| Buffer<ArrayBufferLike>[]
|
||||
| File[]
|
||||
| Uint8Array<ArrayBuffer>[]
|
||||
| string[]
|
||||
| undefined = undefined,
|
||||
filePaths: string[] | undefined = undefined,
|
||||
projectId: string | null = null,
|
||||
organizationId: string | null = null,
|
||||
client: Client | undefined = undefined,
|
||||
pollingInterval: number = 1,
|
||||
maxPollingIterations: number = 1800,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
{
|
||||
fileContents,
|
||||
filePaths,
|
||||
projectId,
|
||||
client,
|
||||
pollingInterval = 1,
|
||||
maxPollingIterations = 1800,
|
||||
maxRetriesOnError = 10,
|
||||
retryInterval = 0.5,
|
||||
}: {
|
||||
fileContents?:
|
||||
| Buffer<ArrayBufferLike>[]
|
||||
| File[]
|
||||
| Uint8Array<ArrayBuffer>[]
|
||||
| string[]
|
||||
| undefined;
|
||||
filePaths?: string[] | undefined;
|
||||
projectId?: string | undefined;
|
||||
client?: Client | undefined;
|
||||
pollingInterval?: number;
|
||||
maxPollingIterations?: number;
|
||||
maxRetriesOnError?: number;
|
||||
retryInterval?: number;
|
||||
},
|
||||
): Promise<ClassifyJobResults> {
|
||||
const fileIds: string[] = [];
|
||||
if (!filePaths && !fileContents) {
|
||||
@@ -191,16 +217,13 @@ export async function classify(
|
||||
if (filePaths) {
|
||||
const uploadPromises = filePaths.map(async (name) => {
|
||||
try {
|
||||
const fileId = await uploadFile(
|
||||
name,
|
||||
undefined,
|
||||
undefined,
|
||||
projectId,
|
||||
organizationId,
|
||||
client,
|
||||
const fileId = await uploadFile({
|
||||
filePath: name,
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
);
|
||||
retryInterval: retryInterval,
|
||||
project_id: projectId,
|
||||
client: client,
|
||||
});
|
||||
if (fileId) {
|
||||
return fileId;
|
||||
} else {
|
||||
@@ -220,16 +243,13 @@ export async function classify(
|
||||
if (fileContents) {
|
||||
const uploadPromises = fileContents.map(async (content) => {
|
||||
try {
|
||||
const fileId = await uploadFile(
|
||||
undefined,
|
||||
content,
|
||||
undefined,
|
||||
projectId,
|
||||
organizationId,
|
||||
client,
|
||||
const fileId = await uploadFile({
|
||||
fileContent: content,
|
||||
...(projectId ? { project_id: projectId } : {}),
|
||||
...(client ? { client: client } : {}),
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
);
|
||||
});
|
||||
if (fileId) {
|
||||
return fileId;
|
||||
} else {
|
||||
@@ -252,33 +272,31 @@ export async function classify(
|
||||
);
|
||||
}
|
||||
|
||||
const jobId = await createClassifyJob(
|
||||
const jobId = await createClassifyJob({
|
||||
fileIds,
|
||||
rules,
|
||||
parsingConfiguration,
|
||||
organizationId,
|
||||
projectId,
|
||||
client,
|
||||
...(projectId ? { projectId: projectId } : {}),
|
||||
...(client ? { client: client } : {}),
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
);
|
||||
const success = await pollForJobCompletion(
|
||||
});
|
||||
const success = await pollForJobCompletion({
|
||||
jobId,
|
||||
pollingInterval,
|
||||
maxPollingIterations,
|
||||
interval: pollingInterval,
|
||||
maxIterations: maxPollingIterations,
|
||||
client,
|
||||
);
|
||||
});
|
||||
if (!success) {
|
||||
throw new Error("Your job is taking longer than 10 minutes, timing out...");
|
||||
} else {
|
||||
return (await getJobResult(
|
||||
return (await getJobResult({
|
||||
jobId,
|
||||
client,
|
||||
projectId,
|
||||
organizationId,
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
)) as ClassifyJobResults;
|
||||
})) as ClassifyJobResults;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -378,16 +378,16 @@ export async function extract(
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<ExtractResult | undefined> {
|
||||
const fileId = (await uploadFile(
|
||||
const fileId = (await uploadFile({
|
||||
filePath,
|
||||
fileContent,
|
||||
fileName,
|
||||
project_id,
|
||||
organization_id,
|
||||
project_id: project_id ?? undefined,
|
||||
organization_id: organization_id ?? undefined,
|
||||
client,
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
)) as string;
|
||||
})) as string;
|
||||
const extractJobCreate = {
|
||||
extraction_agent_id: agentId,
|
||||
file_id: fileId,
|
||||
@@ -457,16 +457,16 @@ export async function extractStateless(
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<ExtractResult | undefined> {
|
||||
const fileId = (await uploadFile(
|
||||
const fileId = (await uploadFile({
|
||||
filePath,
|
||||
fileContent,
|
||||
fileName,
|
||||
project_id,
|
||||
organization_id,
|
||||
project_id: project_id ?? undefined,
|
||||
organization_id: organization_id ?? undefined,
|
||||
client,
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
)) as string;
|
||||
})) as string;
|
||||
const extractStatetelessCreate = {
|
||||
data_schema: dataSchema,
|
||||
file_id: fileId,
|
||||
|
||||
@@ -23,21 +23,30 @@ function textToFile(text: string, fileName: string | null = null) {
|
||||
);
|
||||
}
|
||||
|
||||
export async function uploadFile(
|
||||
filePath: string | undefined = undefined,
|
||||
fileContent:
|
||||
export async function uploadFile({
|
||||
filePath,
|
||||
fileContent,
|
||||
fileName,
|
||||
project_id,
|
||||
organization_id,
|
||||
client,
|
||||
maxRetriesOnError = 10,
|
||||
retryInterval = 0.5,
|
||||
}: {
|
||||
filePath?: string | undefined;
|
||||
fileContent?:
|
||||
| Buffer<ArrayBufferLike>
|
||||
| File
|
||||
| Uint8Array<ArrayBuffer>
|
||||
| string
|
||||
| undefined = undefined,
|
||||
fileName: string | undefined = undefined,
|
||||
project_id: string | null = null,
|
||||
organization_id: string | null = null,
|
||||
client: Client | undefined = undefined,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<string | undefined> {
|
||||
| undefined;
|
||||
fileName?: string | undefined;
|
||||
project_id?: string | undefined;
|
||||
organization_id?: string | undefined;
|
||||
client?: Client | undefined;
|
||||
maxRetriesOnError?: number;
|
||||
retryInterval?: number;
|
||||
}): Promise<string | undefined> {
|
||||
let file: File | undefined = undefined;
|
||||
if (typeof filePath === "undefined" && typeof fileContent === "undefined") {
|
||||
throw new Error(
|
||||
@@ -79,7 +88,7 @@ export async function uploadFile(
|
||||
} as BodyUploadFileApiV1FilesPost;
|
||||
const uploadData = {
|
||||
body: fileToUpload,
|
||||
query: { organization_id: organization_id, project_id: project_id },
|
||||
query: { project_id: project_id, organization_id: organization_id },
|
||||
} as UploadFileApiV1FilesPostData;
|
||||
const uploadOptions = uploadData as Options<UploadFileApiV1FilesPostData>;
|
||||
if (typeof client != "undefined") {
|
||||
@@ -95,6 +104,8 @@ export async function uploadFile(
|
||||
const uploadResponse = await uploadFileApiV1FilesPost(uploadOptions);
|
||||
let fileId: string | undefined = undefined;
|
||||
if (!uploadResponse.response.ok) {
|
||||
const error = await uploadResponse.response.text();
|
||||
console.error("Error while uploading file: ", error);
|
||||
retries++;
|
||||
await sleep(retryInterval * 1000);
|
||||
}
|
||||
|
||||
@@ -3,7 +3,10 @@ import { LlamaParseReader } from "../src/reader.js";
|
||||
import { LlamaCloudIndex } from "../src/LlamaCloudIndex.js";
|
||||
import { LlamaExtract, LlamaExtractAgent } from "../src/LlamaExtract.js";
|
||||
import { LlamaClassify } from "../src/LlamaClassify.js";
|
||||
import { ClassifierRule, ClassifyParsingConfiguration } from "../src/classify.js";
|
||||
import {
|
||||
ClassifierRule,
|
||||
ClassifyParsingConfiguration,
|
||||
} from "../src/classify.js";
|
||||
import { Document } from "@llamaindex/core/schema";
|
||||
import { fs } from "@llamaindex/env";
|
||||
import { ExtractConfig } from "../src/api.js";
|
||||
@@ -499,25 +502,29 @@ describe("Integration Tests", () => {
|
||||
process.env.LLAMA_CLOUD_API_KEY!,
|
||||
"https://api.cloud.llamaindex.ai",
|
||||
);
|
||||
const testContent =
|
||||
`A Fox one day spied a beautiful bunch of ripe grapes hanging from a vine trained along the branches of a tree. The grapes seemed ready to burst with juice, and the Fox's mouth watered as he gazed longingly at them. The bunch hung from a high branch, and the Fox had to jump for it. The first time he jumped he missed it by a long way. So he walked off a short distance and took a running leap at it, only to fall short once more. Again and again he tried, but in vain. Now he sat down and looked at the grapes in disgust. "What a fool I am," he said. "Here I am wearing myself out to get a bunch of sour grapes that are not worth gaping for." And off he walked very, very scornfully.There are many who pretend to despise and belittle that which is beyond their reach.`;
|
||||
const testContent = `A Fox one day spied a beautiful bunch of ripe grapes hanging from a vine trained along the branches of a tree. The grapes seemed ready to burst with juice, and the Fox's mouth watered as he gazed longingly at them. The bunch hung from a high branch, and the Fox had to jump for it. The first time he jumped he missed it by a long way. So he walked off a short distance and took a running leap at it, only to fall short once more. Again and again he tried, but in vain. Now he sat down and looked at the grapes in disgust. "What a fool I am," he said. "Here I am wearing myself out to get a bunch of sour grapes that are not worth gaping for." And off he walked very, very scornfully.There are many who pretend to despise and belittle that which is beyond their reach.`;
|
||||
const testFilePath = "the_fox_and_the_grapes.md";
|
||||
|
||||
await fs.writeFile(testFilePath, new TextEncoder().encode(testContent));
|
||||
|
||||
const rules: ClassifierRule[] = [
|
||||
{type: "fable", description: "A short story featuring animals whose aim is to teach the reader a lesson (the moral of the story)"},
|
||||
{type: "fairy_tale", description: "A mid-to-long story featuring humans, magic creatures and other characters, whose main aim is to entertain the readers."}
|
||||
]
|
||||
{
|
||||
type: "fable",
|
||||
description:
|
||||
"A short story featuring animals whose aim is to teach the reader a lesson (the moral of the story)",
|
||||
},
|
||||
{
|
||||
type: "fairy_tale",
|
||||
description:
|
||||
"A mid-to-long story featuring humans, magic creatures and other characters, whose main aim is to entertain the readers.",
|
||||
},
|
||||
];
|
||||
|
||||
const parsingConfig: ClassifyParsingConfiguration = {lang: "en"}
|
||||
const parsingConfig: ClassifyParsingConfiguration = { lang: "en" };
|
||||
|
||||
const result = await classifyClient.classify(
|
||||
rules,
|
||||
parsingConfig,
|
||||
undefined,
|
||||
["the_fox_and_the_grapes.md"]
|
||||
);
|
||||
const result = await classifyClient.classify(rules, parsingConfig, {
|
||||
filePaths: ["the_fox_and_the_grapes.md"],
|
||||
});
|
||||
expect("items" in result).toBeTruthy();
|
||||
expect(result.items.length).toBeGreaterThan(0);
|
||||
expect("result" in result.items[0]).toBeTruthy();
|
||||
@@ -527,7 +534,7 @@ describe("Integration Tests", () => {
|
||||
const resultBuffer = await classifyClient.classify(
|
||||
rules,
|
||||
parsingConfig,
|
||||
[buffer],
|
||||
{ fileContents: [buffer] },
|
||||
);
|
||||
expect("items" in resultBuffer).toBeTruthy();
|
||||
expect(resultBuffer.items.length).toBeGreaterThan(0);
|
||||
@@ -535,9 +542,11 @@ describe("Integration Tests", () => {
|
||||
expect(resultBuffer.items[0].result!.type === "fable").toBeTruthy();
|
||||
|
||||
try {
|
||||
await fs.unlink("the_fox_and_the_grapes.md")
|
||||
} catch(err) {
|
||||
console.log(`Unable to delete file the_fox_and_the_grapes.md because of ${err}`)
|
||||
await fs.unlink("the_fox_and_the_grapes.md");
|
||||
} catch (err) {
|
||||
console.log(
|
||||
`Unable to delete file the_fox_and_the_grapes.md because of ${err}`,
|
||||
);
|
||||
}
|
||||
},
|
||||
60000,
|
||||
|
||||
Reference in New Issue
Block a user