mirror of
https://github.com/run-llama/llama_cloud_services.git
synced 2026-07-21 12:05:23 -04:00
Compare commits
21 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b8f95c6a3d | |||
| 878605b347 | |||
| 22130a35d7 | |||
| ed4b55305c | |||
| 4b9d48f5e6 | |||
| ae6d6c276f | |||
| 25f9efd5bc | |||
| 812e2f7d72 | |||
| d7864afe3f | |||
| ade8d027a5 | |||
| 997bcc8531 | |||
| 8be554c234 | |||
| f777cab0c5 | |||
| b9b83c953d | |||
| 3ec7024626 | |||
| d5b18a03fa | |||
| 18dd04b6de | |||
| 685a5e6ccc | |||
| 576c3d9076 | |||
| c8321d2bc5 | |||
| 131bbed7aa |
@@ -0,0 +1,6 @@
|
||||
---
|
||||
"llama_parse": patch
|
||||
"llama-cloud-services-py": patch
|
||||
---
|
||||
|
||||
Remove extension filter
|
||||
+1
-1
@@ -398,6 +398,6 @@ Another option (orthogonal to the above) is to break the document into smaller s
|
||||
## Additional Resources
|
||||
|
||||
- [Extract Documentation](https://docs.cloud.llamaindex.ai/llamaextract/getting_started) - Details on Extract features, API and examples.
|
||||
- [Example Notebook](docs/examples-py/extract/resume_screening.ipynb) - Detailed walkthrough of resume parsing
|
||||
- [Example Notebook](examples/extract/resume_screening.ipynb) - Detailed walkthrough of resume parsing
|
||||
- [Example Application with TypeScript](./examples-ts/extract/) - End-to-end examples using LlamaExtract TypeScript client.
|
||||
- [Discord Community](https://discord.com/invite/eN6D2HQ4aX) - Get help and share feedback
|
||||
|
||||
@@ -97,7 +97,7 @@ for page in result.pages:
|
||||
print(page.structuredData)
|
||||
```
|
||||
|
||||
See more details about the result object in the [example notebook](./docs/examples-py/parse/demo_json_tour.ipynb).
|
||||
See more details about the result object in the [example notebook](./examples/parse/demo_json_tour.ipynb).
|
||||
|
||||
### Using with file object / bytes
|
||||
|
||||
@@ -153,10 +153,10 @@ Full documentation for `SimpleDirectoryReader` can be found on the [LlamaIndex D
|
||||
|
||||
Several end-to-end indexing examples can be found in the examples folder
|
||||
|
||||
- [Getting Started](docs/examples-py/parse/demo_basic.ipynb)
|
||||
- [Advanced RAG Example](docs/examples-py/parse/demo_advanced.ipynb)
|
||||
- [Raw API Usage](docs/examples-py/parse/demo_api.ipynb)
|
||||
- [Result Object Tour](docs/examples-py/parse/demo_json_tour.ipynb)
|
||||
- [Getting Started](examples/parse/demo_basic.ipynb)
|
||||
- [Advanced RAG Example](examples/parse/demo_advanced.ipynb)
|
||||
- [Raw API Usage](examples/parse/demo_api.ipynb)
|
||||
- [Result Object Tour](examples/parse/demo_json_tour.ipynb)
|
||||
|
||||
## Documentation
|
||||
|
||||
|
||||
Generated
+48
-3260
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,11 @@
|
||||
# llama-cloud-services-py
|
||||
|
||||
## 0.6.89
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- b9b83c9: Parse bounding boxes from extract jobs results in agent data
|
||||
|
||||
## 0.6.88
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -11,6 +11,9 @@ from .schema import (
|
||||
InvalidExtractionData,
|
||||
ExtractedFieldMetadata,
|
||||
ExtractedFieldMetaDataDict,
|
||||
FieldCitation,
|
||||
BoundingBox,
|
||||
PageDimensions,
|
||||
)
|
||||
from .client import AsyncAgentDataClient
|
||||
|
||||
@@ -28,4 +31,7 @@ __all__ = [
|
||||
"InvalidExtractionData",
|
||||
"ExtractedFieldMetadata",
|
||||
"ExtractedFieldMetaDataDict",
|
||||
"FieldCitation",
|
||||
"BoundingBox",
|
||||
"PageDimensions",
|
||||
]
|
||||
|
||||
@@ -174,6 +174,22 @@ class TypedAgentDataItems(BaseModel, Generic[AgentDataT]):
|
||||
)
|
||||
|
||||
|
||||
class BoundingBox(BaseModel):
|
||||
"""Bounding box coordinates for a citation location on a page."""
|
||||
|
||||
x: float = Field(description="X coordinate of the bounding box origin")
|
||||
y: float = Field(description="Y coordinate of the bounding box origin")
|
||||
w: float = Field(description="Width of the bounding box")
|
||||
h: float = Field(description="Height of the bounding box")
|
||||
|
||||
|
||||
class PageDimensions(BaseModel):
|
||||
"""Dimensions of a page in the source document."""
|
||||
|
||||
width: float = Field(description="Width of the page")
|
||||
height: float = Field(description="Height of the page")
|
||||
|
||||
|
||||
class FieldCitation(BaseModel):
|
||||
page: Optional[int] = Field(
|
||||
None, description="The page number that the field occurred on"
|
||||
@@ -182,6 +198,14 @@ class FieldCitation(BaseModel):
|
||||
None,
|
||||
description="The original text this field's value was derived from",
|
||||
)
|
||||
bounding_boxes: Optional[List[BoundingBox]] = Field(
|
||||
None,
|
||||
description="Bounding boxes indicating where the citation appears on the page",
|
||||
)
|
||||
page_dimensions: Optional[PageDimensions] = Field(
|
||||
None,
|
||||
description="Dimensions of the page containing the citation",
|
||||
)
|
||||
|
||||
|
||||
class ExtractedFieldMetadata(BaseModel):
|
||||
@@ -201,6 +225,10 @@ class ExtractedFieldMetadata(BaseModel):
|
||||
None,
|
||||
description="The confidence score for the field based on the extracted text only",
|
||||
)
|
||||
parsing_confidence: Optional[float] = Field(
|
||||
None,
|
||||
description="The confidence score for the field based on the parsing/OCR quality",
|
||||
)
|
||||
citation: Optional[List[FieldCitation]] = Field(
|
||||
None,
|
||||
description="The citation for the field, including page number and matching text",
|
||||
|
||||
@@ -751,11 +751,9 @@ class LlamaParse(BasePydanticReader):
|
||||
file_path = str(file_input)
|
||||
file_ext = os.path.splitext(file_path)[1].lower()
|
||||
if file_ext not in SUPPORTED_FILE_TYPES:
|
||||
raise Exception(
|
||||
f"Currently, only the following file types are supported: {SUPPORTED_FILE_TYPES}\n"
|
||||
f"Current file type: {file_ext}"
|
||||
)
|
||||
mime_type = mimetypes.guess_type(file_path)[0]
|
||||
mime_type = "application/octet-stream"
|
||||
else:
|
||||
mime_type = mimetypes.guess_type(file_path)[0]
|
||||
# Open the file here for the duration of the async context
|
||||
# load data, set the mime type
|
||||
fs = fs or get_default_fs()
|
||||
|
||||
@@ -1,5 +1,12 @@
|
||||
# llama_parse
|
||||
|
||||
## 0.6.89
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [b9b83c9]
|
||||
- llama-cloud-services-py@0.6.89
|
||||
|
||||
## 0.6.88
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -146,9 +146,9 @@ Full documentation for `SimpleDirectoryReader` can be found on the [LlamaIndex D
|
||||
|
||||
Several end-to-end indexing examples can be found in the examples folder
|
||||
|
||||
- [Getting Started](/docs/examples-py/parse/demo_basic.ipynb)
|
||||
- [Advanced RAG Example](/docs/examples-py/parse/demo_advanced.ipynb)
|
||||
- [Raw API Usage](/docs/examples-py/parse/demo_api.ipynb)
|
||||
- [Getting Started](../../examples/parse/demo_basic.ipynb)
|
||||
- [Advanced RAG Example](../../examples/parse/demo_advanced.ipynb)
|
||||
- [Raw API Usage](../../examples/parse/demo_api.ipynb)
|
||||
|
||||
## Documentation
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "llama_parse",
|
||||
"version": "0.6.88",
|
||||
"version": "0.6.89",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"private": false,
|
||||
|
||||
@@ -11,13 +11,13 @@ dev = [
|
||||
|
||||
[project]
|
||||
name = "llama-parse"
|
||||
version = "0.6.88"
|
||||
version = "0.6.89"
|
||||
description = "Parse files into RAG-Optimized formats."
|
||||
authors = [{name = "Logan Markewich", email = "logan@llamaindex.ai"}]
|
||||
requires-python = ">=3.9,<4.0"
|
||||
readme = "README.md"
|
||||
license = "MIT"
|
||||
dependencies = ["llama-cloud-services>=0.6.88"]
|
||||
dependencies = ["llama-cloud-services>=0.6.89"]
|
||||
|
||||
[project.scripts]
|
||||
llama-parse = "llama_parse.cli.main:parse"
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "llama-cloud-services-py",
|
||||
"version": "0.6.88",
|
||||
"version": "0.6.89",
|
||||
"private": false,
|
||||
"license": "MIT",
|
||||
"scripts": {},
|
||||
|
||||
+1
-1
@@ -23,7 +23,7 @@ dev = [
|
||||
|
||||
[project]
|
||||
name = "llama-cloud-services"
|
||||
version = "0.6.88"
|
||||
version = "0.6.89"
|
||||
description = "Tailored SDK clients for LlamaCloud services."
|
||||
authors = [{name = "Logan Markewich", email = "logan@runllama.ai"}]
|
||||
requires-python = ">=3.9,<4.0"
|
||||
|
||||
@@ -11,10 +11,12 @@ from llama_cloud.types.aggregate_group import AggregateGroup
|
||||
from pydantic import BaseModel, Field, ValidationError
|
||||
|
||||
from llama_cloud_services.beta.agent_data.schema import (
|
||||
BoundingBox,
|
||||
ExtractedData,
|
||||
ExtractedFieldMetadata,
|
||||
FieldCitation,
|
||||
InvalidExtractionData,
|
||||
PageDimensions,
|
||||
TypedAgentData,
|
||||
TypedAggregateGroup,
|
||||
calculate_overall_confidence,
|
||||
@@ -663,3 +665,69 @@ def test_field_conflict_in_schema():
|
||||
assert isinstance(
|
||||
extracted["majority_opinion"]["reasoning"], ExtractedFieldMetadata
|
||||
)
|
||||
|
||||
|
||||
def test_parse_extracted_field_metadata_with_bounding_boxes():
|
||||
"""Test parse_extracted_field_metadata with bounding boxes and page dimensions."""
|
||||
raw_metadata = {
|
||||
"document_type": {
|
||||
"citation": [
|
||||
{
|
||||
"page": 1,
|
||||
"matching_text": "FACTURE ORIGINALE",
|
||||
"bounding_boxes": [{"x": 77.28, "y": 615.12, "w": 70.6, "h": 7.2}],
|
||||
"page_dimensions": {"width": 222.24, "height": 736.56},
|
||||
}
|
||||
],
|
||||
"parsing_confidence": 1.0,
|
||||
"extraction_confidence": 0.7252506422636493,
|
||||
"confidence": 0.7252506422636493,
|
||||
},
|
||||
"summary": {
|
||||
"citation": [
|
||||
{
|
||||
"page": 1,
|
||||
"matching_text": "FACTURE ORIGINALE",
|
||||
"bounding_boxes": [{"x": 77.28, "y": 615.12, "w": 70.6, "h": 7.2}],
|
||||
"page_dimensions": {"width": 222.24, "height": 736.56},
|
||||
},
|
||||
{
|
||||
"page": 1,
|
||||
"matching_text": "Café filtre assiette — $1.90",
|
||||
"bounding_boxes": [
|
||||
{"x": 10.56, "y": 172.83, "w": 171.85, "h": 497.01}
|
||||
],
|
||||
"page_dimensions": {"width": 222.24, "height": 736.56},
|
||||
},
|
||||
],
|
||||
"parsing_confidence": 1.0,
|
||||
"extraction_confidence": 0.5700013128334419,
|
||||
"confidence": 0.5700013128334419,
|
||||
},
|
||||
}
|
||||
|
||||
result = parse_extracted_field_metadata(raw_metadata)
|
||||
|
||||
# Verify document_type citation with bounding boxes
|
||||
assert isinstance(result["document_type"], ExtractedFieldMetadata)
|
||||
assert result["document_type"].parsing_confidence == 1.0
|
||||
assert result["document_type"].extraction_confidence == 0.7252506422636493
|
||||
assert result["document_type"].confidence == 0.7252506422636493
|
||||
assert len(result["document_type"].citation) == 1
|
||||
|
||||
citation = result["document_type"].citation[0]
|
||||
assert citation.page == 1
|
||||
assert citation.matching_text == "FACTURE ORIGINALE"
|
||||
assert len(citation.bounding_boxes) == 1
|
||||
assert citation.bounding_boxes[0] == BoundingBox(x=77.28, y=615.12, w=70.6, h=7.2)
|
||||
assert citation.page_dimensions == PageDimensions(width=222.24, height=736.56)
|
||||
|
||||
# Verify summary citation with multiple bounding boxes
|
||||
assert isinstance(result["summary"], ExtractedFieldMetadata)
|
||||
assert len(result["summary"].citation) == 2
|
||||
assert result["summary"].citation[0].bounding_boxes[0].x == 77.28
|
||||
assert result["summary"].citation[1].bounding_boxes[0].x == 10.56
|
||||
|
||||
# Verify round-trip serialization
|
||||
result2 = parse_extracted_field_metadata(result)
|
||||
assert result2 == result
|
||||
|
||||
Generated
+2
-2
@@ -1,5 +1,5 @@
|
||||
version = 1
|
||||
revision = 2
|
||||
revision = 3
|
||||
requires-python = ">=3.9, <4.0"
|
||||
resolution-markers = [
|
||||
"python_full_version >= '3.14'",
|
||||
@@ -1609,7 +1609,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "llama-cloud-services"
|
||||
version = "0.6.85"
|
||||
version = "0.6.88"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
|
||||
|
||||
@@ -1,5 +1,38 @@
|
||||
# llama-cloud-services
|
||||
|
||||
## 0.5.3
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- d7864af: bugfixes in retry logic for LlamaExtract and LlamaClassify
|
||||
|
||||
## 0.5.2
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 997bcc8: Add types for bounding boxes
|
||||
|
||||
## 0.5.1
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- d5b18a0: Fix publishing
|
||||
|
||||
## 0.5.0
|
||||
|
||||
### Minor Changes
|
||||
|
||||
- 576c3d9: feat: support zod v4 & v3
|
||||
|
||||
Adds support for zod v4 while maintaining backward compatibility with v3.
|
||||
- Updated zod peer dependency to accept both v3 and v4: `^3.25.76 || ^4.0.0`
|
||||
- Migrated all import statements to use `zod/v4` import path for compatibility
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- c8321d2: Improve parse results polling
|
||||
- 576c3d9: Support zod v3 an v4
|
||||
|
||||
## 0.4.3
|
||||
|
||||
### Patch Changes
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "llama-cloud-services",
|
||||
"version": "0.4.3",
|
||||
"version": "0.5.3",
|
||||
"type": "module",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
"get-openapi": "node ./scripts/get-openapi.js",
|
||||
"generate": "./node_modules/.bin/openapi-ts",
|
||||
"build": "pnpm run generate && bunchee",
|
||||
"build": "bunchee",
|
||||
"dev": "bunchee --watch",
|
||||
"lint": "eslint src/ --ignore-pattern client/*.ts --no-warn-ignored",
|
||||
"format": "prettier --write ./src/ tests/",
|
||||
@@ -116,9 +116,9 @@
|
||||
"@eslint/js": "^9.32.0",
|
||||
"@hey-api/client-fetch": "^0.10.1",
|
||||
"@hey-api/openapi-ts": "^0.67.5",
|
||||
"@llamaindex/core": "^0.6.19",
|
||||
"@llamaindex/core": "^0.6.22",
|
||||
"@llamaindex/env": "^0.1.30",
|
||||
"@llamaindex/workflow-core": "^0.4.1",
|
||||
"@llamaindex/workflow-core": "^1.3.3",
|
||||
"@types/node": "^20.19.9",
|
||||
"@typescript-eslint/eslint-plugin": "^8.38.0",
|
||||
"@typescript-eslint/parser": "^8.38.0",
|
||||
@@ -131,18 +131,19 @@
|
||||
"turbo": "^2.5.5",
|
||||
"typescript": "^5.8.3",
|
||||
"typescript-eslint": "^8.38.0",
|
||||
"vitest": "^2.0.0"
|
||||
"vitest": "^2.0.0",
|
||||
"zod": "^4.1.13"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@llamaindex/core": "^0.6.19",
|
||||
"@llamaindex/env": "^0.1.30",
|
||||
"@llamaindex/workflow-core": "^0.4.1"
|
||||
"@llamaindex/workflow-core": "^1.3.3",
|
||||
"zod": "^3.25.0 || ^4.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"ajv": "^8.17.1",
|
||||
"file-type": "^21.0.0",
|
||||
"p-retry": "^6.2.1",
|
||||
"zod": "^3.25.76"
|
||||
"p-retry": "^6.2.1"
|
||||
},
|
||||
"packageManager": "pnpm@10.8.1"
|
||||
}
|
||||
|
||||
@@ -2,11 +2,14 @@ export { AgentClient, createAgentDataClient } from "./client";
|
||||
|
||||
export type {
|
||||
AggregateAgentDataOptions,
|
||||
BoundingBox,
|
||||
ComparisonOperator,
|
||||
ExtractedData,
|
||||
ExtractedFieldMetadata,
|
||||
ExtractedFieldMetadataDict,
|
||||
FieldCitation,
|
||||
FilterOperation,
|
||||
PageDimensions,
|
||||
SearchAgentDataOptions,
|
||||
StatusType,
|
||||
TypedAgentData,
|
||||
|
||||
@@ -28,6 +28,44 @@ export type ComparisonOperator =
|
||||
*/
|
||||
export type FilterOperation = RawFilterOperation;
|
||||
|
||||
/**
|
||||
* Bounding box coordinates for a citation location on a page
|
||||
*/
|
||||
export interface BoundingBox {
|
||||
/** X coordinate of the bounding box origin */
|
||||
x: number;
|
||||
/** Y coordinate of the bounding box origin */
|
||||
y: number;
|
||||
/** Width of the bounding box */
|
||||
w: number;
|
||||
/** Height of the bounding box */
|
||||
h: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Dimensions of a page in the source document
|
||||
*/
|
||||
export interface PageDimensions {
|
||||
/** Width of the page */
|
||||
width: number;
|
||||
/** Height of the page */
|
||||
height: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Citation information for an extracted field
|
||||
*/
|
||||
export interface FieldCitation {
|
||||
/** The page number that the field occurred on */
|
||||
page?: number;
|
||||
/** The original text this field's value was derived from */
|
||||
matching_text?: string;
|
||||
/** Bounding boxes indicating where the citation appears on the page */
|
||||
bounding_boxes?: BoundingBox[];
|
||||
/** Dimensions of the page containing the citation */
|
||||
page_dimensions?: PageDimensions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Metadata for an extracted field, including confidence and citation information
|
||||
*/
|
||||
@@ -38,16 +76,11 @@ export interface ExtractedFieldMetadata {
|
||||
confidence?: number;
|
||||
/** The confidence score for the field based on the extracted text only */
|
||||
extraction_confidence?: number;
|
||||
/** The confidence score for the field based on the parsing/OCR quality */
|
||||
parsing_confidence?: number;
|
||||
citation?: FieldCitation[];
|
||||
}
|
||||
|
||||
export interface FieldCitation {
|
||||
/** The page number that the field occurred on */
|
||||
page?: number;
|
||||
/** The original text this field's value was derived from */
|
||||
matching_text?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Dictionary mapping field names to their metadata
|
||||
* Values can be ExtractedFieldMetadata objects, nested dictionaries, or arrays
|
||||
|
||||
@@ -108,20 +108,19 @@ async function pollForJobCompletion({
|
||||
}
|
||||
const response =
|
||||
await getClassifyJobApiV1ClassifierJobsClassifyJobIdGet(jobOptions);
|
||||
if (!response.response.ok) {
|
||||
numIterations++;
|
||||
}
|
||||
if (typeof response.data != "undefined") {
|
||||
status = response.data.status as StatusEnum;
|
||||
if (status == StatusEnum.CANCELLED || status == StatusEnum.ERROR) {
|
||||
throw new Error("There was an error during the classification job.");
|
||||
} else if (status == StatusEnum.SUCCESS) {
|
||||
throw new Error("There was an error extracting data from your file.");
|
||||
} else if (
|
||||
status == StatusEnum.SUCCESS ||
|
||||
status == StatusEnum.PARTIAL_SUCCESS
|
||||
) {
|
||||
return true;
|
||||
} else {
|
||||
numIterations++;
|
||||
await sleep(interval * 1000);
|
||||
}
|
||||
}
|
||||
numIterations++;
|
||||
await sleep(interval * 1000);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,7 +168,7 @@ async function getJobResult({
|
||||
retries++;
|
||||
await sleep(retryInterval * 1000);
|
||||
}
|
||||
if (typeof response.data != "undefined") {
|
||||
if (response.response.ok && typeof response.data != "undefined") {
|
||||
return response.data as ClassifyJobResults;
|
||||
} else {
|
||||
throw new Error(
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// This file is auto-generated by @hey-api/openapi-ts
|
||||
|
||||
import { z } from "zod";
|
||||
import { z } from "zod/v4";
|
||||
|
||||
export const zNoneSegmentationConfig = z.object({
|
||||
mode: z.literal("none").optional().default("none"),
|
||||
@@ -770,7 +770,7 @@ export const zMetadataFilter = z.object({
|
||||
|
||||
export const zFilterCondition: z.ZodTypeAny = z.enum(["and", "or", "not"]);
|
||||
|
||||
export const zMetadataFilters: z.AnyZodObject = z.object({
|
||||
export const zMetadataFilters: z.ZodObject<z.ZodRawShape> = z.object({
|
||||
filters: z.array(z.unknown()),
|
||||
condition: z.union([zFilterCondition, z.null()]).optional(),
|
||||
});
|
||||
@@ -782,7 +782,7 @@ export const zRetrievalMode: z.ZodTypeAny = z.enum([
|
||||
"auto_routed",
|
||||
]);
|
||||
|
||||
export const zPresetRetrievalParams: z.AnyZodObject = z.object({
|
||||
export const zPresetRetrievalParams: z.ZodObject<z.ZodRawShape> = z.object({
|
||||
dense_similarity_top_k: z
|
||||
.union([z.number().int().gte(1).lte(100), z.null()])
|
||||
.optional(),
|
||||
@@ -825,7 +825,7 @@ export const zSupportedLlmModelNames: z.ZodTypeAny = z.enum([
|
||||
"VERTEX_AI_CLAUDE_3_5_SONNET_V2",
|
||||
]);
|
||||
|
||||
export const zLlmParameters: z.AnyZodObject = z.object({
|
||||
export const zLlmParameters: z.ZodObject<z.ZodRawShape> = z.object({
|
||||
model_name: zSupportedLlmModelNames.optional(),
|
||||
system_prompt: z.union([z.string().max(3000), z.null()]).optional(),
|
||||
temperature: z.union([z.number(), z.null()]).optional(),
|
||||
@@ -834,7 +834,7 @@ export const zLlmParameters: z.AnyZodObject = z.object({
|
||||
class_name: z.string().optional().default("base_component"),
|
||||
});
|
||||
|
||||
export const zChatData: z.AnyZodObject = z.object({
|
||||
export const zChatData: z.ZodObject<z.ZodRawShape> = z.object({
|
||||
retrieval_parameters: zPresetRetrievalParams.optional(),
|
||||
llm_parameters: z.union([zLlmParameters, z.null()]).optional(),
|
||||
class_name: z.string().optional().default("base_component"),
|
||||
@@ -2141,7 +2141,7 @@ export const zTextItem = z.object({
|
||||
value: z.string(),
|
||||
});
|
||||
|
||||
export const zListItem: z.AnyZodObject = z.object({
|
||||
export const zListItem: z.ZodObject<z.ZodRawShape> = z.object({
|
||||
type: z.literal("list").optional().default("list"),
|
||||
bBox: z.union([z.unknown(), z.null()]).optional(),
|
||||
items: z.array(z.unknown()),
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { workflowEvent } from "@llamaindex/workflow-core";
|
||||
import { zodEvent } from "@llamaindex/workflow-core/util/zod";
|
||||
import { z } from "zod";
|
||||
import { z } from "zod/v4";
|
||||
import { parseFormSchema } from "./schema";
|
||||
|
||||
export const uploadEvent = zodEvent(
|
||||
|
||||
@@ -296,20 +296,16 @@ async function pollForJobCompletion(
|
||||
return false;
|
||||
}
|
||||
const response = await getJobApiV1ExtractionJobsJobIdGet(jobOptions);
|
||||
if (!response.response.ok) {
|
||||
numIterations++;
|
||||
}
|
||||
if (typeof response.data != "undefined") {
|
||||
status = response.data.status as StatusEnum;
|
||||
if (status == StatusEnum.CANCELLED || status == StatusEnum.ERROR) {
|
||||
throw new Error("There was an error extracting data from your file.");
|
||||
} else if (status == StatusEnum.SUCCESS) {
|
||||
return true;
|
||||
} else {
|
||||
numIterations++;
|
||||
await sleep(interval * 1000);
|
||||
}
|
||||
}
|
||||
numIterations++;
|
||||
await sleep(interval * 1000);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -350,7 +346,7 @@ async function getJobResult(
|
||||
retries++;
|
||||
await sleep(retryInterval * 1000);
|
||||
}
|
||||
if (typeof response.data != "undefined") {
|
||||
if (response.response.ok && typeof response.data != "undefined") {
|
||||
return {
|
||||
data: response.data.data,
|
||||
extractionMetadata: response.data.extraction_metadata,
|
||||
|
||||
@@ -2,7 +2,7 @@ import type { JSONValue } from "@llamaindex/core/global";
|
||||
import type { ToolMetadata } from "@llamaindex/core/llms";
|
||||
import type { BaseQueryEngine } from "@llamaindex/core/query-engine";
|
||||
import { tool } from "@llamaindex/core/tools";
|
||||
import { z } from "zod";
|
||||
import { z } from "zod/v4";
|
||||
|
||||
const DEFAULT_NAME = "llama_cloud_index_tool";
|
||||
const DEFAULT_DESCRIPTION =
|
||||
@@ -21,9 +21,7 @@ export function createQueryEngineTool(
|
||||
name: metadata?.name ?? DEFAULT_NAME,
|
||||
description: metadata?.description ?? DEFAULT_DESCRIPTION,
|
||||
parameters: z.object({
|
||||
query: z.string({
|
||||
description: "The query to search for",
|
||||
}),
|
||||
query: z.string().describe("The query to search for"),
|
||||
}),
|
||||
execute: async ({ query }) => {
|
||||
const response = await queryEngine.query({ query });
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
import { type Client, createClient, createConfig } from "@hey-api/client-fetch";
|
||||
import { type FailedAttemptError } from "p-retry";
|
||||
import { Document, FileReader } from "@llamaindex/core/schema";
|
||||
import { fs, getEnv, path } from "@llamaindex/env";
|
||||
import {
|
||||
type BodyUploadFileApiParsingUploadPost,
|
||||
type BodyUploadFileApiV1ParsingUploadPost,
|
||||
type FailPageMode,
|
||||
type ParserLanguages,
|
||||
type ParsingMode,
|
||||
@@ -32,6 +33,33 @@ type WriteStream = {
|
||||
// eslint-disable-next-line no-var
|
||||
var process: any;
|
||||
|
||||
function handleFailedAttempt(
|
||||
error: FailedAttemptError,
|
||||
jobId: string,
|
||||
verbose: boolean,
|
||||
) {
|
||||
// Retry only on 5XX or socket errors.
|
||||
const status = (error.cause as any)?.response?.status;
|
||||
if (
|
||||
!(
|
||||
(status && status >= 500 && status < 600) ||
|
||||
((error.cause as any)?.code &&
|
||||
((error.cause as any).code === "ECONNRESET" ||
|
||||
(error.cause as any).code === "ETIMEDOUT" ||
|
||||
(error.cause as any).code === "ECONNREFUSED")) ||
|
||||
(status && status === 404)
|
||||
)
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (verbose) {
|
||||
console.warn(
|
||||
`Attempting to get job ${jobId} result (attempt ${error.attemptNumber}) failed. Retrying...`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Represents a reader for parsing files using the LlamaParse API.
|
||||
* See https://github.com/run-llama/llama_parse
|
||||
@@ -188,6 +216,16 @@ export class LlamaParseReader extends FileReader {
|
||||
extract_printed_page_number?: boolean | undefined;
|
||||
tier?: string | undefined;
|
||||
version?: string | undefined;
|
||||
layout_aware?: boolean | undefined;
|
||||
line_level_bounding_box?: boolean | undefined;
|
||||
specialized_image_parsing?: boolean | undefined;
|
||||
aggressive_table_extraction?: boolean | undefined;
|
||||
preserve_very_small_text?: boolean | undefined;
|
||||
spreadsheet_force_formula_computation?: boolean | undefined;
|
||||
inline_images_in_markdown?: boolean | undefined;
|
||||
keep_page_separator_when_merging_tables?: boolean | undefined;
|
||||
remove_hidden_text?: boolean | undefined;
|
||||
presentation_out_of_bounds_content?: boolean | undefined;
|
||||
|
||||
constructor(
|
||||
params: Partial<Omit<LlamaParseReader, "language" | "apiKey">> & {
|
||||
@@ -387,11 +425,25 @@ export class LlamaParseReader extends FileReader {
|
||||
extract_printed_page_number: this.extract_printed_page_number,
|
||||
tier: this.tier,
|
||||
version: this.version,
|
||||
layout_aware: this.layout_aware,
|
||||
line_level_bounding_box: this.line_level_bounding_box,
|
||||
specialized_image_parsing: this.specialized_image_parsing,
|
||||
aggressive_table_extraction: this.aggressive_table_extraction,
|
||||
preserve_very_small_text: this.preserve_very_small_text,
|
||||
spreadsheet_force_formula_computation:
|
||||
this.spreadsheet_force_formula_computation,
|
||||
inline_images_in_markdown: this.inline_images_in_markdown,
|
||||
webhook_configurations: undefined,
|
||||
keep_page_separator_when_merging_tables:
|
||||
this.keep_page_separator_when_merging_tables,
|
||||
remove_hidden_text: this.remove_hidden_text,
|
||||
presentation_out_of_bounds_content:
|
||||
this.presentation_out_of_bounds_content,
|
||||
} satisfies {
|
||||
[Key in keyof BodyUploadFileApiParsingUploadPost]-?:
|
||||
| BodyUploadFileApiParsingUploadPost[Key]
|
||||
[Key in keyof BodyUploadFileApiV1ParsingUploadPost]-?:
|
||||
| BodyUploadFileApiV1ParsingUploadPost[Key]
|
||||
| undefined;
|
||||
} as unknown as BodyUploadFileApiParsingUploadPost;
|
||||
} as unknown as BodyUploadFileApiV1ParsingUploadPost;
|
||||
|
||||
const response = await uploadFileApiV1ParsingUploadPost({
|
||||
client: this.#client,
|
||||
@@ -443,26 +495,8 @@ export class LlamaParseReader extends FileReader {
|
||||
}),
|
||||
{
|
||||
retries: this.maxErrorCount,
|
||||
onFailedAttempt: (error) => {
|
||||
// Retry only on 5XX or socket errors.
|
||||
const status = (error.cause as any)?.response?.status;
|
||||
if (
|
||||
!(
|
||||
(status && status >= 500 && status < 600) ||
|
||||
((error.cause as any)?.code &&
|
||||
((error.cause as any).code === "ECONNRESET" ||
|
||||
(error.cause as any).code === "ETIMEDOUT" ||
|
||||
(error.cause as any).code === "ECONNREFUSED"))
|
||||
)
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
if (this.verbose) {
|
||||
console.warn(
|
||||
`Attempting to get job ${jobId} result (attempt ${error.attemptNumber}) failed. Retrying...`,
|
||||
);
|
||||
}
|
||||
},
|
||||
onFailedAttempt: (error) =>
|
||||
handleFailedAttempt(error, jobId, this.verbose),
|
||||
},
|
||||
);
|
||||
} catch (e: any) {
|
||||
@@ -475,49 +509,69 @@ export class LlamaParseReader extends FileReader {
|
||||
const status = (data as Record<string, unknown>)["status"];
|
||||
|
||||
if (status === "SUCCESS") {
|
||||
let resultData;
|
||||
switch (resultType) {
|
||||
case "json": {
|
||||
resultData =
|
||||
await getJobJsonResultApiV1ParsingJobJobIdResultJsonGet({
|
||||
client: this.#client,
|
||||
throwOnError: true,
|
||||
path: { job_id: jobId },
|
||||
query: {
|
||||
organization_id: this.organization_id ?? null,
|
||||
},
|
||||
signal: AbortSignal.timeout(this.maxTimeout * 1000),
|
||||
});
|
||||
break;
|
||||
const resultData = await pRetry(
|
||||
() =>
|
||||
getJobJsonResultApiV1ParsingJobJobIdResultJsonGet({
|
||||
client: this.#client,
|
||||
throwOnError: true,
|
||||
path: { job_id: jobId },
|
||||
query: {
|
||||
organization_id: this.organization_id ?? null,
|
||||
},
|
||||
signal: AbortSignal.timeout(this.maxTimeout * 1000),
|
||||
}),
|
||||
{
|
||||
retries: this.maxErrorCount,
|
||||
onFailedAttempt: (error) =>
|
||||
handleFailedAttempt(error, jobId, this.verbose),
|
||||
},
|
||||
);
|
||||
return resultData.data;
|
||||
}
|
||||
case "markdown": {
|
||||
resultData =
|
||||
await getJobResultApiV1ParsingJobJobIdResultMarkdownGet({
|
||||
client: this.#client,
|
||||
throwOnError: true,
|
||||
path: { job_id: jobId },
|
||||
query: {
|
||||
organization_id: this.organization_id ?? null,
|
||||
},
|
||||
signal: AbortSignal.timeout(this.maxTimeout * 1000),
|
||||
});
|
||||
break;
|
||||
const resultData = await pRetry(
|
||||
() =>
|
||||
getJobResultApiV1ParsingJobJobIdResultMarkdownGet({
|
||||
client: this.#client,
|
||||
throwOnError: true,
|
||||
path: { job_id: jobId },
|
||||
query: {
|
||||
organization_id: this.organization_id ?? null,
|
||||
},
|
||||
signal: AbortSignal.timeout(this.maxTimeout * 1000),
|
||||
}),
|
||||
{
|
||||
retries: this.maxErrorCount,
|
||||
onFailedAttempt: (error) =>
|
||||
handleFailedAttempt(error, jobId, this.verbose),
|
||||
},
|
||||
);
|
||||
return resultData.data;
|
||||
}
|
||||
case "text": {
|
||||
resultData =
|
||||
await getJobTextResultApiV1ParsingJobJobIdResultTextGet({
|
||||
client: this.#client,
|
||||
throwOnError: true,
|
||||
path: { job_id: jobId },
|
||||
query: {
|
||||
organization_id: this.organization_id ?? null,
|
||||
},
|
||||
signal: AbortSignal.timeout(this.maxTimeout * 1000),
|
||||
});
|
||||
break;
|
||||
const resultData = await pRetry(
|
||||
() =>
|
||||
getJobTextResultApiV1ParsingJobJobIdResultTextGet({
|
||||
client: this.#client,
|
||||
throwOnError: true,
|
||||
path: { job_id: jobId },
|
||||
query: {
|
||||
organization_id: this.organization_id ?? null,
|
||||
},
|
||||
signal: AbortSignal.timeout(this.maxTimeout * 1000),
|
||||
}),
|
||||
{
|
||||
retries: this.maxErrorCount,
|
||||
onFailedAttempt: (error) =>
|
||||
handleFailedAttempt(error, jobId, this.verbose),
|
||||
},
|
||||
);
|
||||
|
||||
return resultData.data;
|
||||
}
|
||||
}
|
||||
return resultData.data;
|
||||
} else if (status === "PENDING") {
|
||||
if (this.verbose && tries % 10 === 0) {
|
||||
this.stdout?.write(".");
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { FailPageMode, ParserLanguages, ParsingMode } from "./client";
|
||||
|
||||
import { z } from "zod";
|
||||
import { z } from "zod/v4";
|
||||
|
||||
type Language = ParserLanguages;
|
||||
const VALUES: [Language, ...Language[]] = [
|
||||
@@ -52,9 +52,10 @@ export const parseFormSchema = z.object({
|
||||
html_remove_navigation_elements: z.boolean().optional(),
|
||||
http_proxy: z
|
||||
.string()
|
||||
.url(
|
||||
'Set a valid URL for the HTTP proxy, e.g., "http://proxy.example.com:8080"',
|
||||
)
|
||||
.url({
|
||||
error:
|
||||
'Set a valid URL for the HTTP proxy, e.g., "http://proxy.example.com:8080"',
|
||||
})
|
||||
.refine(
|
||||
(url) => {
|
||||
try {
|
||||
@@ -67,7 +68,7 @@ export const parseFormSchema = z.object({
|
||||
}
|
||||
},
|
||||
{
|
||||
message: "Invalid HTTP proxy URL",
|
||||
error: "Invalid HTTP proxy URL",
|
||||
},
|
||||
)
|
||||
.optional(),
|
||||
@@ -100,7 +101,7 @@ export const parseFormSchema = z.object({
|
||||
vendor_multimodal_model_name: z.string().optional(),
|
||||
model: z.string().optional(),
|
||||
webhook_url: z.string().url().optional(),
|
||||
parse_mode: z.nativeEnum(ParsingMode).nullable().optional(),
|
||||
parse_mode: z.enum(ParsingMode).nullable().optional(),
|
||||
system_prompt: z.string().optional(),
|
||||
system_prompt_append: z.string().optional(),
|
||||
user_prompt: z.string().optional(),
|
||||
@@ -129,7 +130,7 @@ export const parseFormSchema = z.object({
|
||||
compact_markdown_table: z.boolean().optional(),
|
||||
markdown_table_multiline_header_separator: z.string().optional(),
|
||||
page_error_tolerance: z.number().min(0).max(1).optional(),
|
||||
replace_failed_page_mode: z.nativeEnum(FailPageMode).nullable().optional(),
|
||||
replace_failed_page_mode: z.enum(FailPageMode).nullable().optional(),
|
||||
replace_failed_page_with_error_message_prefix: z.string().optional(),
|
||||
replace_failed_page_with_error_message_suffix: z.string().optional(),
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user