Compare commits

...

3 Commits

Author SHA1 Message Date
Adrian Lyjak 60ad38f253 Re-enabling this test 2025-08-01 16:11:17 -04:00
Adrian Lyjak c21cb34ff6 fix: Fix bugs in ExtractedFieldMetadata parser (#834)
* fix: Fix bugs in ExtractedFieldMetadata parser

- Wasn't recursing through lists properly
- Fix field names, names changed or I copied incorrectly
- Handle reasoning on a parent object

* version script fixes

* update versions

* skip the unrelated failing test for now
2025-08-01 16:08:16 -04:00
Adrian Lyjak e28c7b9d92 Copy extracted citations to the new repo (#832)
* Copy extracted citations to the new repo

* fix spell check

* ignore examples too

* tweak timeout

* add changes to github actions

* shrug
2025-07-31 19:34:24 +02:00
20 changed files with 1561 additions and 2553 deletions
+4 -1
View File
@@ -6,8 +6,11 @@ on:
push:
branches:
- main
paths:
- "py/**"
pull_request:
paths:
- "py/**"
env:
UV_VERSION: "0.7.20"
+9 -1
View File
@@ -1,5 +1,13 @@
name: Build Package - TypeScript
on: [pull_request]
on:
push:
branches:
- main
paths:
- "ts/**"
pull_request:
paths:
- "ts/**"
jobs:
pre_release:
+4 -2
View File
@@ -4,9 +4,11 @@ on:
push:
branches:
- main
paths:
- "ts/**"
pull_request:
branches:
- main
paths:
- "ts/**"
env:
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
+4
View File
@@ -4,7 +4,11 @@ on:
push:
branches:
- main
paths:
- "py/**"
pull_request:
paths:
- "py/**"
env:
UV_VERSION: "0.7.20"
+7 -4
View File
@@ -4,9 +4,11 @@ on:
push:
branches:
- main
paths:
- "ts/**"
pull_request:
branches:
- main
paths:
- "ts/**"
env:
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
@@ -15,7 +17,8 @@ env:
LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
jobs:
lint:
test:
name: Test - TypeScript
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
@@ -31,4 +34,4 @@ jobs:
run: pnpm install --no-frozen-lockfile
- name: Run tests
working-directory: ts/llama_cloud_services/
run: pnpm test --run
run: pnpm test
+2 -2
View File
@@ -63,13 +63,13 @@ repos:
rev: v3.0.3
hooks:
- id: prettier
exclude: uv.lock
exclude: ^(uv.lock|ts/llama_cloud_services/pnpm-lock.yaml)
- repo: https://github.com/codespell-project/codespell
rev: v2.2.6
hooks:
- id: codespell
additional_dependencies: [tomli]
exclude: ^(uv.lock|docs|ts)
exclude: ^(uv.lock|docs|ts|examples)
args:
[
"--ignore-words-list",
@@ -147,7 +147,7 @@
"documents = []\n",
"\n",
"for i, page in enumerate(pages):\n",
" # loop trough items of the page\n",
" # loop through items of the page\n",
" for item in page[\"items\"]:\n",
" document = Document(\n",
" text=item[\"md\"], extra_info={\"bbox\": item[\"bBox\"], \"page\": i}\n",
@@ -181,11 +181,15 @@ class ExtractedFieldMetadata(BaseModel):
Metadata for an extracted data field, such as confidence, and citation information.
"""
reasoning: Optional[str] = Field(
None,
description="symbol for how the citation/confidence was derived: 'INFERRED FROM TEXT', 'VERBATIM EXTRACTION'",
)
confidence: Optional[float] = Field(
None,
description="The confidence score for the field, combined with parsing confidence if applicable",
)
extracted_confidence: Optional[float] = Field(
extraction_confidence: Optional[float] = Field(
None,
description="The confidence score for the field based on the extracted text only",
)
@@ -206,42 +210,66 @@ ExtractedFieldMetaDataDict = Dict[
def parse_extracted_field_metadata(
field_metadata: dict[str, Any],
) -> ExtractedFieldMetaDataDict:
return {
k: _parse_extracted_field_metadata_recursive(v)
for k, v in field_metadata.items()
if k not in _METADATA_FIELDS_SIBLING_TO_LEAF
}
_METADATA_FIELDS_SIBLING_TO_LEAF = {"reasoning"}
def _parse_extracted_field_metadata_recursive(
field_value: Any,
additional_fields: dict[str, Any] = {},
) -> Union[ExtractedFieldMetadata, Dict[str, Any], list[Any]]:
"""
Parse the extracted field metadata into a dictionary of field names to field metadata.
"""
result: ExtractedFieldMetaDataDict = {}
for field_name, field_value in field_metadata.items():
if isinstance(field_value, ExtractedFieldMetadata):
# support running this multiple times
result[field_name] = field_value
elif isinstance(field_value, dict):
if "confidence" in field_value or "citations" in field_value:
try:
validated = ExtractedFieldMetadata.model_validate(field_value)
# grab the citation from the array. This is just an array for backwards compatibility.
if "citations" in field_value and len(field_value["citations"]) > 0:
first_citation = field_value["citations"][0]
if "page_number" in first_citation and isinstance(
first_citation["page_number"], numbers.Number
):
validated.page_number = int(first_citation["page_number"]) # type: ignore
if "matching_text" in first_citation and isinstance(
first_citation["matching_text"], str
):
validated.matching_text = first_citation["matching_text"]
result[field_name] = validated
continue
except ValidationError:
pass
result[field_name] = parse_extracted_field_metadata(field_value)
elif isinstance(field_value, list):
result[field_name] = [
parse_extracted_field_metadata(item) for item in field_value
]
else:
result[field_name] = field_value
return result
if isinstance(field_value, ExtractedFieldMetadata):
# support running this multiple times
return field_value
elif isinstance(field_value, dict):
# reasoning explicitly excluded, as it is included next to subfields, for example
# "dimensions.width" is a leaf, but there will still potentially be a "dimensions.reasoning"
indicator_fields = {"confidence", "extraction_confidence", "citation"}
if len(indicator_fields.intersection(field_value.keys())) > 0:
try:
merged = {**field_value, **additional_fields}
validated = ExtractedFieldMetadata.model_validate(merged)
# grab the citation from the array. This is just an array for backwards compatibility.
if "citation" in field_value and len(field_value["citation"]) > 0:
first_citation = field_value["citation"][0]
if "page" in first_citation and isinstance(
first_citation["page"], numbers.Number
):
validated.page_number = int(first_citation["page"]) # type: ignore
if "matching_text" in first_citation and isinstance(
first_citation["matching_text"], str
):
validated.matching_text = first_citation["matching_text"]
return validated
except ValidationError:
pass
additional_fields = {
k: v
for k, v in field_value.items()
if k in _METADATA_FIELDS_SIBLING_TO_LEAF
}
return {
k: _parse_extracted_field_metadata_recursive(v, additional_fields)
for k, v in field_value.items()
if k not in _METADATA_FIELDS_SIBLING_TO_LEAF
}
elif isinstance(field_value, list):
return [_parse_extracted_field_metadata_recursive(item) for item in field_value]
else:
raise ValueError(
f"Invalid field value: {field_value}. Expected ExtractedFieldMetadata, dict, or list"
)
class ExtractedData(BaseModel, Generic[ExtractedT]):
+2 -2
View File
@@ -11,13 +11,13 @@ dev = [
[project]
name = "llama-parse"
version = "0.6.53"
version = "0.6.54"
description = "Parse files into RAG-Optimized formats."
authors = [{name = "Logan Markewich", email = "logan@llamaindex.ai"}]
requires-python = ">=3.9,<4.0"
readme = "README.md"
license = "MIT"
dependencies = ["llama-cloud-services>=0.6.53"]
dependencies = ["llama-cloud-services>=0.6.54"]
[project.scripts]
llama-parse = "llama_parse.cli.main:parse"
+1 -1
View File
@@ -17,7 +17,7 @@ dev = [
[project]
name = "llama-cloud-services"
version = "0.6.53"
version = "0.6.54"
description = "Tailored SDK clients for LlamaCloud services."
authors = [{name = "Logan Markewich", email = "logan@runllama.ai"}]
requires-python = ">=3.9,<4.0"
@@ -230,20 +230,20 @@ def test_parse_extracted_field_metadata():
raw_metadata = {
"name": {
"confidence": 0.95,
"citations": [{"page_number": 1, "matching_text": "John Smith"}],
"citation": [{"page": 1, "matching_text": "John Smith"}],
},
"age": {
"confidence": 0.87,
"citations": [
"citation": [
{
"page_number": 2.0, # Float page number
"page": 2.0, # Float page number
"matching_text": "25 years old",
}
],
},
"email": {
"confidence": 0.92,
"citations": [], # Empty citations
"citation": [], # Empty citations
},
}
@@ -268,6 +268,110 @@ def test_parse_extracted_field_metadata():
assert result["email"].confidence == 0.92
def test_parse_extracted_field_metadata_complex():
"""Test parse_extracted_field_metadata with new citation format and reasoning field."""
raw_metadata = {
"title": {
"reasoning": "Combined key parametrics and construction from the datasheet for a structured title.",
"citation": [
{
"page": 1,
"matching_text": "PHE844/F844, Film, Metallized Polypropylene, Safety, 0.47 uF",
}
],
"extraction_confidence": 0.9470628580889779,
"confidence": 0.9470628580889779,
},
"manufacturer": {
"reasoning": "VERBATIM EXTRACTION",
"citation": [{"page": 1, "matching_text": "YAGEO KEMET"}],
"extraction_confidence": 0.9997446550976602,
"confidence": 0.9997446550976602,
},
"features": [
{
"reasoning": "VERBATIM EXTRACTION",
"citation": [
{"page": 1, "matching_text": "Features</td><td>EMI Safety"}
],
"extraction_confidence": 0.9999308195540074,
"confidence": 0.9999308195540074,
},
{
"reasoning": "VERBATIM EXTRACTION",
"citation": [
{"page": 1, "matching_text": "THB Performance</td><td>Yes"}
],
"extraction_confidence": 0.8642493886452225,
"confidence": 0.8642493886452225,
},
],
"dimensions": {
"length": {
"citation": [{"page": 1, "matching_text": "L</td><td>41mm MAX"}],
"extraction_confidence": 0.8986941382802304,
"confidence": 0.8986941382802304,
},
"width": {
"citation": [{"page": 1, "matching_text": "T</td><td>13mm MAX"}],
"extraction_confidence": 0.9999377974447091,
"confidence": 0.9999377974447091,
},
"reasoning": "VERBATIM EXTRACTION",
},
}
result = parse_extracted_field_metadata(raw_metadata)
assert result == {
"title": ExtractedFieldMetadata(
reasoning="Combined key parametrics and construction from the datasheet for a structured title.",
confidence=0.9470628580889779,
extraction_confidence=0.9470628580889779,
page_number=1,
matching_text="PHE844/F844, Film, Metallized Polypropylene, Safety, 0.47 uF",
),
"manufacturer": ExtractedFieldMetadata(
reasoning="VERBATIM EXTRACTION",
confidence=0.9997446550976602,
extraction_confidence=0.9997446550976602,
page_number=1,
matching_text="YAGEO KEMET",
),
"features": [
ExtractedFieldMetadata(
reasoning="VERBATIM EXTRACTION",
confidence=0.9999308195540074,
extraction_confidence=0.9999308195540074,
page_number=1,
matching_text="Features</td><td>EMI Safety",
),
ExtractedFieldMetadata(
reasoning="VERBATIM EXTRACTION",
confidence=0.8642493886452225,
extraction_confidence=0.8642493886452225,
page_number=1,
matching_text="THB Performance</td><td>Yes",
),
],
"dimensions": {
"length": ExtractedFieldMetadata(
reasoning="VERBATIM EXTRACTION",
confidence=0.8986941382802304,
extraction_confidence=0.8986941382802304,
page_number=1,
matching_text="L</td><td>41mm MAX",
),
"width": ExtractedFieldMetadata(
reasoning="VERBATIM EXTRACTION",
confidence=0.9999377974447091,
extraction_confidence=0.9999377974447091,
page_number=1,
matching_text="T</td><td>13mm MAX",
),
},
}
def create_file(
id: str = "file-456",
name: str = "resume.pdf",
@@ -290,12 +394,12 @@ def create_extract_run(
extraction_metadata: Dict[str, Any] = {
"name": {
"confidence": 0.95,
"citations": [{"page_number": 1, "matching_text": "John Doe"}],
"citation": [{"page": 1, "matching_text": "John Doe"}],
},
"age": {"confidence": 0.87},
"email": {
"confidence": 0.92,
"citations": [{"page_number": 1, "matching_text": "john@example.com"}],
"citation": [{"page": 1, "matching_text": "john@example.com"}],
},
},
data_schema: Dict[str, Any] = {},
+2
View File
@@ -28,6 +28,8 @@ api_key = os.environ.get("LLAMA_CLOUD_API_KEY", None)
organization_id = os.environ.get("LLAMA_CLOUD_ORGANIZATION_ID", None)
project_name = os.environ.get("LLAMA_CLOUD_PROJECT_NAME", "framework_integration_test")
print("api-key", api_key, "base-url", base_url)
@pytest.fixture()
def remote_file() -> Tuple[str, str]:
Generated
+2 -2
View File
@@ -734,7 +734,7 @@ name = "exceptiongroup"
version = "1.3.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
{ name = "typing-extensions", marker = "python_full_version < '3.11'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" }
wheels = [
@@ -1587,7 +1587,7 @@ wheels = [
[[package]]
name = "llama-cloud-services"
version = "0.6.53"
version = "0.6.54"
source = { editable = "." }
dependencies = [
{ name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+56 -23
View File
@@ -13,17 +13,26 @@ from pathlib import Path
def get_current_versions() -> tuple[str, str, str]:
"""Get current versions from both pyproject.toml files."""
# Read main pyproject.toml
main_content = Path("pyproject.toml").read_text()
main_content = Path("py/pyproject.toml").read_text()
main_doc = tomlkit.parse(main_content)
main_version = main_doc["tool"]["poetry"]["version"]
main_version = main_doc["project"]["version"]
# Read llama_parse/pyproject.toml
llama_parse_content = Path("llama_parse/pyproject.toml").read_text()
llama_parse_content = Path("py/llama_parse/pyproject.toml").read_text()
llama_parse_doc = tomlkit.parse(llama_parse_content)
llama_parse_version = llama_parse_doc["tool"]["poetry"]["version"]
dependency_version = llama_parse_doc["tool"]["poetry"]["dependencies"][
"llama-cloud-services"
]
llama_parse_version = llama_parse_doc["project"]["version"]
# Find llama-cloud-services dependency in the dependencies list
dependency_version = None
for dep in llama_parse_doc["project"]["dependencies"]:
if isinstance(dep, str) and dep.startswith("llama-cloud-services"):
dependency_version = (
dep.split("==")[1]
if "==" in dep
else dep.split(">=")[1]
if ">=" in dep
else None
)
break
return str(main_version), str(llama_parse_version), str(dependency_version)
@@ -53,19 +62,22 @@ def validate_versions(
def set_version(version: str) -> None:
"""Set version across all pyproject.toml files using tomlkit to preserve formatting."""
# Update main pyproject.toml
main_content = Path("pyproject.toml").read_text()
main_content = Path("py/pyproject.toml").read_text()
main_doc = tomlkit.parse(main_content)
main_doc["tool"]["poetry"]["version"] = version
Path("pyproject.toml").write_text(tomlkit.dumps(main_doc))
main_doc["project"]["version"] = version
Path("py/pyproject.toml").write_text(tomlkit.dumps(main_doc))
# Update llama_parse/pyproject.toml
llama_parse_content = Path("llama_parse/pyproject.toml").read_text()
llama_parse_content = Path("py/llama_parse/pyproject.toml").read_text()
llama_parse_doc = tomlkit.parse(llama_parse_content)
llama_parse_doc["tool"]["poetry"]["version"] = version
llama_parse_doc["tool"]["poetry"]["dependencies"][
"llama-cloud-services"
] = f">={version}"
Path("llama_parse/pyproject.toml").write_text(tomlkit.dumps(llama_parse_doc))
llama_parse_doc["project"]["version"] = version
for dep_index, dep in enumerate(llama_parse_doc["project"]["dependencies"]):
if isinstance(dep, str) and dep.startswith("llama-cloud-services"):
llama_parse_doc["project"]["dependencies"][
dep_index
] = f"llama-cloud-services>={version}"
break
Path("py/llama_parse/pyproject.toml").write_text(tomlkit.dumps(llama_parse_doc))
click.echo(f"Updated all versions to {version}")
@@ -78,7 +90,7 @@ def get_current_branch() -> str:
return result.stdout.strip()
def create_and_push_tag(version: str) -> None:
def create_if_not_exists(version: str) -> None:
"""Create a git tag and push it."""
current_branch = get_current_branch()
if current_branch != "main":
@@ -88,12 +100,26 @@ def create_and_push_tag(version: str) -> None:
sys.exit(1)
tag_name = f"v{version}"
if not tag_exists(version):
# Create tag
subprocess.run(["git", "tag", tag_name], check=True)
click.echo(f"Created tag {tag_name}")
else:
click.echo(f"Tag {tag_name} already exists")
# Create tag
subprocess.run(["git", "tag", tag_name], check=True)
click.echo(f"Created tag {tag_name}")
# Push tag
def tag_exists(version: str) -> bool:
"""Check if a git tag exists."""
tag_name = f"v{version}"
result = subprocess.run(
["git", "tag", "-l", tag_name], capture_output=True, text=True, check=True
)
return tag_name in result.stdout.strip()
def push_tag(version: str) -> None:
"""Push a git tag."""
tag_name = f"v{version}"
subprocess.run(["git", "push", "origin", tag_name], check=True)
click.echo(f"Pushed tag {tag_name}")
@@ -134,13 +160,20 @@ def set(version: str) -> None:
@click.option(
"--version", help="Version to tag (uses current version if not specified)"
)
def tag(version: str | None = None) -> None:
@click.option(
"--push",
is_flag=True,
help="Push the tag to the remote repository",
)
def tag(version: str | None = None, push: bool = False) -> None:
"""Create and push a git tag for the current version."""
if not version:
main_version, _, _ = get_current_versions()
version = main_version
create_and_push_tag(version)
create_if_not_exists(version)
if push:
push_tag(version)
if __name__ == "__main__":
+6 -6
View File
@@ -9,7 +9,7 @@
"dev": "bunchee --watch",
"lint": "eslint src/ --ignore-pattern client/*.ts --no-warn-ignored",
"format": "prettier --write ./src/",
"test": "vitest",
"test": "vitest run --testTimeout=60000",
"test:watch": "vitest --watch",
"test:ui": "vitest --ui",
"test:coverage": "vitest --coverage"
@@ -89,8 +89,8 @@
"@eslint/js": "^9.32.0",
"@hey-api/client-fetch": "^0.10.1",
"@hey-api/openapi-ts": "^0.67.5",
"@llama-flow/core": "^0.4.1",
"@llamaindex/core": "^0.6.18",
"@llamaindex/workflow-core": "^0.4.1",
"@llamaindex/core": "^0.6.19",
"@llamaindex/env": "^0.1.30",
"@typescript-eslint/eslint-plugin": "^8.38.0",
"@typescript-eslint/parser": "^8.38.0",
@@ -107,10 +107,10 @@
"vitest": "^2.0.0"
},
"peerDependencies": {
"@llama-flow/core": "^0.4.1",
"@llamaindex/core": "^0.6.18",
"@llamaindex/workflow-core": "^0.4.1",
"@llamaindex/core": "^0.6.19",
"@llamaindex/env": "^0.1.30",
"llamaindex": "^0.11.23"
"llamaindex": "^0.11.25"
},
"dependencies": {
"p-retry": "^6.2.1",
+1250 -2458
View File
File diff suppressed because it is too large Load Diff
@@ -261,8 +261,7 @@ export class AgentClient<T = unknown> {
}
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
export interface AgentDataClientOptions<T = unknown> {
export interface AgentDataClientOptions {
/** API key for the client */
apiKey?: string;
/** Base URL for the client */
@@ -28,6 +28,31 @@ export type ComparisonOperator =
*/
export type FilterOperation = RawFilterOperation;
/**
* Metadata for an extracted field, including confidence and citation information
*/
export interface ExtractedFieldMetadata {
/** The reasoning for the confidence score */
reasoning?: string;
/** The confidence score for the field, combined with parsing confidence if applicable */
confidence?: number;
/** The confidence score for the field based on the extracted text only */
extraction_confidence?: number;
/** The page number that the field occurred on */
page_number?: number;
/** The original text this field's value was derived from */
matching_text?: string;
}
/**
* Dictionary mapping field names to their metadata
* Values can be ExtractedFieldMetadata objects, nested dictionaries, or arrays
*/
export type ExtractedFieldMetadataDict = Record<
string,
ExtractedFieldMetadata | Record<string, unknown> | unknown[]
>;
/**
* Base extracted data interface
*/
@@ -35,11 +60,13 @@ export interface ExtractedData<T = unknown> {
/** The original data that was extracted from the document. For tracking changes. Should not be updated. */
original_data: T;
/** The latest state of the data. Will differ if data has been updated. */
data?: T;
data: T;
/** The status of the extracted data. Prefer to use the StatusType values, but any string is allowed. */
status: StatusType | string;
/** Confidence scores, if any, for each primitive field in the original_data data. */
confidence?: Record<string, unknown>;
/** The overall confidence score for the extracted data. */
overall_confidence?: number;
/** Page links, and perhaps eventually bounding boxes, for individual fields in the extracted data. */
field_metadata?: ExtractedFieldMetadataDict;
/** The ID of the file that was used to extract the data. */
file_id?: string;
/** The name of the file that was used to extract the data. */
+2 -2
View File
@@ -1,5 +1,5 @@
import { workflowEvent } from "@llama-flow/core";
import { zodEvent } from "@llama-flow/core/util/zod";
import { workflowEvent } from "@llamaindex/workflow-core";
import { zodEvent } from "@llamaindex/workflow-core/util/zod";
import { z } from "zod";
import { parseFormSchema } from "./schema";
+7 -4
View File
@@ -1,8 +1,11 @@
import { createClient, createConfig } from "@hey-api/client-fetch";
import { createWorkflow, type InferWorkflowEventData } from "@llama-flow/core";
import { createStatefulMiddleware } from "@llama-flow/core/middleware/state";
import { withTraceEvents } from "@llama-flow/core/middleware/trace-events";
import { pRetryHandler } from "@llama-flow/core/util/p-retry";
import {
createWorkflow,
type InferWorkflowEventData,
} from "@llamaindex/workflow-core";
import { createStatefulMiddleware } from "@llamaindex/workflow-core/middleware/state";
import { withTraceEvents } from "@llamaindex/workflow-core/middleware/trace-events";
import { pRetryHandler } from "@llamaindex/workflow-core/util/p-retry";
import { fs, getEnv, path } from "@llamaindex/env";
import {
type BodyUploadFileApiV1ParsingUploadPost,