Compare commits

..

6 Commits

Author SHA1 Message Date
Logan Markewich 853aecc625 update CLAUDE.md 2025-11-17 11:11:43 -06:00
Logan Markewich 918190d7de fixes 2025-11-14 21:33:34 -06:00
Logan Markewich 346b1f1f86 update things 2025-11-06 10:13:12 -06:00
Logan Markewich 145bf3476c add examples 2025-11-04 19:19:59 -06:00
Logan Markewich 6118111691 initial 2025-10-30 19:44:39 -06:00
Logan Markewich 018dae3166 initial 2025-10-30 19:44:17 -06:00
28 changed files with 200 additions and 3786 deletions
+5
View File
@@ -0,0 +1,5 @@
---
"llama-cloud-services-py": patch
---
Now return partial results on job failure
+1 -1
View File
@@ -12,7 +12,7 @@
"@tanstack/react-router": "^1.133.22",
"@tanstack/react-router-devtools": "^1.133.22",
"@tanstack/react-start": "^1.133.22",
"llama-cloud-services": "file:../../ts/llama_cloud_services",
"llama-cloud-services": "^0.3.10",
"react": "^19.0.0",
"react-dom": "^19.0.0",
"tailwind-merge": "^2.6.0",
@@ -15,7 +15,7 @@ export const Route = createFileRoute('/api/classify')({
const rawRes = await classifier.classify(
classificationRules,
parsingConfig,
{ fileContents: [new Uint8Array(buff)] },
[new Uint8Array(buff)],
)
const results = rawRes.items
let classification = ""
@@ -280,7 +280,7 @@
"source": [
"## Phase 2: Document Classification\n",
"\n",
"Next, let's classify our documents based on their content using `LlamaClassify`."
"Next, let's classify our documents based on their content using the ClassifyClient."
]
},
{
@@ -298,14 +298,14 @@
}
],
"source": [
"from llama_cloud_services.beta.classifier.client import LlamaClassify\n",
"from llama_cloud_services.beta.classifier.client import ClassifyClient\n",
"from llama_cloud.types import ClassifierRule\n",
"from llama_cloud_services.files.client import FileClient\n",
"from llama_cloud.client import AsyncLlamaCloud\n",
"\n",
"# Initialize the classify client\n",
"api_key = os.environ[\"LLAMA_CLOUD_API_KEY\"]\n",
"classify_client = LlamaClassify.from_api_key(api_key)\n",
"classify_client = ClassifyClient.from_api_key(api_key)\n",
"\n",
"print(\"🏷️ Setting up document classification...\")\n",
"\n",
@@ -1097,7 +1097,7 @@
" - Preserves document structure and formatting\n",
" - Handles various file types (PDF, DOCX, etc.)\n",
"\n",
"2. **LlamaClassify** (`llama_cloud_services.beta.classifier.client.LlamaClassify`):\n",
"2. **ClassifyClient** (`llama_cloud_services.beta.classifier.client.ClassifyClient`):\n",
" - Automatically categorizes documents based on content\n",
" - Uses customizable rules for classification\n",
" - Provides confidence scores for classifications\n",
+1 -1
View File
@@ -19,7 +19,7 @@
"lint-staged": {
"ts/llama_cloud_services/src/**/*.{ts,tsx,js,jsx}": [
"pnpm --filter llama-cloud-services exec eslint --fix",
"pnpm --filter llama-cloud-services exec prettier --write src/ tests/"
"pnpm --filter llama-cloud-services exec prettier --write"
]
},
"packageManager": "pnpm@10.11.1+sha512.e519b9f7639869dc8d5c3c5dfef73b3f091094b0a006d7317353c72b124e80e1afd429732e28705ad6bfa1ee879c1fce46c128ccebd3192101f43dd67c667912"
+19 -3368
View File
File diff suppressed because it is too large Load Diff
-24
View File
@@ -1,29 +1,5 @@
# llama-cloud-services-py
## 0.6.80
### Patch Changes
- 0506c88: Moved ClassifyClient to LlamaClassify (backward compatible)
## 0.6.79
### Patch Changes
- e020e3e: Remove unneeded organization_id param from beta classifier client
## 0.6.78
### Patch Changes
- 9f1ef4e: Fix extract
## 0.6.77
### Patch Changes
- 407292b: Now return partial results on job failure
## 0.6.76
### Patch Changes
@@ -1,9 +1,8 @@
from llama_cloud_services.beta.classifier.client import LlamaClassify, ClassifyClient
from llama_cloud_services.beta.classifier.client import ClassifyClient
from llama_cloud_services.beta.classifier.types import ClassifyJobResultsWithFiles
from llama_cloud_services.utils import SourceText, FileInput
__all__ = [
"LlamaClassify",
"ClassifyClient",
"ClassifyJobResultsWithFiles",
"SourceText",
@@ -31,7 +31,7 @@ class ClassificationOutput(BaseModel):
classification: str
class LlamaClassify:
class ClassifyClient:
"""
Experimental - Client for interacting with the LlamaCloud Classifier API.
The Classification API is currently in beta and may change in the future without notice.
@@ -39,6 +39,7 @@ class LlamaClassify:
Args:
client: The LlamaCloud client to use.
project_id: The project ID to use.
organization_id: The organization ID to use.
polling_interval: The interval to poll for job completion in seconds.
polling_timeout: The timeout for the job to complete in seconds.
"""
@@ -47,13 +48,15 @@ class LlamaClassify:
self,
client: AsyncLlamaCloud,
project_id: Optional[str] = None,
organization_id: Optional[str] = None,
polling_interval: float = 1.0,
polling_timeout: float = POLLING_TIMEOUT_SECONDS,
):
self.client = client
self.project_id = project_id
self.organization_id = organization_id
self.polling_interval = polling_interval
self.file_client = FileClient(client, project_id)
self.file_client = FileClient(client, project_id, organization_id)
self.polling_timeout = polling_timeout
@classmethod
@@ -61,6 +64,7 @@ class LlamaClassify:
cls,
api_key: str,
project_id: Optional[str] = None,
organization_id: Optional[str] = None,
base_url: Optional[str] = None,
) -> "ClassifyClient":
"""
@@ -70,6 +74,7 @@ class LlamaClassify:
return cls(
client,
project_id,
organization_id,
)
async def acreate_classify_job(
@@ -96,6 +101,7 @@ class LlamaClassify:
file_ids=file_ids,
parsing_configuration=parsing_configuration or OMIT,
project_id=self.project_id,
organization_id=self.organization_id,
)
def create_classify_job(
@@ -146,6 +152,7 @@ class LlamaClassify:
results = await self.client.classifier.get_classification_job_results(
classify_job_with_status.id,
project_id=self.project_id,
organization_id=self.organization_id,
)
return results
@@ -352,7 +359,7 @@ class LlamaClassify:
The classify job with status.
"""
job = await self.client.classifier.get_classify_job(
job_id, project_id=self.project_id
job_id, project_id=self.project_id, organization_id=self.organization_id
)
start_time = time.time()
while not is_terminal_status(job.status):
@@ -363,9 +370,6 @@ class LlamaClassify:
)
await asyncio.sleep(self.polling_interval)
job = await self.client.classifier.get_classify_job(
job_id, project_id=self.project_id
job_id, project_id=self.project_id, organization_id=self.organization_id
)
return job
ClassifyClient = LlamaClassify
+10 -31
View File
@@ -19,7 +19,6 @@ from llama_cloud import (
PipelineCreate,
PipelineCreateEmbeddingConfig,
PipelineCreateTransformConfig,
PipelineFileCreateCustomMetadataValue,
PipelineType,
ProjectCreate,
ManagedIngestionStatus,
@@ -334,7 +333,7 @@ class LlamaCloudIndex(BaseManagedIndex):
if file_ids:
self._wait_for_resources(
file_ids,
lambda fid: self._client.pipeline_files.get_pipeline_file_status(
lambda fid: self._client.pipelines.get_pipeline_file_status(
pipeline_id=self.pipeline.id, file_id=fid
),
resource_name="file",
@@ -421,7 +420,7 @@ class LlamaCloudIndex(BaseManagedIndex):
if file_ids:
await self._await_for_resources(
file_ids,
lambda fid: self._aclient.pipeline_files.get_pipeline_file_status(
lambda fid: self._aclient.pipelines.get_pipeline_file_status(
pipeline_id=self.pipeline.id, file_id=fid
),
resource_name="file",
@@ -906,9 +905,6 @@ class LlamaCloudIndex(BaseManagedIndex):
def upload_file(
self,
file_path: str,
custom_metadata: Optional[
dict[str, Optional[PipelineFileCreateCustomMetadataValue]]
] = None,
verbose: bool = False,
wait_for_ingestion: bool = True,
raise_on_error: bool = False,
@@ -922,10 +918,8 @@ class LlamaCloudIndex(BaseManagedIndex):
print(f"Uploaded file {file.id} with name {file.name}")
# Add file to pipeline
pipeline_file_create = PipelineFileCreate(
file_id=file.id, custom_metadata=custom_metadata
)
self._client.pipeline_files.add_files_to_pipeline_api(
pipeline_file_create = PipelineFileCreate(file_id=file.id)
self._client.pipelines.add_files_to_pipeline_api(
pipeline_id=self.pipeline.id, request=[pipeline_file_create]
)
@@ -938,9 +932,6 @@ class LlamaCloudIndex(BaseManagedIndex):
async def aupload_file(
self,
file_path: str,
custom_metadata: Optional[
dict[str, Optional[PipelineFileCreateCustomMetadataValue]]
] = None,
verbose: bool = False,
wait_for_ingestion: bool = True,
raise_on_error: bool = False,
@@ -954,10 +945,8 @@ class LlamaCloudIndex(BaseManagedIndex):
print(f"Uploaded file {file.id} with name {file.name}")
# Add file to pipeline
pipeline_file_create = PipelineFileCreate(
file_id=file.id, custom_metadata=custom_metadata
)
await self._aclient.pipeline_files.add_files_to_pipeline_api(
pipeline_file_create = PipelineFileCreate(file_id=file.id)
await self._aclient.pipelines.add_files_to_pipeline_api(
pipeline_id=self.pipeline.id, request=[pipeline_file_create]
)
@@ -972,9 +961,6 @@ class LlamaCloudIndex(BaseManagedIndex):
self,
file_name: str,
url: str,
custom_metadata: Optional[
dict[str, Optional[PipelineFileCreateCustomMetadataValue]]
] = None,
proxy_url: Optional[str] = None,
request_headers: Optional[Dict[str, str]] = None,
verify_ssl: bool = True,
@@ -997,10 +983,8 @@ class LlamaCloudIndex(BaseManagedIndex):
print(f"Uploaded file {file.id} with ID {file.id}")
# Add file to pipeline
pipeline_file_create = PipelineFileCreate(
file_id=file.id, custom_metadata=custom_metadata
)
self._client.pipeline_files.add_files_to_pipeline_api(
pipeline_file_create = PipelineFileCreate(file_id=file.id)
self._client.pipelines.add_files_to_pipeline_api(
pipeline_id=self.pipeline.id, request=[pipeline_file_create]
)
@@ -1014,9 +998,6 @@ class LlamaCloudIndex(BaseManagedIndex):
self,
file_name: str,
url: str,
custom_metadata: Optional[
dict[str, Optional[PipelineFileCreateCustomMetadataValue]]
] = None,
proxy_url: Optional[str] = None,
request_headers: Optional[Dict[str, str]] = None,
verify_ssl: bool = True,
@@ -1039,10 +1020,8 @@ class LlamaCloudIndex(BaseManagedIndex):
print(f"Uploaded file {file.id} with ID {file.id}")
# Add file to pipeline
pipeline_file_create = PipelineFileCreate(
file_id=file.id, custom_metadata=custom_metadata
)
await self._aclient.pipeline_files.add_files_to_pipeline_api(
pipeline_file_create = PipelineFileCreate(file_id=file.id)
await self._aclient.pipelines.add_files_to_pipeline_api(
pipeline_id=self.pipeline.id, request=[pipeline_file_create]
)
-29
View File
@@ -1,34 +1,5 @@
# llama_parse
## 0.6.80
### Patch Changes
- Updated dependencies [0506c88]
- llama-cloud-services-py@0.6.80
## 0.6.79
### Patch Changes
- Updated dependencies [e020e3e]
- llama-cloud-services-py@0.6.79
## 0.6.78
### Patch Changes
- 9f1ef4e: Fix extract
- Updated dependencies [9f1ef4e]
- llama-cloud-services-py@0.6.78
## 0.6.77
### Patch Changes
- Updated dependencies [407292b]
- llama-cloud-services-py@0.6.77
## 0.6.76
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "llama_parse",
"version": "0.6.80",
"version": "0.6.76",
"description": "",
"main": "index.js",
"private": false,
+2 -2
View File
@@ -11,13 +11,13 @@ dev = [
[project]
name = "llama-parse"
version = "0.6.80"
version = "0.6.76"
description = "Parse files into RAG-Optimized formats."
authors = [{name = "Logan Markewich", email = "logan@llamaindex.ai"}]
requires-python = ">=3.9,<4.0"
readme = "README.md"
license = "MIT"
dependencies = ["llama-cloud-services>=0.6.80"]
dependencies = ["llama-cloud-services>=0.6.76"]
[project.scripts]
llama-parse = "llama_parse.cli.main:parse"
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "llama-cloud-services-py",
"version": "0.6.80",
"version": "0.6.76",
"private": false,
"license": "MIT",
"scripts": {},
+2 -2
View File
@@ -22,7 +22,7 @@ dev = [
[project]
name = "llama-cloud-services"
version = "0.6.80"
version = "0.6.76"
description = "Tailored SDK clients for LlamaCloud services."
authors = [{name = "Logan Markewich", email = "logan@runllama.ai"}]
requires-python = ">=3.9,<4.0"
@@ -30,7 +30,7 @@ readme = "README.md"
license = "MIT"
dependencies = [
"llama-index-core>=0.12.0",
"llama-cloud==0.1.44",
"llama-cloud==0.1.43",
"pydantic>=2.8,!=2.10",
"click>=8.1.7,<9",
"python-dotenv>=1.0.1,<2",
+3
View File
@@ -44,6 +44,7 @@ def classify_client(
return ClassifyClient(
async_llama_cloud_client,
project_id=project.id,
organization_id=project.organization_id,
polling_interval=1,
)
@@ -55,6 +56,7 @@ def file_client(
return FileClient(
async_llama_cloud_client,
project_id=project.id,
organization_id=project.organization_id,
use_presigned_url=False,
)
@@ -146,6 +148,7 @@ async def test_classify_file_ids_from_api_key(
api_key=e2e_test_settings.LLAMA_CLOUD_API_KEY.get_secret_value(),
base_url=e2e_test_settings.LLAMA_CLOUD_BASE_URL,
project_id=pdf_file.project_id,
organization_id=e2e_test_settings.LLAMA_CLOUD_ORGANIZATION_ID,
)
# Classify the uploaded files
-2
View File
@@ -58,8 +58,6 @@ def get_test_cases():
settings = [
ExtractConfig(extraction_mode=ExtractMode.FAST),
ExtractConfig(extraction_mode=ExtractMode.BALANCED),
ExtractConfig(extraction_mode=ExtractMode.MULTIMODAL),
ExtractConfig(extraction_mode=ExtractMode.PREMIUM),
]
for input_file in sorted(input_files):
+2 -121
View File
@@ -44,7 +44,7 @@ def index_name() -> Generator[str, None, None]:
client = LlamaCloud(token=api_key, base_url=base_url)
pipeline = client.pipelines.search_pipelines(project_name=name)
if pipeline:
client.pipelines.delete_pipeline(pipeline_id=pipeline[0].id)
client.pipelines.delete(pipeline_id=pipeline[0].id)
@pytest.fixture()
@@ -83,7 +83,7 @@ def _setup_index_with_file(
# add file to pipeline
pipeline_file_create = PipelineFileCreate(file_id=file.id)
client.pipeline_files.add_files_to_pipeline_api(
client.pipelines.add_files_to_pipeline_api(
pipeline_id=pipeline.id, request=[pipeline_file_create]
)
@@ -170,43 +170,6 @@ def test_upload_file(index_name: str):
os.remove(temp_file_path)
@pytest.mark.skipif(
not base_url or not api_key, reason="No platform base url or api key set"
)
def test_upload_file_with_custom_metadata(index_name: str):
index = LlamaCloudIndex.create_index(
name=index_name,
project_name=project_name,
organization_id=organization_id,
api_key=api_key,
base_url=base_url,
)
# Create a temporary file to upload
with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as temp_file:
temp_file.write(b"Sample content for testing upload.")
temp_file_path = temp_file.name
custom_metadata = {"foo": "bar"}
try:
# Upload the file
file_id = index.upload_file(
temp_file_path, custom_metadata=custom_metadata, verbose=True
)
assert file_id is not None
# Verify the file is part of the index
docs = index.ref_doc_info
temp_file_name = os.path.basename(temp_file_path)
assert any(
temp_file_name == doc.metadata.get("file_name") for doc in docs.values()
)
finally:
# Clean up the temporary file
os.remove(temp_file_path)
@pytest.mark.skipif(
not base_url or not api_key, reason="No platform base url or api key set"
)
@@ -233,38 +196,6 @@ def test_upload_file_from_url(remote_file: Tuple[str, str], index_name: str):
assert any(test_file_name == doc.metadata.get("file_name") for doc in docs.values())
@pytest.mark.skipif(
not base_url or not api_key, reason="No platform base url or api key set"
)
def test_upload_file_from_url_with_custom_metadata(
remote_file: Tuple[str, str], index_name: str
):
index = LlamaCloudIndex.create_index(
name=index_name,
project_name=project_name,
organization_id=organization_id,
api_key=api_key,
base_url=base_url,
)
# Define a URL to a file for testing
custom_metadata = {"foo": "bar"}
test_file_url, test_file_name = remote_file
# Upload the file from the URL
file_id = index.upload_file_from_url(
file_name=test_file_name,
url=test_file_url,
custom_metadata=custom_metadata,
verbose=True,
)
assert file_id is not None
# Verify the file is part of the index
docs = index.ref_doc_info
assert any(test_file_name == doc.metadata.get("file_name") for doc in docs.values())
@pytest.mark.skipif(
not base_url or not api_key, reason="No platform base url or api key set"
)
@@ -576,33 +507,6 @@ async def test_async_upload_file_from_url(
await index.await_for_completion()
@pytest.mark.skipif(
not base_url or not api_key, reason="No platform base url or api key set"
)
@pytest.mark.asyncio
async def test_async_upload_file_from_url_with_custom_metadata(
remote_file: Tuple[str, str], index_name: str
):
index = await LlamaCloudIndex.acreate_index(
name=index_name,
project_name=project_name,
api_key=api_key,
base_url=base_url,
)
custom_metadata = {"foo": "bar"}
test_file_url, test_file_name = remote_file
file_id = await index.aupload_file_from_url(
file_name=test_file_name,
url=test_file_url,
custom_metadata=custom_metadata,
verbose=True,
)
assert file_id is not None
await index.await_for_completion()
@pytest.mark.skipif(
not base_url or not api_key, reason="No platform base url or api key set"
)
@@ -621,29 +525,6 @@ async def test_async_index_from_file(index_name: str, local_file: str):
await index.await_for_completion()
@pytest.mark.skipif(
not base_url or not api_key, reason="No platform base url or api key set"
)
@pytest.mark.asyncio
async def test_async_index_from_file_with_custom_metadata(
index_name: str, local_file: str
):
index = await LlamaCloudIndex.acreate_index(
name=index_name,
project_name=project_name,
api_key=api_key,
base_url=base_url,
)
custom_metadata = {"foo": "bar"}
file_id = await index.aupload_file(
file_path=local_file, custom_metadata=custom_metadata, verbose=True
)
assert file_id is not None
await index.await_for_completion()
class DummySchema(BaseModel):
source: str
@@ -2,7 +2,6 @@ from datetime import datetime
import json
from pathlib import Path
from typing import Any, Dict, Optional
import uuid
import pytest
from llama_cloud import ExtractRun, File
@@ -435,7 +434,6 @@ def create_extract_run(
"extraction_agent_id": "extraction-agent-123",
"config": {},
"status": "SUCCESS",
"project_id": str(uuid.uuid4()),
"from_ui": False,
}
)
-1
View File
@@ -112,6 +112,5 @@
"num_output_tokens": 3440
}
},
"project_id": "77bdc79f-fb69-49ae-a783-fcc573eec7ce",
"from_ui": false
}
Generated
+6 -6
View File
@@ -1,5 +1,5 @@
version = 1
revision = 3
revision = 2
requires-python = ">=3.9, <4.0"
resolution-markers = [
"python_full_version >= '3.14'",
@@ -1582,21 +1582,21 @@ wheels = [
[[package]]
name = "llama-cloud"
version = "0.1.44"
version = "0.1.43"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "certifi" },
{ name = "httpx" },
{ name = "pydantic" },
]
sdist = { url = "https://files.pythonhosted.org/packages/54/eb/16e31fb0fc4df91b08fa19cc3f28ac6e3c7d4df0bcbb71dd2bf596e9586f/llama_cloud-0.1.44.tar.gz", hash = "sha256:276a2b4f94463da037431ca3063331b3b6be398bbfb003113ee76b7c2a873b53", size = 120502, upload-time = "2025-11-04T00:51:58.578Z" }
sdist = { url = "https://files.pythonhosted.org/packages/9b/33/33a8bd3a617c071caf450ca2627969f8b28272d0692f122997c10a32247e/llama_cloud-0.1.43.tar.gz", hash = "sha256:00429f05aea515449d90cde91ef3ed3687fcd93e46f6246d08cbea02f9b397a9", size = 112992, upload-time = "2025-10-02T21:55:38.355Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/69/0a/fabe54c21d5927d626550cb9560a20e51e42468355f5f0fb300f84806e28/llama_cloud-0.1.44-py3-none-any.whl", hash = "sha256:dfdcc4932353711fc8639f14261cbb54a88139b7790ebdd3ed4fde29bbbc0b88", size = 332779, upload-time = "2025-11-04T00:51:57.371Z" },
{ url = "https://files.pythonhosted.org/packages/2b/54/559a67542396d5660a71115b29e0160e9dd784e570e1f4ef55ad22bf5b39/llama_cloud-0.1.43-py3-none-any.whl", hash = "sha256:540605d4dd13c6536a3b75cd4d04b211f29b16d17faee9381e3793a651f1dec1", size = 311460, upload-time = "2025-10-02T21:55:37.282Z" },
]
[[package]]
name = "llama-cloud-services"
version = "0.6.79"
version = "0.6.76"
source = { editable = "." }
dependencies = [
{ name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
@@ -1631,7 +1631,7 @@ dev = [
requires-dist = [
{ name = "click", specifier = ">=8.1.7,<9" },
{ name = "eval-type-backport", marker = "python_full_version < '3.10'", specifier = ">=0.2.0,<0.3" },
{ name = "llama-cloud", specifier = "==0.1.44" },
{ name = "llama-cloud", specifier = "==0.1.43" },
{ name = "llama-index-core", specifier = ">=0.12.0" },
{ name = "packaging", specifier = ">=23.0" },
{ name = "platformdirs", specifier = ">=4.3.7,<5" },
-6
View File
@@ -1,11 +1,5 @@
# llama-cloud-services
## 0.4.0
### Minor Changes
- f293547: Switch to keyword arguments rather than positional args
## 0.3.10
### Patch Changes
+3 -3
View File
@@ -1,6 +1,6 @@
{
"name": "llama-cloud-services",
"version": "0.4.0",
"version": "0.3.10",
"type": "module",
"license": "MIT",
"scripts": {
@@ -9,8 +9,8 @@
"build": "pnpm run generate && bunchee",
"dev": "bunchee --watch",
"lint": "eslint src/ --ignore-pattern client/*.ts --no-warn-ignored",
"format": "prettier --write ./src/ tests/",
"format:check": "prettier --check ./src/ tests/",
"format": "prettier --write ./src/",
"format:check": "prettier --check ./src/",
"test": "vitest run --testTimeout=60000",
"test:watch": "vitest --watch",
"test:ui": "vitest --ui",
+21 -27
View File
@@ -36,40 +36,34 @@ export class LlamaClassify {
async classify(
rules: ClassifierRule[],
configuration: ClassifyParsingConfiguration,
{
parsingConfiguration: ClassifyParsingConfiguration,
fileContents:
| Buffer<ArrayBufferLike>[]
| File[]
| Uint8Array<ArrayBuffer>[]
| string[]
| undefined = undefined,
filePaths: string[] | undefined = undefined,
projectId: string | null = null,
organizationId: string | null = null,
pollingInterval: number = 1,
maxPollingIterations: number = 1800,
maxRetriesOnError: number = 10,
retryInterval: number = 0.5,
): Promise<ClassifyJobResults> {
const result = await classify(
rules,
parsingConfiguration,
fileContents,
filePaths,
projectId,
pollingInterval = 1,
maxPollingIterations = 1800,
maxRetriesOnError = 10,
retryInterval = 0.5,
}: {
fileContents?:
| Buffer<ArrayBufferLike>[]
| File[]
| Uint8Array<ArrayBuffer>[]
| string[]
| undefined;
filePaths?: string[] | undefined;
projectId?: string;
pollingInterval?: number;
maxPollingIterations?: number;
maxRetriesOnError?: number;
retryInterval?: number;
},
): Promise<ClassifyJobResults> {
const result = await classify(rules, configuration, {
fileContents,
filePaths,
projectId: projectId ?? undefined,
client: this.client,
organizationId,
this.client,
pollingInterval,
maxPollingIterations,
maxRetriesOnError,
retryInterval,
});
);
return result;
}
}
+69 -87
View File
@@ -19,23 +19,16 @@ import { sleep } from "./utils";
import { uploadFile } from "./fileUpload";
import { File } from "buffer";
async function createClassifyJob({
fileIds,
rules,
parsingConfiguration,
projectId,
client,
maxRetriesOnError = 10,
retryInterval = 0.5,
}: {
fileIds: string[];
rules: ClassifierRule[];
parsingConfiguration: ClassifyParsingConfiguration;
projectId?: string | undefined;
client?: Client | undefined;
maxRetriesOnError?: number;
retryInterval?: number;
}): Promise<string> {
async function createClassifyJob(
fileIds: string[],
rules: ClassifierRule[],
parsingConfiguration: ClassifyParsingConfiguration,
organizationId: null | string,
projectId: null | string,
client: Client | undefined,
maxRetriesOnError: number = 10,
retryInterval: number = 0.5,
): Promise<string> {
const rawData = {
file_ids: fileIds,
rules: rules,
@@ -45,6 +38,7 @@ async function createClassifyJob({
body: rawData,
query: {
project_id: projectId,
organization_id: organizationId,
},
} as CreateClassifyJobApiV1ClassifierJobsPostData;
const options = data as Options<CreateClassifyJobApiV1ClassifierJobsPostData>;
@@ -81,17 +75,12 @@ async function createClassifyJob({
}
}
async function pollForJobCompletion({
jobId,
interval = 1,
maxIterations = 1800,
client,
}: {
jobId: string;
interval?: number;
maxIterations?: number;
client?: Client | undefined;
}): Promise<boolean> {
async function pollForJobCompletion(
jobId: string,
interval: number = 1,
maxIterations: number = 1800,
client: Client | undefined = undefined,
): Promise<boolean> {
let status: StatusEnum | undefined = undefined;
const jobData = {
path: { classify_job_id: jobId },
@@ -125,22 +114,17 @@ async function pollForJobCompletion({
}
}
async function getJobResult({
jobId,
client,
projectId,
maxRetriesOnError = 10,
retryInterval = 0.5,
}: {
jobId: string;
client?: Client | undefined;
projectId?: string | undefined;
maxRetriesOnError?: number;
retryInterval?: number;
}): Promise<ClassifyJobResults> {
async function getJobResult(
jobId: string,
client: Client | undefined = undefined,
projectId: string | null = null,
organizationId: string | null = null,
maxRetriesOnError: number = 10,
retryInterval: number = 0.5,
): Promise<ClassifyJobResults> {
const jobData = {
path: { classify_job_id: jobId },
query: { project_id: projectId },
query: { organization_id: organizationId, project_id: projectId },
} as GetClassificationJobResultsApiV1ClassifierJobsClassifyJobIdResultsGetData;
const jobOptions =
jobData as Options<GetClassificationJobResultsApiV1ClassifierJobsClassifyJobIdResultsGetData>;
@@ -182,30 +166,20 @@ async function getJobResult({
export async function classify(
rules: ClassifierRule[],
parsingConfiguration: ClassifyParsingConfiguration,
{
fileContents,
filePaths,
projectId,
client,
pollingInterval = 1,
maxPollingIterations = 1800,
maxRetriesOnError = 10,
retryInterval = 0.5,
}: {
fileContents?:
| Buffer<ArrayBufferLike>[]
| File[]
| Uint8Array<ArrayBuffer>[]
| string[]
| undefined;
filePaths?: string[] | undefined;
projectId?: string | undefined;
client?: Client | undefined;
pollingInterval?: number;
maxPollingIterations?: number;
maxRetriesOnError?: number;
retryInterval?: number;
},
fileContents:
| Buffer<ArrayBufferLike>[]
| File[]
| Uint8Array<ArrayBuffer>[]
| string[]
| undefined = undefined,
filePaths: string[] | undefined = undefined,
projectId: string | null = null,
organizationId: string | null = null,
client: Client | undefined = undefined,
pollingInterval: number = 1,
maxPollingIterations: number = 1800,
maxRetriesOnError: number = 10,
retryInterval: number = 0.5,
): Promise<ClassifyJobResults> {
const fileIds: string[] = [];
if (!filePaths && !fileContents) {
@@ -217,13 +191,16 @@ export async function classify(
if (filePaths) {
const uploadPromises = filePaths.map(async (name) => {
try {
const fileId = await uploadFile({
filePath: name,
const fileId = await uploadFile(
name,
undefined,
undefined,
projectId,
organizationId,
client,
maxRetriesOnError,
retryInterval: retryInterval,
project_id: projectId,
client: client,
});
retryInterval,
);
if (fileId) {
return fileId;
} else {
@@ -243,13 +220,16 @@ export async function classify(
if (fileContents) {
const uploadPromises = fileContents.map(async (content) => {
try {
const fileId = await uploadFile({
fileContent: content,
...(projectId ? { project_id: projectId } : {}),
...(client ? { client: client } : {}),
const fileId = await uploadFile(
undefined,
content,
undefined,
projectId,
organizationId,
client,
maxRetriesOnError,
retryInterval,
});
);
if (fileId) {
return fileId;
} else {
@@ -272,31 +252,33 @@ export async function classify(
);
}
const jobId = await createClassifyJob({
const jobId = await createClassifyJob(
fileIds,
rules,
parsingConfiguration,
...(projectId ? { projectId: projectId } : {}),
...(client ? { client: client } : {}),
organizationId,
projectId,
client,
maxRetriesOnError,
retryInterval,
});
const success = await pollForJobCompletion({
);
const success = await pollForJobCompletion(
jobId,
interval: pollingInterval,
maxIterations: maxPollingIterations,
pollingInterval,
maxPollingIterations,
client,
});
);
if (!success) {
throw new Error("Your job is taking longer than 10 minutes, timing out...");
} else {
return (await getJobResult({
return (await getJobResult(
jobId,
client,
projectId,
organizationId,
maxRetriesOnError,
retryInterval,
})) as ClassifyJobResults;
)) as ClassifyJobResults;
}
}
+8 -8
View File
@@ -378,16 +378,16 @@ export async function extract(
maxRetriesOnError: number = 10,
retryInterval: number = 0.5,
): Promise<ExtractResult | undefined> {
const fileId = (await uploadFile({
const fileId = (await uploadFile(
filePath,
fileContent,
fileName,
project_id: project_id ?? undefined,
organization_id: organization_id ?? undefined,
project_id,
organization_id,
client,
maxRetriesOnError,
retryInterval,
})) as string;
)) as string;
const extractJobCreate = {
extraction_agent_id: agentId,
file_id: fileId,
@@ -457,16 +457,16 @@ export async function extractStateless(
maxRetriesOnError: number = 10,
retryInterval: number = 0.5,
): Promise<ExtractResult | undefined> {
const fileId = (await uploadFile({
const fileId = (await uploadFile(
filePath,
fileContent,
fileName,
project_id: project_id ?? undefined,
organization_id: organization_id ?? undefined,
project_id,
organization_id,
client,
maxRetriesOnError,
retryInterval,
})) as string;
)) as string;
const extractStatetelessCreate = {
data_schema: dataSchema,
file_id: fileId,
+12 -23
View File
@@ -23,30 +23,21 @@ function textToFile(text: string, fileName: string | null = null) {
);
}
export async function uploadFile({
filePath,
fileContent,
fileName,
project_id,
organization_id,
client,
maxRetriesOnError = 10,
retryInterval = 0.5,
}: {
filePath?: string | undefined;
fileContent?:
export async function uploadFile(
filePath: string | undefined = undefined,
fileContent:
| Buffer<ArrayBufferLike>
| File
| Uint8Array<ArrayBuffer>
| string
| undefined;
fileName?: string | undefined;
project_id?: string | undefined;
organization_id?: string | undefined;
client?: Client | undefined;
maxRetriesOnError?: number;
retryInterval?: number;
}): Promise<string | undefined> {
| undefined = undefined,
fileName: string | undefined = undefined,
project_id: string | null = null,
organization_id: string | null = null,
client: Client | undefined = undefined,
maxRetriesOnError: number = 10,
retryInterval: number = 0.5,
): Promise<string | undefined> {
let file: File | undefined = undefined;
if (typeof filePath === "undefined" && typeof fileContent === "undefined") {
throw new Error(
@@ -88,7 +79,7 @@ export async function uploadFile({
} as BodyUploadFileApiV1FilesPost;
const uploadData = {
body: fileToUpload,
query: { project_id: project_id, organization_id: organization_id },
query: { organization_id: organization_id, project_id: project_id },
} as UploadFileApiV1FilesPostData;
const uploadOptions = uploadData as Options<UploadFileApiV1FilesPostData>;
if (typeof client != "undefined") {
@@ -104,8 +95,6 @@ export async function uploadFile({
const uploadResponse = await uploadFileApiV1FilesPost(uploadOptions);
let fileId: string | undefined = undefined;
if (!uploadResponse.response.ok) {
const error = await uploadResponse.response.text();
console.error("Error while uploading file: ", error);
retries++;
await sleep(retryInterval * 1000);
}
@@ -3,10 +3,7 @@ import { LlamaParseReader } from "../src/reader.js";
import { LlamaCloudIndex } from "../src/LlamaCloudIndex.js";
import { LlamaExtract, LlamaExtractAgent } from "../src/LlamaExtract.js";
import { LlamaClassify } from "../src/LlamaClassify.js";
import {
ClassifierRule,
ClassifyParsingConfiguration,
} from "../src/classify.js";
import { ClassifierRule, ClassifyParsingConfiguration } from "../src/classify.js";
import { Document } from "@llamaindex/core/schema";
import { fs } from "@llamaindex/env";
import { ExtractConfig } from "../src/api.js";
@@ -502,29 +499,25 @@ describe("Integration Tests", () => {
process.env.LLAMA_CLOUD_API_KEY!,
"https://api.cloud.llamaindex.ai",
);
const testContent = `A Fox one day spied a beautiful bunch of ripe grapes hanging from a vine trained along the branches of a tree. The grapes seemed ready to burst with juice, and the Fox's mouth watered as he gazed longingly at them. The bunch hung from a high branch, and the Fox had to jump for it. The first time he jumped he missed it by a long way. So he walked off a short distance and took a running leap at it, only to fall short once more. Again and again he tried, but in vain. Now he sat down and looked at the grapes in disgust. "What a fool I am," he said. "Here I am wearing myself out to get a bunch of sour grapes that are not worth gaping for." And off he walked very, very scornfully.There are many who pretend to despise and belittle that which is beyond their reach.`;
const testContent =
`A Fox one day spied a beautiful bunch of ripe grapes hanging from a vine trained along the branches of a tree. The grapes seemed ready to burst with juice, and the Fox's mouth watered as he gazed longingly at them. The bunch hung from a high branch, and the Fox had to jump for it. The first time he jumped he missed it by a long way. So he walked off a short distance and took a running leap at it, only to fall short once more. Again and again he tried, but in vain. Now he sat down and looked at the grapes in disgust. "What a fool I am," he said. "Here I am wearing myself out to get a bunch of sour grapes that are not worth gaping for." And off he walked very, very scornfully.There are many who pretend to despise and belittle that which is beyond their reach.`;
const testFilePath = "the_fox_and_the_grapes.md";
await fs.writeFile(testFilePath, new TextEncoder().encode(testContent));
const rules: ClassifierRule[] = [
{
type: "fable",
description:
"A short story featuring animals whose aim is to teach the reader a lesson (the moral of the story)",
},
{
type: "fairy_tale",
description:
"A mid-to-long story featuring humans, magic creatures and other characters, whose main aim is to entertain the readers.",
},
];
{type: "fable", description: "A short story featuring animals whose aim is to teach the reader a lesson (the moral of the story)"},
{type: "fairy_tale", description: "A mid-to-long story featuring humans, magic creatures and other characters, whose main aim is to entertain the readers."}
]
const parsingConfig: ClassifyParsingConfiguration = { lang: "en" };
const parsingConfig: ClassifyParsingConfiguration = {lang: "en"}
const result = await classifyClient.classify(rules, parsingConfig, {
filePaths: ["the_fox_and_the_grapes.md"],
});
const result = await classifyClient.classify(
rules,
parsingConfig,
undefined,
["the_fox_and_the_grapes.md"]
);
expect("items" in result).toBeTruthy();
expect(result.items.length).toBeGreaterThan(0);
expect("result" in result.items[0]).toBeTruthy();
@@ -534,7 +527,7 @@ describe("Integration Tests", () => {
const resultBuffer = await classifyClient.classify(
rules,
parsingConfig,
{ fileContents: [buffer] },
[buffer],
);
expect("items" in resultBuffer).toBeTruthy();
expect(resultBuffer.items.length).toBeGreaterThan(0);
@@ -542,11 +535,9 @@ describe("Integration Tests", () => {
expect(resultBuffer.items[0].result!.type === "fable").toBeTruthy();
try {
await fs.unlink("the_fox_and_the_grapes.md");
} catch (err) {
console.log(
`Unable to delete file the_fox_and_the_grapes.md because of ${err}`,
);
await fs.unlink("the_fox_and_the_grapes.md")
} catch(err) {
console.log(`Unable to delete file the_fox_and_the_grapes.md because of ${err}`)
}
},
60000,