Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ed363fbea5 | |||
| b2bb7a179e | |||
| c0b9c23bb0 | |||
| 399076d253 | |||
| 5b43f0d63b | |||
| a76aeac498 | |||
| 5250244113 | |||
| b57cd0dcbc | |||
| df9bfeed67 | |||
| ef4596cf14 |
@@ -1,8 +0,0 @@
|
||||
# Changesets
|
||||
|
||||
Hello and welcome! This folder has been automatically generated by `@changesets/cli`, a build tool that works
|
||||
with multi-package repos, or single-package repos to help you version and publish your code. You can
|
||||
find the full documentation for it [in our repository](https://github.com/changesets/changesets)
|
||||
|
||||
We have a quick list of common questions to get you started engaging with this project in
|
||||
[our documentation](https://github.com/changesets/changesets/blob/main/docs/common-questions.md)
|
||||
@@ -1,11 +0,0 @@
|
||||
{
|
||||
"$schema": "https://unpkg.com/@changesets/config@3.1.1/schema.json",
|
||||
"changelog": "@changesets/cli/changelog",
|
||||
"commit": false,
|
||||
"fixed": [],
|
||||
"linked": [],
|
||||
"access": "restricted",
|
||||
"baseBranch": "main",
|
||||
"updateInternalDependencies": "patch",
|
||||
"ignore": []
|
||||
}
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
version: ${{ env.UV_VERSION }}
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
- uses: pnpm/action-setup@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v5
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: "ts/llama_cloud_services/.nvmrc"
|
||||
|
||||
|
||||
@@ -30,12 +30,12 @@ jobs:
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4
|
||||
uses: github/codeql-action/init@v3
|
||||
with:
|
||||
languages: python
|
||||
dependency-caching: true
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v4
|
||||
uses: github/codeql-action/analyze@v3
|
||||
with:
|
||||
category: "/language:python"
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
with:
|
||||
fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }}
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
version: ${{ env.UV_VERSION }}
|
||||
|
||||
@@ -31,7 +31,7 @@ jobs:
|
||||
|
||||
- uses: pnpm/action-setup@v4
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v5
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: "ts/llama_cloud_services/.nvmrc"
|
||||
- name: Install dependencies
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
name: Publish Release - Python
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
UV_VERSION: "0.7.20"
|
||||
|
||||
jobs:
|
||||
build-n-publish:
|
||||
name: Build and publish to PyPI
|
||||
if: github.repository == 'run-llama/llama_cloud_services'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
version: ${{ env.UV_VERSION }}
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install
|
||||
|
||||
- name: Display Python version
|
||||
run: python --version
|
||||
|
||||
- name: Build
|
||||
working-directory: py
|
||||
run: uv build
|
||||
|
||||
- name: Test installing built package
|
||||
shell: bash
|
||||
working-directory: py
|
||||
run: |
|
||||
uv venv
|
||||
uv pip install dist/*.whl
|
||||
|
||||
- name: Publish package
|
||||
shell: bash
|
||||
working-directory: py
|
||||
run: uv publish --token ${{ secrets.LLAMA_PARSE_PYPI_TOKEN }}
|
||||
|
||||
- name: Build and publish llama-parse
|
||||
working-directory: py/llama_parse/
|
||||
run: |
|
||||
uv build
|
||||
uv publish --token ${{ secrets.LLAMA_PARSE_PYPI_TOKEN }}
|
||||
|
||||
- name: Create GitHub Release
|
||||
id: create_release
|
||||
uses: actions/create-release@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token
|
||||
with:
|
||||
tag_name: ${{ github.ref }}
|
||||
release_name: ${{ github.ref }} - LlamaCloud Services PY
|
||||
artifacts: "py/**/dist/*"
|
||||
generateReleaseNotes: true
|
||||
draft: false
|
||||
prerelease: false
|
||||
@@ -0,0 +1,52 @@
|
||||
name: Publish Release - TypeScript
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "llama-cloud-services@*"
|
||||
|
||||
jobs:
|
||||
build-and-publish:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout Repo
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- uses: pnpm/action-setup@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: "ts/llama_cloud_services/.nvmrc"
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --no-frozen-lockfile
|
||||
|
||||
- name: Run Build
|
||||
working-directory: ts/llama_cloud_services/
|
||||
run: pnpm build
|
||||
|
||||
- name: Build tarball
|
||||
run: |
|
||||
pnpm pack
|
||||
working-directory: ts/llama_cloud_services
|
||||
|
||||
- name: Setup npm authentication
|
||||
run: echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/.npmrc
|
||||
env:
|
||||
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
|
||||
- name: Release
|
||||
working-directory: ts/llama_cloud_services
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
run: pnpm publish --access public --no-git-checks
|
||||
|
||||
- name: Create release
|
||||
uses: ncipollo/release-action@v1
|
||||
with:
|
||||
artifacts: "ts/llama_cloud_services/llama-cloud-services*.tgz"
|
||||
name: Release ${{ github.ref_name }} - LlamaCloud Services TS
|
||||
generateReleaseNotes: true
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
version: ${{ env.UV_VERSION }}
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
version: ${{ env.UV_VERSION }}
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ jobs:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: pnpm/action-setup@v4
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v5
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: "ts/llama_cloud_services/.nvmrc"
|
||||
- name: Install dependencies
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
name: Version Bump and Release
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
concurrency: ${{ github.workflow }}-${{ github.ref }}
|
||||
|
||||
jobs:
|
||||
release:
|
||||
name: Release
|
||||
runs-on: ubuntu-latest
|
||||
# Only run on main branch pushes
|
||||
if: github.ref == 'refs/heads/main'
|
||||
steps:
|
||||
- name: Checkout Repo
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- uses: pnpm/action-setup@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v5
|
||||
with:
|
||||
node-version: "22"
|
||||
cache: "pnpm"
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install
|
||||
|
||||
- name: Add auth token to .npmrc file
|
||||
run: |
|
||||
cat << EOF >> ".npmrc"
|
||||
//registry.npmjs.org/:_authToken=$NPM_TOKEN
|
||||
EOF
|
||||
env:
|
||||
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
|
||||
- name: Create Release Pull Request or Publish packages
|
||||
id: changesets
|
||||
uses: changesets/action@v1
|
||||
with:
|
||||
commit: "chore: version packages"
|
||||
title: "chore: version packages"
|
||||
# Custom version script
|
||||
version: pnpm -w run version
|
||||
# Custom publish script
|
||||
publish: pnpm -w run publish
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }}
|
||||
LLAMA_PARSE_PYPI_TOKEN: ${{ secrets.LLAMA_PARSE_PYPI_TOKEN }}
|
||||
@@ -9,4 +9,3 @@ __pycache__/
|
||||
node_modules/
|
||||
.turbo/
|
||||
dist/
|
||||
.npmrc
|
||||
|
||||
@@ -29,7 +29,7 @@ repos:
|
||||
- id: black-jupyter
|
||||
name: black-src
|
||||
alias: black
|
||||
exclude: ".*uv.lock|examples/extract/solar_panel_e2e_comparison.ipynb"
|
||||
exclude: ".*uv.lock"
|
||||
- repo: https://github.com/pre-commit/mirrors-mypy
|
||||
rev: v1.0.1
|
||||
hooks:
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
node_modules
|
||||
package-lock.json
|
||||
yarn.lock
|
||||
|
||||
.DS_Store
|
||||
.cache
|
||||
.env
|
||||
.vercel
|
||||
.output
|
||||
.nitro
|
||||
/build/
|
||||
/api/
|
||||
/server/build
|
||||
/public/build# Sentry Config File
|
||||
.env.sentry-build-plugin
|
||||
/test-results/
|
||||
/playwright-report/
|
||||
/blob-report/
|
||||
/playwright/.cache/
|
||||
.tanstack
|
||||
.vscode
|
||||
@@ -1,4 +0,0 @@
|
||||
**/build
|
||||
**/public
|
||||
pnpm-lock.yaml
|
||||
routeTree.gen.ts
|
||||
@@ -1,88 +0,0 @@
|
||||
# LlamaClassify Demo
|
||||
|
||||
A TypeScript demo application showcasing the power of **LlamaClassify** - an agentic documents classification service from [LlamaCloud](https://cloud.llamaindex.ai). This demo allows you to classify financial documents among three different types (Cash flow statement, Income Statement and Balance Sheet).
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Features](#features)
|
||||
- [Prerequisites](#prerequisites)
|
||||
- [Installation](#installation)
|
||||
- [Usage](#usage)
|
||||
- [Start the Demo](#start-the-demo)
|
||||
- [How It Works](#how-it-works)
|
||||
- [Troubleshooting](#troubleshooting)
|
||||
- [Common Issues](#common-issues)
|
||||
- [License](#license)
|
||||
- [Contributing](#contributing)
|
||||
|
||||
## Features
|
||||
|
||||
- 📄 **Documemt Classification**: Classify files based on well-defined rules you can customized and play around with.
|
||||
- 🤖 **Reasoning-based Actionable Insights**: Get in-depth, reasoning based insights on the document classification, accompanied by confidence scores.
|
||||
- 🎨 **Beautiful UI**: [DaisyUI](https://daisyui.com)-based interface powered by [TanStack](https://tanstack.com)
|
||||
- ⚡ **Fast Development**: Hot reload support with development mode
|
||||
- 🛠️ **TypeScript**: Full TypeScript support with strict type checking
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Node.js (version 22 or higher)
|
||||
- pnpm package manager
|
||||
- LlamaCloud API key
|
||||
|
||||
## Installation
|
||||
|
||||
1. Clone the repository:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/run-llama/llama_cloud_services
|
||||
cd lama_cloud_services/examples-ts/classify/
|
||||
```
|
||||
|
||||
2. Install dependencies:
|
||||
|
||||
```bash
|
||||
npm install
|
||||
```
|
||||
|
||||
3. Set up your environment variables:
|
||||
|
||||
```bash
|
||||
# Add your API key to your environment
|
||||
export LLAMA_CLOUD_API_KEY="your-llamacloud-api-key"
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### Start the Demo
|
||||
|
||||
```bash
|
||||
npm run dev
|
||||
```
|
||||
|
||||
The application will be up and running on http://localhost:3000
|
||||
|
||||
## How It Works
|
||||
|
||||
1. **Document Input**: Enter the path to your document when prompted
|
||||
2. **Parsing**: LlamaClassify, based on the rules you can find [here](./src/utils/classifier.ts), processes the document and classifies it
|
||||
3. **Results**: The classification outcome, as well as the reasoning behind it and the confidence score, are displayed in the UI.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
1. **Module Resolution Errors**: Ensure you're using Node.js 22+ and have all dependencies installed
|
||||
2. **API Key Issues**: Verify your LlamaCloud API key is correctly set
|
||||
3. **File Path Errors**: Use absolute paths or ensure relative paths are correct from the project root
|
||||
|
||||
## License
|
||||
|
||||
MIT License - see the [LICENSE](../../LICENSE) file for details.
|
||||
|
||||
## Contributing
|
||||
|
||||
1. Fork the repository
|
||||
2. Create a feature branch
|
||||
3. Make your changes
|
||||
4. Run `npm run format` and `npm run lint`
|
||||
5. Submit a pull request
|
||||
@@ -1,34 +0,0 @@
|
||||
{
|
||||
"name": "tanstack-start-example-basic",
|
||||
"private": true,
|
||||
"sideEffects": false,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite dev",
|
||||
"build": "vite build && tsc --noEmit",
|
||||
"start": "node .output/server/index.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@tanstack/react-router": "^1.133.22",
|
||||
"@tanstack/react-router-devtools": "^1.133.22",
|
||||
"@tanstack/react-start": "^1.133.22",
|
||||
"llama-cloud-services": "^0.3.10",
|
||||
"react": "^19.0.0",
|
||||
"react-dom": "^19.0.0",
|
||||
"tailwind-merge": "^2.6.0",
|
||||
"zod": "^3.24.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/postcss": "^4.1.15",
|
||||
"@types/node": "^22.5.4",
|
||||
"@types/react": "^19.0.8",
|
||||
"@types/react-dom": "^19.0.3",
|
||||
"@vitejs/plugin-react": "^4.6.0",
|
||||
"daisyui": "^5.3.7",
|
||||
"postcss": "^8.5.1",
|
||||
"tailwindcss": "^4.1.15",
|
||||
"typescript": "^5.7.2",
|
||||
"vite": "^7.1.7",
|
||||
"vite-tsconfig-paths": "^5.1.4"
|
||||
}
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
export default {
|
||||
plugins: {
|
||||
'@tailwindcss/postcss': {},
|
||||
},
|
||||
}
|
||||
|
Before Width: | Height: | Size: 3.3 KiB |
|
Before Width: | Height: | Size: 21 KiB |
|
Before Width: | Height: | Size: 3.8 KiB |
|
Before Width: | Height: | Size: 862 B |
|
Before Width: | Height: | Size: 1.1 KiB |
|
Before Width: | Height: | Size: 1.1 KiB |
|
Before Width: | Height: | Size: 2.0 KiB |
@@ -1,19 +0,0 @@
|
||||
{
|
||||
"name": "",
|
||||
"short_name": "",
|
||||
"icons": [
|
||||
{
|
||||
"src": "/android-chrome-192x192.png",
|
||||
"sizes": "192x192",
|
||||
"type": "image/png"
|
||||
},
|
||||
{
|
||||
"src": "/android-chrome-512x512.png",
|
||||
"sizes": "512x512",
|
||||
"type": "image/png"
|
||||
}
|
||||
],
|
||||
"theme_color": "#ffffff",
|
||||
"background_color": "#ffffff",
|
||||
"display": "standalone"
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
import {
|
||||
ErrorComponent,
|
||||
Link,
|
||||
rootRouteId,
|
||||
useMatch,
|
||||
useRouter,
|
||||
} from '@tanstack/react-router'
|
||||
import type { ErrorComponentProps } from '@tanstack/react-router'
|
||||
|
||||
export function DefaultCatchBoundary({ error }: ErrorComponentProps) {
|
||||
const router = useRouter()
|
||||
const isRoot = useMatch({
|
||||
strict: false,
|
||||
select: (state) => state.id === rootRouteId,
|
||||
})
|
||||
|
||||
console.error('DefaultCatchBoundary Error:', error)
|
||||
|
||||
return (
|
||||
<div className="min-w-0 flex-1 p-4 flex flex-col items-center justify-center gap-6">
|
||||
<ErrorComponent error={error} />
|
||||
<div className="flex gap-2 items-center flex-wrap">
|
||||
<button
|
||||
onClick={() => {
|
||||
router.invalidate()
|
||||
}}
|
||||
className={`px-2 py-1 bg-gray-600 dark:bg-gray-700 rounded-sm text-white uppercase font-extrabold`}
|
||||
>
|
||||
Try Again
|
||||
</button>
|
||||
{isRoot ? (
|
||||
<Link
|
||||
to="/"
|
||||
className={`px-2 py-1 bg-gray-600 dark:bg-gray-700 rounded-sm text-white uppercase font-extrabold`}
|
||||
>
|
||||
Home
|
||||
</Link>
|
||||
) : (
|
||||
<Link
|
||||
to="/"
|
||||
className={`px-2 py-1 bg-gray-600 dark:bg-gray-700 rounded-sm text-white uppercase font-extrabold`}
|
||||
onClick={(e) => {
|
||||
e.preventDefault()
|
||||
window.history.back()
|
||||
}}
|
||||
>
|
||||
Go Back
|
||||
</Link>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
import { Link } from '@tanstack/react-router'
|
||||
|
||||
export function NotFound({ children }: { children?: any }) {
|
||||
return (
|
||||
<div className="space-y-2 p-2">
|
||||
<div className="text-gray-600 dark:text-gray-400">
|
||||
{children || <p>The page you are looking for does not exist.</p>}
|
||||
</div>
|
||||
<p className="flex items-center gap-2 flex-wrap">
|
||||
<button
|
||||
onClick={() => window.history.back()}
|
||||
className="bg-emerald-500 text-white px-2 py-1 rounded-sm uppercase font-black text-sm"
|
||||
>
|
||||
Go back
|
||||
</button>
|
||||
<Link
|
||||
to="/"
|
||||
className="bg-cyan-600 text-white px-2 py-1 rounded-sm uppercase font-black text-sm"
|
||||
>
|
||||
Start Over
|
||||
</Link>
|
||||
</p>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,225 +0,0 @@
|
||||
/* eslint-disable */
|
||||
|
||||
// @ts-nocheck
|
||||
|
||||
// noinspection JSUnusedGlobalSymbols
|
||||
|
||||
// This file was automatically generated by TanStack Router.
|
||||
// You should NOT make any changes in this file as it will be overwritten.
|
||||
// Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified.
|
||||
|
||||
import { Route as rootRouteImport } from './routes/__root'
|
||||
import { Route as UsersRouteImport } from './routes/users'
|
||||
import { Route as IndexRouteImport } from './routes/index'
|
||||
import { Route as UsersIndexRouteImport } from './routes/users.index'
|
||||
import { Route as PostsIndexRouteImport } from './routes/posts.index'
|
||||
import { Route as UsersUserIdRouteImport } from './routes/users.$userId'
|
||||
import { Route as PostsPostIdRouteImport } from './routes/posts.$postId'
|
||||
import { Route as ApiClassifyRouteImport } from './routes/api/classify'
|
||||
import { Route as PostsPostIdDeepRouteImport } from './routes/posts_.$postId.deep'
|
||||
|
||||
const UsersRoute = UsersRouteImport.update({
|
||||
id: '/users',
|
||||
path: '/users',
|
||||
getParentRoute: () => rootRouteImport,
|
||||
} as any)
|
||||
const IndexRoute = IndexRouteImport.update({
|
||||
id: '/',
|
||||
path: '/',
|
||||
getParentRoute: () => rootRouteImport,
|
||||
} as any)
|
||||
const UsersIndexRoute = UsersIndexRouteImport.update({
|
||||
id: '/',
|
||||
path: '/',
|
||||
getParentRoute: () => UsersRoute,
|
||||
} as any)
|
||||
const PostsIndexRoute = PostsIndexRouteImport.update({
|
||||
id: '/posts/',
|
||||
path: '/posts/',
|
||||
getParentRoute: () => rootRouteImport,
|
||||
} as any)
|
||||
const UsersUserIdRoute = UsersUserIdRouteImport.update({
|
||||
id: '/$userId',
|
||||
path: '/$userId',
|
||||
getParentRoute: () => UsersRoute,
|
||||
} as any)
|
||||
const PostsPostIdRoute = PostsPostIdRouteImport.update({
|
||||
id: '/posts/$postId',
|
||||
path: '/posts/$postId',
|
||||
getParentRoute: () => rootRouteImport,
|
||||
} as any)
|
||||
const ApiClassifyRoute = ApiClassifyRouteImport.update({
|
||||
id: '/api/classify',
|
||||
path: '/api/classify',
|
||||
getParentRoute: () => rootRouteImport,
|
||||
} as any)
|
||||
const PostsPostIdDeepRoute = PostsPostIdDeepRouteImport.update({
|
||||
id: '/posts_/$postId/deep',
|
||||
path: '/posts/$postId/deep',
|
||||
getParentRoute: () => rootRouteImport,
|
||||
} as any)
|
||||
|
||||
export interface FileRoutesByFullPath {
|
||||
'/': typeof IndexRoute
|
||||
'/users': typeof UsersRouteWithChildren
|
||||
'/api/classify': typeof ApiClassifyRoute
|
||||
'/posts/$postId': typeof PostsPostIdRoute
|
||||
'/users/$userId': typeof UsersUserIdRoute
|
||||
'/posts': typeof PostsIndexRoute
|
||||
'/users/': typeof UsersIndexRoute
|
||||
'/posts/$postId/deep': typeof PostsPostIdDeepRoute
|
||||
}
|
||||
export interface FileRoutesByTo {
|
||||
'/': typeof IndexRoute
|
||||
'/api/classify': typeof ApiClassifyRoute
|
||||
'/posts/$postId': typeof PostsPostIdRoute
|
||||
'/users/$userId': typeof UsersUserIdRoute
|
||||
'/posts': typeof PostsIndexRoute
|
||||
'/users': typeof UsersIndexRoute
|
||||
'/posts/$postId/deep': typeof PostsPostIdDeepRoute
|
||||
}
|
||||
export interface FileRoutesById {
|
||||
__root__: typeof rootRouteImport
|
||||
'/': typeof IndexRoute
|
||||
'/users': typeof UsersRouteWithChildren
|
||||
'/api/classify': typeof ApiClassifyRoute
|
||||
'/posts/$postId': typeof PostsPostIdRoute
|
||||
'/users/$userId': typeof UsersUserIdRoute
|
||||
'/posts/': typeof PostsIndexRoute
|
||||
'/users/': typeof UsersIndexRoute
|
||||
'/posts_/$postId/deep': typeof PostsPostIdDeepRoute
|
||||
}
|
||||
export interface FileRouteTypes {
|
||||
fileRoutesByFullPath: FileRoutesByFullPath
|
||||
fullPaths:
|
||||
| '/'
|
||||
| '/users'
|
||||
| '/api/classify'
|
||||
| '/posts/$postId'
|
||||
| '/users/$userId'
|
||||
| '/posts'
|
||||
| '/users/'
|
||||
| '/posts/$postId/deep'
|
||||
fileRoutesByTo: FileRoutesByTo
|
||||
to:
|
||||
| '/'
|
||||
| '/api/classify'
|
||||
| '/posts/$postId'
|
||||
| '/users/$userId'
|
||||
| '/posts'
|
||||
| '/users'
|
||||
| '/posts/$postId/deep'
|
||||
id:
|
||||
| '__root__'
|
||||
| '/'
|
||||
| '/users'
|
||||
| '/api/classify'
|
||||
| '/posts/$postId'
|
||||
| '/users/$userId'
|
||||
| '/posts/'
|
||||
| '/users/'
|
||||
| '/posts_/$postId/deep'
|
||||
fileRoutesById: FileRoutesById
|
||||
}
|
||||
export interface RootRouteChildren {
|
||||
IndexRoute: typeof IndexRoute
|
||||
UsersRoute: typeof UsersRouteWithChildren
|
||||
ApiClassifyRoute: typeof ApiClassifyRoute
|
||||
PostsPostIdRoute: typeof PostsPostIdRoute
|
||||
PostsIndexRoute: typeof PostsIndexRoute
|
||||
PostsPostIdDeepRoute: typeof PostsPostIdDeepRoute
|
||||
}
|
||||
|
||||
declare module '@tanstack/react-router' {
|
||||
interface FileRoutesByPath {
|
||||
'/users': {
|
||||
id: '/users'
|
||||
path: '/users'
|
||||
fullPath: '/users'
|
||||
preLoaderRoute: typeof UsersRouteImport
|
||||
parentRoute: typeof rootRouteImport
|
||||
}
|
||||
'/': {
|
||||
id: '/'
|
||||
path: '/'
|
||||
fullPath: '/'
|
||||
preLoaderRoute: typeof IndexRouteImport
|
||||
parentRoute: typeof rootRouteImport
|
||||
}
|
||||
'/users/': {
|
||||
id: '/users/'
|
||||
path: '/'
|
||||
fullPath: '/users/'
|
||||
preLoaderRoute: typeof UsersIndexRouteImport
|
||||
parentRoute: typeof UsersRoute
|
||||
}
|
||||
'/posts/': {
|
||||
id: '/posts/'
|
||||
path: '/posts'
|
||||
fullPath: '/posts'
|
||||
preLoaderRoute: typeof PostsIndexRouteImport
|
||||
parentRoute: typeof rootRouteImport
|
||||
}
|
||||
'/users/$userId': {
|
||||
id: '/users/$userId'
|
||||
path: '/$userId'
|
||||
fullPath: '/users/$userId'
|
||||
preLoaderRoute: typeof UsersUserIdRouteImport
|
||||
parentRoute: typeof UsersRoute
|
||||
}
|
||||
'/posts/$postId': {
|
||||
id: '/posts/$postId'
|
||||
path: '/posts/$postId'
|
||||
fullPath: '/posts/$postId'
|
||||
preLoaderRoute: typeof PostsPostIdRouteImport
|
||||
parentRoute: typeof rootRouteImport
|
||||
}
|
||||
'/api/classify': {
|
||||
id: '/api/classify'
|
||||
path: '/api/classify'
|
||||
fullPath: '/api/classify'
|
||||
preLoaderRoute: typeof ApiClassifyRouteImport
|
||||
parentRoute: typeof rootRouteImport
|
||||
}
|
||||
'/posts_/$postId/deep': {
|
||||
id: '/posts_/$postId/deep'
|
||||
path: '/posts/$postId/deep'
|
||||
fullPath: '/posts/$postId/deep'
|
||||
preLoaderRoute: typeof PostsPostIdDeepRouteImport
|
||||
parentRoute: typeof rootRouteImport
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
interface UsersRouteChildren {
|
||||
UsersUserIdRoute: typeof UsersUserIdRoute
|
||||
UsersIndexRoute: typeof UsersIndexRoute
|
||||
}
|
||||
|
||||
const UsersRouteChildren: UsersRouteChildren = {
|
||||
UsersUserIdRoute: UsersUserIdRoute,
|
||||
UsersIndexRoute: UsersIndexRoute,
|
||||
}
|
||||
|
||||
const UsersRouteWithChildren = UsersRoute._addFileChildren(UsersRouteChildren)
|
||||
|
||||
const rootRouteChildren: RootRouteChildren = {
|
||||
IndexRoute: IndexRoute,
|
||||
UsersRoute: UsersRouteWithChildren,
|
||||
ApiClassifyRoute: ApiClassifyRoute,
|
||||
PostsPostIdRoute: PostsPostIdRoute,
|
||||
PostsIndexRoute: PostsIndexRoute,
|
||||
PostsPostIdDeepRoute: PostsPostIdDeepRoute,
|
||||
}
|
||||
export const routeTree = rootRouteImport
|
||||
._addFileChildren(rootRouteChildren)
|
||||
._addFileTypes<FileRouteTypes>()
|
||||
|
||||
import type { getRouter } from './router.tsx'
|
||||
import type { createStart } from '@tanstack/react-start'
|
||||
declare module '@tanstack/react-start' {
|
||||
interface Register {
|
||||
ssr: true
|
||||
router: Awaited<ReturnType<typeof getRouter>>
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
import { createRouter } from '@tanstack/react-router'
|
||||
import { routeTree } from './routeTree.gen'
|
||||
import { DefaultCatchBoundary } from './components/DefaultCatchBoundary'
|
||||
import { NotFound } from './components/NotFound'
|
||||
|
||||
export function getRouter() {
|
||||
const router = createRouter({
|
||||
routeTree,
|
||||
defaultPreload: 'intent',
|
||||
defaultErrorComponent: DefaultCatchBoundary,
|
||||
defaultNotFoundComponent: () => <NotFound />,
|
||||
scrollRestoration: true,
|
||||
})
|
||||
return router
|
||||
}
|
||||
@@ -1,128 +0,0 @@
|
||||
/// <reference types="vite/client" />
|
||||
import {
|
||||
HeadContent,
|
||||
Scripts,
|
||||
createRootRoute,
|
||||
} from '@tanstack/react-router'
|
||||
import * as React from 'react'
|
||||
import { DefaultCatchBoundary } from '~/components/DefaultCatchBoundary'
|
||||
import { NotFound } from '~/components/NotFound'
|
||||
import { seo } from '~/utils/seo'
|
||||
|
||||
export const Route = createRootRoute({
|
||||
head: () => ({
|
||||
meta: [
|
||||
{
|
||||
charSet: 'utf-8',
|
||||
},
|
||||
{
|
||||
name: 'viewport',
|
||||
content: 'width=device-width, initial-scale=1',
|
||||
},
|
||||
...seo({
|
||||
title:
|
||||
'Financial Documents Classification Agent',
|
||||
description: `Classify financial documents as balance sheets, income statements and cash flow statemets. `,
|
||||
}),
|
||||
],
|
||||
links: [
|
||||
{ rel: 'stylesheet', href: "https://cdn.jsdelivr.net/npm/daisyui@5" },
|
||||
{
|
||||
rel: 'apple-touch-icon',
|
||||
sizes: '180x180',
|
||||
href: '/apple-touch-icon.png',
|
||||
},
|
||||
{
|
||||
rel: 'icon',
|
||||
type: 'image/png',
|
||||
sizes: '32x32',
|
||||
href: '/favicon-32x32.png',
|
||||
},
|
||||
{
|
||||
rel: 'icon',
|
||||
type: 'image/png',
|
||||
sizes: '16x16',
|
||||
href: '/favicon-16x16.png',
|
||||
},
|
||||
{ rel: 'manifest', href: '/site.webmanifest', color: '#fffff' },
|
||||
{ rel: 'icon', href: '/favicon.ico' },
|
||||
],
|
||||
scripts: [
|
||||
{
|
||||
src: '/customScript.js',
|
||||
type: 'text/javascript',
|
||||
},
|
||||
{
|
||||
src: "https://cdn.jsdelivr.net/npm/@tailwindcss/browser@4",
|
||||
type: "text/javascript",
|
||||
}
|
||||
],
|
||||
}),
|
||||
errorComponent: DefaultCatchBoundary,
|
||||
notFoundComponent: () => <NotFound />,
|
||||
shellComponent: RootDocument,
|
||||
})
|
||||
|
||||
function RootDocument({ children }: { children: React.ReactNode }) {
|
||||
return (
|
||||
<html>
|
||||
<head>
|
||||
<HeadContent />
|
||||
</head>
|
||||
<body>
|
||||
<div className="navbar bg-base-100 shadow-sm">
|
||||
<div className="navbar-start">
|
||||
<div className="dropdown">
|
||||
<div tabIndex={0} role="button" className="btn btn-ghost btn-circle">
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
className="h-5 w-5"
|
||||
fill="none"
|
||||
viewBox="0 0 24 24"
|
||||
stroke="currentColor"
|
||||
>
|
||||
<path
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
strokeWidth="2"
|
||||
d="M4 6h16M4 12h16M4 18h7"
|
||||
/>
|
||||
</svg>
|
||||
</div>
|
||||
<ul
|
||||
tabIndex={0}
|
||||
className="menu menu-lg dropdown-content bg-base-100 rounded-box z-1 mt-3 w-80 p-2 shadow"
|
||||
>
|
||||
<li><a href="/">Home</a></li>
|
||||
<li><a href="https://cloud.llamaindex.ai">Get Started with LlamaCloud</a></li>
|
||||
<li><a href="https://developers.llamaindex.ai/python/cloud/llamaclassify/getting_started/">LlamaClassify Docs</a></li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
<div className="navbar-center">
|
||||
<a className="btn btn-ghost text-xl" href="/">Financial Documents Classification Agent</a>
|
||||
</div>
|
||||
<div className="navbar-end">
|
||||
<a href="https://github.com/run-llama/llama_cloud_services/main/blob/examples-ts/classify">
|
||||
<button className="btn btn-ghost btn-circle">
|
||||
<div className="indicator">
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
className="h-10 w-10"
|
||||
fill="currentColor"
|
||||
viewBox="0 0 640 512"
|
||||
>
|
||||
<path d="M237.9 461.4C237.9 463.4 235.6 465 232.7 465C229.4 465.3 227.1 463.7 227.1 461.4C227.1 459.4 229.4 457.8 232.3 457.8C235.3 457.5 237.9 459.1 237.9 461.4zM206.8 456.9C206.1 458.9 208.1 461.2 211.1 461.8C213.7 462.8 216.7 461.8 217.3 459.8C217.9 457.8 216 455.5 213 454.6C210.4 453.9 207.5 454.9 206.8 456.9zM251 455.2C248.1 455.9 246.1 457.8 246.4 460.1C246.7 462.1 249.3 463.4 252.3 462.7C255.2 462 257.2 460.1 256.9 458.1C256.6 456.2 253.9 454.9 251 455.2zM316.8 72C178.1 72 72 177.3 72 316C72 426.9 141.8 521.8 241.5 555.2C254.3 557.5 258.8 549.6 258.8 543.1C258.8 536.9 258.5 502.7 258.5 481.7C258.5 481.7 188.5 496.7 173.8 451.9C173.8 451.9 162.4 422.8 146 415.3C146 415.3 123.1 399.6 147.6 399.9C147.6 399.9 172.5 401.9 186.2 425.7C208.1 464.3 244.8 453.2 259.1 446.6C261.4 430.6 267.9 419.5 275.1 412.9C219.2 406.7 162.8 398.6 162.8 302.4C162.8 274.9 170.4 261.1 186.4 243.5C183.8 237 175.3 210.2 189 175.6C209.9 169.1 258 202.6 258 202.6C278 197 299.5 194.1 320.8 194.1C342.1 194.1 363.6 197 383.6 202.6C383.6 202.6 431.7 169 452.6 175.6C466.3 210.3 457.8 237 455.2 243.5C471.2 261.2 481 275 481 302.4C481 398.9 422.1 406.6 366.2 412.9C375.4 420.8 383.2 435.8 383.2 459.3C383.2 493 382.9 534.7 382.9 542.9C382.9 549.4 387.5 557.3 400.2 555C500.2 521.8 568 426.9 568 316C568 177.3 455.5 72 316.8 72zM169.2 416.9C167.9 417.9 168.2 420.2 169.9 422.1C171.5 423.7 173.8 424.4 175.1 423.1C176.4 422.1 176.1 419.8 174.4 417.9C172.8 416.3 170.5 415.6 169.2 416.9zM158.4 408.8C157.7 410.1 158.7 411.7 160.7 412.7C162.3 413.7 164.3 413.4 165 412C165.7 410.7 164.7 409.1 162.7 408.1C160.7 407.5 159.1 407.8 158.4 408.8zM190.8 444.4C189.2 445.7 189.8 448.7 192.1 450.6C194.4 452.9 197.3 453.2 198.6 451.6C199.9 450.3 199.3 447.3 197.3 445.4C195.1 443.1 192.1 442.8 190.8 444.4zM179.4 429.7C177.8 430.7 177.8 433.3 179.4 435.6C181 437.9 183.7 438.9 185 437.9C186.6 436.6 186.6 434 185 431.7C183.6 429.4 181 428.4 179.4 429.7z" />
|
||||
</svg>
|
||||
</div>
|
||||
</button>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
<hr />
|
||||
{children}
|
||||
<Scripts />
|
||||
</body>
|
||||
</html>
|
||||
)
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
import { createFileRoute } from '@tanstack/react-router'
|
||||
import { classifier, classificationRules, parsingConfig } from '~/utils/classifier'
|
||||
|
||||
export const Route = createFileRoute('/api/classify')({
|
||||
component: RouteComponent,
|
||||
server: {
|
||||
handlers: {
|
||||
POST: async ({ request }) => {
|
||||
const body = await request.formData()
|
||||
const fl = body.get("file") as File;
|
||||
if (!fl) {
|
||||
return new Response(JSON.stringify({"result": "you need to provide a file"}))
|
||||
}
|
||||
const buff = await fl.arrayBuffer()
|
||||
const rawRes = await classifier.classify(
|
||||
classificationRules,
|
||||
parsingConfig,
|
||||
[new Uint8Array(buff)],
|
||||
)
|
||||
const results = rawRes.items
|
||||
let classification = ""
|
||||
|
||||
for (const result of results) {
|
||||
if ("result" in result && result.result) {
|
||||
classification += `
|
||||
<div class="card bg-base-100 shadow-xl p-6 mb-4">
|
||||
<div class="space-y-3">
|
||||
<p><span class="font-semibold">📄 Document:</span> ${fl.name}</p>
|
||||
<p><span class="font-semibold">🏷️ Type:</span> <span class="badge badge-primary">${result.result.type}</span></p>
|
||||
<p><span class="font-semibold">📊 Confidence:</span> ${result.result.confidence*100}%</p>
|
||||
<p><span class="font-semibold">💭 Reasoning:</span> ${result.result.reasoning}</p>
|
||||
</div>
|
||||
</div>
|
||||
`
|
||||
}
|
||||
}
|
||||
return new Response(JSON.stringify({"result": classification}))
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
function RouteComponent() {
|
||||
return
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
import { createFileRoute } from '@tanstack/react-router'
|
||||
import { useRef, useState } from 'react'
|
||||
|
||||
export const Route = createFileRoute('/')({
|
||||
component: Home,
|
||||
})
|
||||
|
||||
function Home() {
|
||||
const [file, setFile] = useState<null | File>(null)
|
||||
const fileInputRef = useRef<HTMLInputElement>(null)
|
||||
const [reply, setReply] = useState<null | string>(null)
|
||||
const [loading, setLoading] = useState<boolean>(false)
|
||||
const handleFileChange = (event: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const selectedFile = event.target.files?.[0]
|
||||
if (selectedFile) {
|
||||
setFile(selectedFile)
|
||||
}
|
||||
}
|
||||
const handleClearFile = () => {
|
||||
if (file) {
|
||||
setFile(null)
|
||||
}
|
||||
if (fileInputRef.current) {
|
||||
fileInputRef.current.value = ''
|
||||
}
|
||||
if (reply) {
|
||||
setReply(null)
|
||||
}
|
||||
}
|
||||
|
||||
const handleClassify = async () => {
|
||||
if (!file) return
|
||||
|
||||
if (reply) {
|
||||
setReply(null)
|
||||
}
|
||||
setLoading(true)
|
||||
try {
|
||||
const formData = new FormData()
|
||||
formData.append('file', file)
|
||||
|
||||
const res = await fetch('/api/classify', {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
})
|
||||
|
||||
const data = await res.json()
|
||||
setReply(data.result)
|
||||
} catch (error) {
|
||||
console.error('Error:', error)
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex flex-col justify-center items-center gap-y-8">
|
||||
<br />
|
||||
<h1 className="text-xl font-bold text-gray-700">AI-Powered finacial document classification</h1>
|
||||
<h2 className="text-lg font-semibold text-gray-500">Need help sorting out the financial documents jungle? Let our classification agent handle it!</h2>
|
||||
<fieldset className="fieldset bg-base-100 border-base-300 rounded-box w-200 border p-4">
|
||||
<legend className="fieldset-legend text-lg">Upload your financial document here</legend>
|
||||
<label className="label flex justify-center">
|
||||
<input type="file" className="file-input" onChange={handleFileChange} accept='application/pdf' ref={fileInputRef} />
|
||||
</label>
|
||||
</fieldset>
|
||||
{file && (
|
||||
<div className="flex flex-col justify-center items-center gap-y-8">
|
||||
<p className="text-sm text-gray-600">Selected file: {file.name}</p>
|
||||
<div className='grid grid-cols-2 gap-x-6'>
|
||||
<button
|
||||
type="button"
|
||||
className='btn bg-gray-500 text-white shadow-lg hover:bg-gray-600 hover:shadow-xl rounded'
|
||||
onClick={handleClassify}
|
||||
>
|
||||
Classify
|
||||
</button>
|
||||
<button
|
||||
onClick={handleClearFile}
|
||||
type="button"
|
||||
className="px-4 py-2 bg-red-300 text-black rounded hover:bg-red-400 hover:shadow-xl shadow-lg"
|
||||
>
|
||||
Clear
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{loading && (
|
||||
<span className="loading loading-spinner text-primary"></span>
|
||||
)}
|
||||
{reply && (
|
||||
<div
|
||||
className="max-w-2xl w-full"
|
||||
dangerouslySetInnerHTML={{ __html: reply }}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
import { LlamaClassify, ClassifierRule, ClassifyParsingConfiguration } from "llama-cloud-services"
|
||||
|
||||
export const classifier = new LlamaClassify(process.env.LLAMA_CLOUD_API_KEY);
|
||||
|
||||
export const classificationRules: ClassifierRule[] = [
|
||||
{
|
||||
description: "Shows a company's assets, liabilities, and shareholders' equity at a specific point in time, providing a snapshot of financial position.",
|
||||
type: "balance_sheet"
|
||||
},
|
||||
{
|
||||
description: "Reports cash inflows and outflows from operating, investing, and financing activities, highlighting liquidity and cash management.",
|
||||
type: "cash_flow_statement"
|
||||
},
|
||||
{
|
||||
description: "Summarizes revenues, expenses, and profits over a period, indicating financial performance and profitability.",
|
||||
type: "income_statement"
|
||||
},
|
||||
];
|
||||
|
||||
export const parsingConfig: ClassifyParsingConfiguration = {
|
||||
lang: "en",
|
||||
max_pages: 20,
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
export const seo = ({
|
||||
title,
|
||||
description,
|
||||
keywords,
|
||||
image,
|
||||
}: {
|
||||
title: string
|
||||
description?: string
|
||||
image?: string
|
||||
keywords?: string
|
||||
}) => {
|
||||
const tags = [
|
||||
{ title },
|
||||
{ name: 'description', content: description },
|
||||
{ name: 'keywords', content: keywords },
|
||||
{ name: 'twitter:title', content: title },
|
||||
{ name: 'twitter:description', content: description },
|
||||
{ name: 'twitter:creator', content: '@tannerlinsley' },
|
||||
{ name: 'twitter:site', content: '@tannerlinsley' },
|
||||
{ name: 'og:type', content: 'website' },
|
||||
{ name: 'og:title', content: title },
|
||||
{ name: 'og:description', content: description },
|
||||
...(image
|
||||
? [
|
||||
{ name: 'twitter:image', content: image },
|
||||
{ name: 'twitter:card', content: 'summary_large_image' },
|
||||
{ name: 'og:image', content: image },
|
||||
]
|
||||
: []),
|
||||
]
|
||||
|
||||
return tags
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
{
|
||||
"include": ["**/*.ts", "**/*.tsx"],
|
||||
"compilerOptions": {
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"jsx": "react-jsx",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"lib": ["DOM", "DOM.Iterable", "ES2022"],
|
||||
"isolatedModules": true,
|
||||
"resolveJsonModule": true,
|
||||
"skipLibCheck": true,
|
||||
"target": "ES2022",
|
||||
"allowJs": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"~/*": ["./src/*"]
|
||||
},
|
||||
"noEmit": true
|
||||
}
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
import { tanstackStart } from '@tanstack/react-start/plugin/vite'
|
||||
import { defineConfig } from 'vite'
|
||||
import tsConfigPaths from 'vite-tsconfig-paths'
|
||||
import viteReact from '@vitejs/plugin-react'
|
||||
|
||||
export default defineConfig({
|
||||
server: {
|
||||
port: 3000,
|
||||
},
|
||||
plugins: [
|
||||
tsConfigPaths({
|
||||
projects: ['./tsconfig.json'],
|
||||
}),
|
||||
tanstackStart({
|
||||
srcDirectory: 'src',
|
||||
}),
|
||||
viteReact(),
|
||||
],
|
||||
})
|
||||
@@ -7,7 +7,7 @@
|
||||
"source": [
|
||||
"# Extraction and Analysis over a Fidelity Multi-Fund Annual Report\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/extract/asset_manager_fund_analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services-demo/blob/main/examples/extract/asset_manager_fund_analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"\n",
|
||||
"In this notebook we show you how to create an agentic document workflow over a complex document that contains annual reports for multiple funds - each fund reports financials in a standardized reporting structure, and it's all consolidated in the same document.\n",
|
||||
"\n",
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
"source": [
|
||||
"# Automotive Equity Research: A Multi-Step Agentic Workflow\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/extract/automotive_sector_analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services-demo/blob/main/examples/extract/automotive_sector_analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"\n",
|
||||
"This notebook demonstrates an end‑to‑end agentic workflow using LlamaExtract and the LlamaIndex event‑driven workflow framework for automotive sector analysis.\n",
|
||||
"\n",
|
||||
|
||||
@@ -1035,7 +1035,7 @@
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"display_name": ".venv",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
@@ -1052,5 +1052,5 @@
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
"source": [
|
||||
"# Dynamic Section Retrieval with LlamaParse\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/parse/advanced_rag/dynamic_section_retrieval.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services-demo/blob/main/examples/parse/advanced_rag/dynamic_section_retrieval.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"\n",
|
||||
"This notebook showcases a concept called \"dynamic section retrieval\".\n",
|
||||
"\n",
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"source": [
|
||||
"# Advanced RAG with LlamaParse\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_parse/blob/main/examples/parse/demo_advanced.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_parse/blob/main/examples/demo_advanced.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"\n",
|
||||
"This notebook is a complete walkthrough for using LlamaParse with advanced indexing/retrieval techniques in LlamaIndex over the Apple 10K Filing. \n",
|
||||
"\n",
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"source": [
|
||||
"# RAG with Excel Spreadsheet using LlamaPrase\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/parse/demo_excel.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/demo_excel.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"\n",
|
||||
"This notebook shows you using LlamaParse with Excel Spreadsheet.\n",
|
||||
"\n",
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
"source": [
|
||||
"# Download Charts\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/parse/demo_get_charts.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/demo_get_charts.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"\n",
|
||||
"This notebook demonstrates how to download charts from a document using the result object.\n",
|
||||
"\n",
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"source": [
|
||||
"# LlamaParse - Fast checking Insurance Contract for Coverage\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/parse/demo_insurance.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/demo_insurance.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"\n",
|
||||
"In this notebook we will look at how LlamaParse can be used to extract structured coverage information from an insurance policy.\n",
|
||||
"\n",
|
||||
@@ -36,7 +36,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Download an insurance policy from IRDAI\n",
|
||||
"## Download an insurance policy fron IRDAI\n",
|
||||
"\n",
|
||||
"The Insurance Regulatory and Development Authority of India (IRDAI) maintains a great resource: https://policyholder.gov.in/web/guest/non-life-insurance-products where all insurance policies available in India are publicly available for download! Let's download a complex health insurance policy as an example."
|
||||
]
|
||||
@@ -228,11 +228,11 @@
|
||||
" result_type=\"markdown\",\n",
|
||||
" system_prompt_append=\"\"\"\n",
|
||||
"This document is an insurance policy.\n",
|
||||
"When a benefits/coverage/exlusion is describe in the document amend to it add a text in the following benefits string format (where coverage could be an exclusion).\n",
|
||||
"When a benefits/coverage/exlusion is describe in the document ammend to it add a text in the follwing benefits string format (where coverage could be an exclusion).\n",
|
||||
"\n",
|
||||
"For {nameofrisk} and in this condition {whenDoesThecoverageApply} the coverage is {coverageDescription}. \n",
|
||||
" \n",
|
||||
"If the document contain a benefits TABLE that describe coverage amounts, do not output it as a table, but instead as a list of benefits string.\n",
|
||||
"If the document contain a benefits TABLE that describe coverage amounts, do not ouput it as a table, but instead as a list of benefits string.\n",
|
||||
" \n",
|
||||
"\"\"\",\n",
|
||||
").aparse(\"./policy.pdf\")\n",
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
"source": [
|
||||
"# LlamaParse `JobResult` Tour\n",
|
||||
"\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/parse/demo_json.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/demo_json.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n",
|
||||
"\n",
|
||||
"The `JobResult` object is the main object returned by the LlamaParse API. It contains all the information about the job, including the parsed data, metadata, and any errors.\n",
|
||||
"\n",
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
"\n",
|
||||
"LlamaParse supports users to specify a `language` parameter before uploading documents, giving users better OCR capabilities over non-English PDFs, parsing images into more accurate representations.\n",
|
||||
"\n",
|
||||
"You can specify 80+ different languages: see this file for a full list of supported languages: https://github.com/run-llama/llama_cloud_services/blob/main/py/llama_cloud_services/parse/base.py.\n",
|
||||
"You can specify 80+ different languages: see this file for a full list of supported languages: https://github.com/run-llama/llama_cloud_services/blob/main/llama_parse/base.py.\n",
|
||||
"\n",
|
||||
"This notebook shows a demo of this in action. \n",
|
||||
"\n",
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/parse/excel/o1_excel_rag.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
|
||||
"<a href=\"https://colab.research.google.com/github/run-llama/llama_cloud_services/blob/main/examples/excel/o1_excel_rag.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -740,7 +740,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"In this example, these pages aren't going to be that different when parsed, but we can verify which pages triggered auto-made by looking at the [JSON output](https://github.com/run-llama/llama_cloud_services/blob/main/examples/parse/demo_json_tour.ipynb) of LlamaParse:"
|
||||
"In this example, these pages aren't going to be that different when parsed, but we can verify which pages triggered auto-made by looking at the [JSON output](https://github.com/run-llama/llama_cloud_services/blob/main/examples/demo_json_tour.ipynb) of LlamaParse:"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -5,16 +5,9 @@
|
||||
"private": true,
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"scripts": {
|
||||
"pre-commit-version": "pnpm changeset",
|
||||
"version": "./scripts/changeset-version.py version",
|
||||
"publish": "./scripts/changeset-version.py publish --tag"
|
||||
},
|
||||
"devDependencies": {
|
||||
"prettier": "^3.6.2",
|
||||
"lint-staged": "^15.4.2",
|
||||
"@changesets/cli": "^2.29.5",
|
||||
"changesets": "^1.0.2"
|
||||
"lint-staged": "^15.4.2"
|
||||
},
|
||||
"lint-staged": {
|
||||
"ts/llama_cloud_services/src/**/*.{ts,tsx,js,jsx}": [
|
||||
|
||||
@@ -147,7 +147,7 @@ documents = SimpleDirectoryReader(
|
||||
).load_data()
|
||||
```
|
||||
|
||||
Full documentation for `SimpleDirectoryReader` can be found on the [LlamaIndex Documentation](https://developers.llamaindex.ai/python/framework/module_guides/loading/simpledirectoryreader/).
|
||||
Full documentation for `SimpleDirectoryReader` can be found on the [LlamaIndex Documentation](https://docs.llamaindex.ai/en/stable/module_guides/loading/simpledirectoryreader.html).
|
||||
|
||||
## Examples
|
||||
|
||||
|
||||
@@ -1,4 +1,2 @@
|
||||
packages:
|
||||
- "ts/*"
|
||||
- "py"
|
||||
- "py/*"
|
||||
- "ts/**"
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
# llama-cloud-services-py
|
||||
|
||||
## 0.6.78
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 9f1ef4e: Fix extract
|
||||
|
||||
## 0.6.77
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 407292b: Now return partial results on job failure
|
||||
|
||||
## 0.6.76
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 4f24f53: Add aggressive_table_extraction flag in python sdk
|
||||
|
||||
## 0.6.75
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- f81532e: Safest types possible for parse
|
||||
|
||||
## 0.6.74
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 1bf5223: Fix default bbox values
|
||||
- 24166dc: Now only escape single dollar signs - preserve double for latex equations
|
||||
|
||||
## 0.6.73
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- e6a7939: Loosen packaging dep requirement
|
||||
|
||||
## 0.6.72
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- ad6734b: Fixup and test versioning
|
||||
|
||||
## 0.6.71
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 51011b9: Escape dollar signs in jupyter notebooks
|
||||
|
||||
## 0.6.70
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- d028397: Update llama-cloud api version, and integrate with agent data deletion
|
||||
@@ -1,6 +1,5 @@
|
||||
from llama_cloud_services.parse import LlamaParse
|
||||
from llama_cloud_services.extract import LlamaExtract, ExtractionAgent
|
||||
from llama_cloud_services.utils import SourceText, FileInput
|
||||
from llama_cloud_services.extract import LlamaExtract, ExtractionAgent, SourceText
|
||||
from llama_cloud_services.constants import EU_BASE_URL
|
||||
from llama_cloud_services.index import (
|
||||
LlamaCloudCompositeRetriever,
|
||||
@@ -13,7 +12,6 @@ __all__ = [
|
||||
"LlamaExtract",
|
||||
"ExtractionAgent",
|
||||
"SourceText",
|
||||
"FileInput",
|
||||
"EU_BASE_URL",
|
||||
"LlamaCloudIndex",
|
||||
"LlamaCloudRetriever",
|
||||
|
||||
@@ -1,11 +1,6 @@
|
||||
import os
|
||||
from typing import Any, Dict, Generic, List, Optional, Type
|
||||
|
||||
from llama_cloud import (
|
||||
AgentData,
|
||||
PaginatedResponseAgentData,
|
||||
PaginatedResponseAggregateGroup,
|
||||
)
|
||||
from llama_cloud.client import AsyncLlamaCloud
|
||||
from tenacity import (
|
||||
WrappedFn,
|
||||
@@ -91,7 +86,7 @@ class AsyncAgentDataClient(Generic[AgentDataT]):
|
||||
client=llama_client,
|
||||
type=ExtractedPerson,
|
||||
collection="extracted_people",
|
||||
deployment_name="person-extraction-agent"
|
||||
agent_url_id="person-extraction-agent"
|
||||
)
|
||||
|
||||
# Create data
|
||||
@@ -114,12 +109,10 @@ class AsyncAgentDataClient(Generic[AgentDataT]):
|
||||
self,
|
||||
type: Type[AgentDataT],
|
||||
collection: str = "default",
|
||||
deployment_name: Optional[str] = None,
|
||||
agent_url_id: Optional[str] = None,
|
||||
client: Optional[AsyncLlamaCloud] = None,
|
||||
token: Optional[str] = None,
|
||||
base_url: Optional[str] = None,
|
||||
# deprecated, use deployment_name instead
|
||||
agent_url_id: Optional[str] = None,
|
||||
):
|
||||
"""
|
||||
Initialize the AsyncAgentDataClient.
|
||||
@@ -130,11 +123,11 @@ class AsyncAgentDataClient(Generic[AgentDataT]):
|
||||
collection: Named collection within the agent for organizing data.
|
||||
Defaults to "default". Collections allow logical separation of
|
||||
different data types or workflows within the same agent.
|
||||
deployment_name: Unique identifier for the agent deployment. This normally
|
||||
appears in the URL of an agent within the Llama Cloud platform. If not
|
||||
provided, will attempt to use the LLAMA_DEPLOY_DEPLOYMENT_NAME
|
||||
environment variable. Data can only be added to an already existing
|
||||
agent in the platform.
|
||||
agent_url_id: Unique identifier for the agent. This normally appears in the
|
||||
url of an agent within the llama cloud platform. If not provided,
|
||||
will attempt to use the LLAMA_DEPLOY_DEPLOYMENT_NAME environment
|
||||
variable. Data can only be added to an already existing agent in the
|
||||
platform.
|
||||
client: AsyncLlamaCloud client instance for API communication. If not provided, will
|
||||
construct one from the provided api token and base url
|
||||
token: Llama Cloud API token. Reads from LLAMA_CLOUD_API_KEY if not provided
|
||||
@@ -142,14 +135,15 @@ class AsyncAgentDataClient(Generic[AgentDataT]):
|
||||
defaults to https://api.cloud.llamaindex.ai
|
||||
|
||||
Raises:
|
||||
ValueError: If deployment_name is not provided and the
|
||||
ValueError: If agent_url_id is not provided and the
|
||||
LLAMA_DEPLOY_DEPLOYMENT_NAME environment variable is not set
|
||||
|
||||
Note:
|
||||
The client automatically applies retry logic to all API calls with
|
||||
exponential backoff for timeout, connection, and HTTP status errors.
|
||||
"""
|
||||
self.deployment_name = deployment_name or agent_url_id or get_default_agent_id()
|
||||
|
||||
self.agent_url_id = agent_url_id or get_default_agent_id()
|
||||
|
||||
self.collection = collection
|
||||
if not client:
|
||||
@@ -162,19 +156,15 @@ class AsyncAgentDataClient(Generic[AgentDataT]):
|
||||
|
||||
@agent_data_retry
|
||||
async def get_item(self, item_id: str) -> TypedAgentData[AgentDataT]:
|
||||
raw_data = await self.untyped_get_item(item_id)
|
||||
return TypedAgentData.from_raw(raw_data, self.type)
|
||||
|
||||
@agent_data_retry
|
||||
async def untyped_get_item(self, item_id: str) -> AgentData:
|
||||
return await self.client.beta.get_agent_data(
|
||||
raw_data = await self.client.beta.get_agent_data(
|
||||
item_id=item_id,
|
||||
)
|
||||
return TypedAgentData.from_raw(raw_data, validator=self.type)
|
||||
|
||||
@agent_data_retry
|
||||
async def create_item(self, data: AgentDataT) -> TypedAgentData[AgentDataT]:
|
||||
raw_data = await self.client.beta.create_agent_data(
|
||||
deployment_name=self.deployment_name,
|
||||
agent_slug=self.agent_url_id,
|
||||
collection=self.collection,
|
||||
data=data.model_dump(),
|
||||
)
|
||||
@@ -194,21 +184,6 @@ class AsyncAgentDataClient(Generic[AgentDataT]):
|
||||
async def delete_item(self, item_id: str) -> None:
|
||||
await self.client.beta.delete_agent_data(item_id=item_id)
|
||||
|
||||
@agent_data_retry
|
||||
async def delete(
|
||||
self, filter: Optional[Dict[str, Dict[ComparisonOperator, Any]]] = None
|
||||
) -> int:
|
||||
"""
|
||||
Delete agent data by query, similar to search.
|
||||
Returns the number of deleted items.
|
||||
"""
|
||||
response = await self.client.beta.delete_agent_data_by_query_api_v_1_beta_agent_data_delete_post(
|
||||
deployment_name=self.deployment_name,
|
||||
collection=self.collection,
|
||||
filter=filter,
|
||||
)
|
||||
return response.deleted_count
|
||||
|
||||
@agent_data_retry
|
||||
async def search(
|
||||
self,
|
||||
@@ -235,7 +210,9 @@ class AsyncAgentDataClient(Generic[AgentDataT]):
|
||||
offset: Number of items to skip from the beginning. Defaults to 0.
|
||||
include_total: Whether to include the total count in the response. Defaults to False to improve performance. It's recommended to only request on the first page.
|
||||
"""
|
||||
raw = await self.untyped_search(
|
||||
raw = await self.client.beta.search_agent_data_api_v_1_beta_agent_data_search_post(
|
||||
agent_slug=self.agent_url_id,
|
||||
collection=self.collection,
|
||||
filter=filter,
|
||||
order_by=order_by,
|
||||
offset=offset,
|
||||
@@ -250,25 +227,6 @@ class AsyncAgentDataClient(Generic[AgentDataT]):
|
||||
total=raw.total_size,
|
||||
)
|
||||
|
||||
@agent_data_retry
|
||||
async def untyped_search(
|
||||
self,
|
||||
filter: Optional[Dict[str, Dict[ComparisonOperator, Any]]] = None,
|
||||
order_by: Optional[str] = None,
|
||||
offset: Optional[int] = None,
|
||||
page_size: Optional[int] = None,
|
||||
include_total: bool = False,
|
||||
) -> PaginatedResponseAgentData:
|
||||
return await self.client.beta.search_agent_data_api_v_1_beta_agent_data_search_post(
|
||||
deployment_name=self.deployment_name,
|
||||
collection=self.collection,
|
||||
filter=filter,
|
||||
order_by=order_by,
|
||||
offset=offset,
|
||||
page_size=page_size,
|
||||
include_total=include_total,
|
||||
)
|
||||
|
||||
@agent_data_retry
|
||||
async def aggregate(
|
||||
self,
|
||||
@@ -295,38 +253,8 @@ class AsyncAgentDataClient(Generic[AgentDataT]):
|
||||
offset: Number of groups to skip from the beginning. Defaults to 0.
|
||||
page_size: Maximum number of groups to return per page.
|
||||
"""
|
||||
raw = await self.untyped_aggregate(
|
||||
filter=filter,
|
||||
group_by=group_by,
|
||||
count=count,
|
||||
first=first,
|
||||
order_by=order_by,
|
||||
offset=offset,
|
||||
page_size=page_size,
|
||||
)
|
||||
|
||||
return TypedAggregateGroupItems(
|
||||
items=[
|
||||
TypedAggregateGroup.from_raw(grp, validator=self.type)
|
||||
for grp in raw.items
|
||||
],
|
||||
has_more=raw.next_page_token is not None,
|
||||
total=raw.total_size,
|
||||
)
|
||||
|
||||
@agent_data_retry
|
||||
async def untyped_aggregate(
|
||||
self,
|
||||
filter: Optional[Dict[str, Dict[ComparisonOperator, Any]]] = None,
|
||||
group_by: Optional[List[str]] = None,
|
||||
count: Optional[bool] = None,
|
||||
first: Optional[bool] = None,
|
||||
order_by: Optional[str] = None,
|
||||
offset: Optional[int] = None,
|
||||
page_size: Optional[int] = None,
|
||||
) -> PaginatedResponseAggregateGroup:
|
||||
return await self.client.beta.aggregate_agent_data_api_v_1_beta_agent_data_aggregate_post(
|
||||
deployment_name=self.deployment_name,
|
||||
raw = await self.client.beta.aggregate_agent_data_api_v_1_beta_agent_data_aggregate_post(
|
||||
agent_slug=self.agent_url_id,
|
||||
collection=self.collection,
|
||||
page_size=page_size,
|
||||
filter=filter,
|
||||
@@ -336,3 +264,11 @@ class AsyncAgentDataClient(Generic[AgentDataT]):
|
||||
first=first,
|
||||
offset=offset,
|
||||
)
|
||||
return TypedAggregateGroupItems(
|
||||
items=[
|
||||
TypedAggregateGroup.from_raw(item, validator=self.type)
|
||||
for item in raw.items
|
||||
],
|
||||
has_more=raw.next_page_token is not None,
|
||||
total=raw.total_size,
|
||||
)
|
||||
|
||||
@@ -10,7 +10,7 @@ CRUD operations, search capabilities, filtering, and aggregation functionality
|
||||
for managing agent-generated data at scale.
|
||||
|
||||
Key Concepts:
|
||||
- Deployment Name: Unique identifier for an agent deployment
|
||||
- Agent Slug: Unique identifier for an agent instance
|
||||
- Collection: Named grouping of data within an agent (defaults to "default"). Data within a collection should be of the same type.
|
||||
- Agent Data: Individual structured data records with metadata and timestamps
|
||||
|
||||
@@ -26,7 +26,7 @@ Example Usage:
|
||||
client=async_llama_cloud,
|
||||
type=Person,
|
||||
collection="people",
|
||||
deployment_name="my-extraction-agent-xyz"
|
||||
agent_url_id="my-extraction-agent-xyz"
|
||||
)
|
||||
|
||||
# Create typed data
|
||||
@@ -56,6 +56,7 @@ from typing import (
|
||||
|
||||
# Type variable for user-defined data models
|
||||
AgentDataT = TypeVar("AgentDataT", bound=BaseModel)
|
||||
|
||||
# Type variable for extracted data (can be dict or Pydantic model)
|
||||
ExtractedT = TypeVar("ExtractedT", bound=Union[BaseModel, dict])
|
||||
|
||||
@@ -77,7 +78,7 @@ class TypedAgentData(BaseModel, Generic[AgentDataT]):
|
||||
|
||||
Attributes:
|
||||
id: Unique identifier for this data record
|
||||
deployment_name: Identifier of the agent deployment that created this data
|
||||
agent_url_id: Identifier of the agent that created this data
|
||||
collection: Named collection within the agent (used for organization)
|
||||
data: The actual structured data payload (typed as AgentDataT)
|
||||
created_at: Timestamp when the record was first created
|
||||
@@ -93,8 +94,8 @@ class TypedAgentData(BaseModel, Generic[AgentDataT]):
|
||||
"""
|
||||
|
||||
id: Optional[str] = Field(description="Unique identifier for this data record")
|
||||
deployment_name: str = Field(
|
||||
description="Identifier of the agent deployment that created this data"
|
||||
agent_url_id: str = Field(
|
||||
description="Identifier of the agent that created this data"
|
||||
)
|
||||
collection: Optional[str] = Field(
|
||||
description="Named collection within the agent for data organization"
|
||||
@@ -115,15 +116,15 @@ class TypedAgentData(BaseModel, Generic[AgentDataT]):
|
||||
Args:
|
||||
raw_data: Raw agent data from the API
|
||||
validator: Pydantic model class to validate the data field
|
||||
|
||||
Returns:
|
||||
TypedAgentData instance with validated data
|
||||
"""
|
||||
|
||||
data: AgentDataT = validator.model_validate(raw_data.data)
|
||||
|
||||
return cls(
|
||||
id=raw_data.id,
|
||||
deployment_name=raw_data.deployment_name,
|
||||
agent_url_id=raw_data.agent_slug,
|
||||
collection=raw_data.collection,
|
||||
data=data,
|
||||
created_at=raw_data.created_at,
|
||||
@@ -221,16 +222,12 @@ def parse_extracted_field_metadata(
|
||||
return {
|
||||
k: _parse_extracted_field_metadata_recursive(v)
|
||||
for k, v in field_metadata.items()
|
||||
if not _is_reasoning_field(k, v) and k not in _ADDITIONAL_ROOT_METADATA_FIELDS
|
||||
if k not in _METADATA_FIELDS_SIBLING_TO_LEAF
|
||||
and k not in _ADDITIONAL_ROOT_METADATA_FIELDS
|
||||
}
|
||||
|
||||
|
||||
def _is_reasoning_field(field_name: str, field_value: Any) -> bool:
|
||||
# There can either be a user specified reasoning field (from the schema), or a reasoning metadata field for the
|
||||
# dict of values
|
||||
return field_name == "reasoning" and isinstance(field_value, str)
|
||||
|
||||
|
||||
_METADATA_FIELDS_SIBLING_TO_LEAF = {"reasoning"}
|
||||
_ADDITIONAL_ROOT_METADATA_FIELDS = {"error"}
|
||||
|
||||
|
||||
@@ -260,12 +257,14 @@ def _parse_extracted_field_metadata_recursive(
|
||||
except ValidationError:
|
||||
pass
|
||||
additional_fields = {
|
||||
k: v for k, v in field_value.items() if _is_reasoning_field(k, v)
|
||||
k: v
|
||||
for k, v in field_value.items()
|
||||
if k in _METADATA_FIELDS_SIBLING_TO_LEAF
|
||||
}
|
||||
return {
|
||||
k: _parse_extracted_field_metadata_recursive(v, additional_fields)
|
||||
for k, v in field_value.items()
|
||||
if not _is_reasoning_field(k, v)
|
||||
if k not in _METADATA_FIELDS_SIBLING_TO_LEAF
|
||||
}
|
||||
elif isinstance(field_value, list):
|
||||
return [_parse_extracted_field_metadata_recursive(item) for item in field_value]
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
from llama_cloud_services.beta.classifier.client import ClassifyClient
|
||||
from llama_cloud_services.beta.classifier.types import ClassifyJobResultsWithFiles
|
||||
from llama_cloud_services.utils import SourceText, FileInput
|
||||
|
||||
__all__ = [
|
||||
"ClassifyClient",
|
||||
"ClassifyJobResultsWithFiles",
|
||||
"SourceText",
|
||||
"FileInput",
|
||||
]
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import asyncio
|
||||
import time
|
||||
import warnings
|
||||
from typing import Optional, List, Union
|
||||
from typing import Optional
|
||||
from pydantic import BaseModel
|
||||
from llama_cloud.client import AsyncLlamaCloud
|
||||
from llama_cloud.types import (
|
||||
@@ -15,11 +14,7 @@ from llama_cloud.types import (
|
||||
from llama_cloud.resources.classifier.client import OMIT
|
||||
from llama_cloud_services.files.client import FileClient
|
||||
from llama_cloud_services.constants import POLLING_TIMEOUT_SECONDS
|
||||
from llama_cloud_services.utils import (
|
||||
is_terminal_status,
|
||||
augment_async_errors,
|
||||
FileInput,
|
||||
)
|
||||
from llama_cloud_services.utils import is_terminal_status, augment_async_errors
|
||||
from llama_index.core.async_utils import DEFAULT_NUM_WORKERS, run_jobs
|
||||
from llama_cloud_services.beta.classifier.types import (
|
||||
ClassifyJobResultsWithFiles,
|
||||
@@ -171,98 +166,6 @@ class ClassifyClient:
|
||||
)
|
||||
)
|
||||
|
||||
async def aclassify(
|
||||
self,
|
||||
rules: list[ClassifierRule],
|
||||
files: Union[FileInput, List[FileInput]],
|
||||
parsing_configuration: Optional[ClassifyParsingConfiguration] = None,
|
||||
raise_on_error: bool = True,
|
||||
workers: int = DEFAULT_NUM_WORKERS,
|
||||
show_progress: bool = False,
|
||||
) -> ClassifyJobResultsWithFiles:
|
||||
"""
|
||||
Classify one or more files from various input types.
|
||||
|
||||
Args:
|
||||
rules: The rules to use for classification.
|
||||
files: The file(s) to classify. Can be a single file or list of files. Each can be:
|
||||
- str/Path: File path
|
||||
- SourceText: Text content or file with explicit filename
|
||||
- File: Already uploaded file
|
||||
- BufferedIOBase: File-like object
|
||||
parsing_configuration: The parsing configuration to use for classification.
|
||||
raise_on_error: Whether to raise an error if the classification job fails.
|
||||
workers: Number of parallel workers for uploading files.
|
||||
show_progress: Whether to show progress bars.
|
||||
|
||||
Returns:
|
||||
The results of the classification job with file metadata.
|
||||
"""
|
||||
# Normalize to list
|
||||
if not isinstance(files, list):
|
||||
files = [files]
|
||||
|
||||
# Upload all files
|
||||
coroutines = [
|
||||
self.file_client.upload_content(file_input) for file_input in files
|
||||
]
|
||||
uploaded_files: List[File] = await run_jobs(
|
||||
coroutines,
|
||||
show_progress=show_progress,
|
||||
workers=workers,
|
||||
desc="Uploading files for classification",
|
||||
)
|
||||
|
||||
# Classify
|
||||
results = await self.aclassify_file_ids(
|
||||
rules,
|
||||
[file.id for file in uploaded_files],
|
||||
parsing_configuration,
|
||||
raise_on_error,
|
||||
)
|
||||
return ClassifyJobResultsWithFiles.from_classify_job_results(
|
||||
results, uploaded_files
|
||||
)
|
||||
|
||||
def classify(
|
||||
self,
|
||||
rules: list[ClassifierRule],
|
||||
files: Union[FileInput, List[FileInput]],
|
||||
parsing_configuration: Optional[ClassifyParsingConfiguration] = None,
|
||||
raise_on_error: bool = True,
|
||||
workers: int = DEFAULT_NUM_WORKERS,
|
||||
show_progress: bool = False,
|
||||
) -> ClassifyJobResultsWithFiles:
|
||||
"""
|
||||
Classify one or more files from various input types (synchronous version).
|
||||
|
||||
Args:
|
||||
rules: The rules to use for classification.
|
||||
files: The file(s) to classify. Can be a single file or list of files. Each can be:
|
||||
- str/Path: File path
|
||||
- SourceText: Text content or file with explicit filename
|
||||
- File: Already uploaded file
|
||||
- BufferedIOBase: File-like object
|
||||
parsing_configuration: The parsing configuration to use for classification.
|
||||
raise_on_error: Whether to raise an error if the classification job fails.
|
||||
workers: Number of parallel workers for uploading files.
|
||||
show_progress: Whether to show progress bars.
|
||||
|
||||
Returns:
|
||||
The results of the classification job with file metadata.
|
||||
"""
|
||||
with augment_async_errors():
|
||||
return asyncio.run(
|
||||
self.aclassify(
|
||||
rules,
|
||||
files,
|
||||
parsing_configuration,
|
||||
raise_on_error,
|
||||
workers,
|
||||
show_progress,
|
||||
)
|
||||
)
|
||||
|
||||
async def aclassify_file_path(
|
||||
self,
|
||||
rules: list[ClassifierRule],
|
||||
@@ -270,17 +173,11 @@ class ClassifyClient:
|
||||
parsing_configuration: Optional[ClassifyParsingConfiguration] = None,
|
||||
raise_on_error: bool = True,
|
||||
) -> ClassifyJobResultsWithFiles:
|
||||
"""
|
||||
Deprecated: Use aclassify() instead.
|
||||
"""
|
||||
warnings.warn(
|
||||
"aclassify_file_path is deprecated, use aclassify() instead",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return await self.aclassify(
|
||||
rules, file_input_path, parsing_configuration, raise_on_error
|
||||
file = await self.file_client.upload_file(file_input_path)
|
||||
results = await self.aclassify_file_ids(
|
||||
rules, [file.id], parsing_configuration, raise_on_error
|
||||
)
|
||||
return ClassifyJobResultsWithFiles.from_classify_job_results(results, [file])
|
||||
|
||||
def classify_file_path(
|
||||
self,
|
||||
@@ -289,17 +186,12 @@ class ClassifyClient:
|
||||
parsing_configuration: Optional[ClassifyParsingConfiguration] = None,
|
||||
raise_on_error: bool = True,
|
||||
) -> ClassifyJobResultsWithFiles:
|
||||
"""
|
||||
Deprecated: Use classify() instead.
|
||||
"""
|
||||
warnings.warn(
|
||||
"classify_file_path is deprecated, use classify() instead",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return self.classify(
|
||||
rules, file_input_path, parsing_configuration, raise_on_error
|
||||
)
|
||||
with augment_async_errors():
|
||||
return asyncio.run(
|
||||
self.aclassify_file_path(
|
||||
rules, file_input_path, parsing_configuration, raise_on_error
|
||||
)
|
||||
)
|
||||
|
||||
async def aclassify_file_paths(
|
||||
self,
|
||||
@@ -310,22 +202,17 @@ class ClassifyClient:
|
||||
workers: int = DEFAULT_NUM_WORKERS,
|
||||
show_progress: bool = False,
|
||||
) -> ClassifyJobResultsWithFiles:
|
||||
"""
|
||||
Deprecated: Use aclassify() instead.
|
||||
"""
|
||||
warnings.warn(
|
||||
"aclassify_file_paths is deprecated, use aclassify() instead",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
coroutines = [self.file_client.upload_file(path) for path in file_input_paths]
|
||||
files: list[File] = await run_jobs(
|
||||
coroutines,
|
||||
show_progress=show_progress,
|
||||
workers=workers,
|
||||
desc="Uploading files for classification",
|
||||
)
|
||||
return await self.aclassify(
|
||||
rules,
|
||||
file_input_paths,
|
||||
parsing_configuration,
|
||||
raise_on_error,
|
||||
workers,
|
||||
show_progress,
|
||||
results = await self.aclassify_file_ids(
|
||||
rules, [file.id for file in files], parsing_configuration, raise_on_error
|
||||
)
|
||||
return ClassifyJobResultsWithFiles.from_classify_job_results(results, files)
|
||||
|
||||
def classify_file_paths(
|
||||
self,
|
||||
@@ -334,17 +221,12 @@ class ClassifyClient:
|
||||
parsing_configuration: Optional[ClassifyParsingConfiguration] = None,
|
||||
raise_on_error: bool = True,
|
||||
) -> ClassifyJobResultsWithFiles:
|
||||
"""
|
||||
Deprecated: Use classify() instead.
|
||||
"""
|
||||
warnings.warn(
|
||||
"classify_file_paths is deprecated, use classify() instead",
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return self.classify(
|
||||
rules, file_input_paths, parsing_configuration, raise_on_error
|
||||
)
|
||||
with augment_async_errors():
|
||||
return asyncio.run(
|
||||
self.aclassify_file_paths(
|
||||
rules, file_input_paths, parsing_configuration, raise_on_error
|
||||
)
|
||||
)
|
||||
|
||||
async def wait_for_job_completion(self, job_id: str) -> ClassifyJob:
|
||||
"""
|
||||
|
||||
@@ -2,16 +2,15 @@ from llama_cloud_services.extract.extract import (
|
||||
LlamaExtract,
|
||||
ExtractConfig,
|
||||
ExtractionAgent,
|
||||
SourceText,
|
||||
ExtractTarget,
|
||||
ExtractMode,
|
||||
)
|
||||
from llama_cloud_services.utils import SourceText, FileInput
|
||||
|
||||
__all__ = [
|
||||
"LlamaExtract",
|
||||
"ExtractionAgent",
|
||||
"SourceText",
|
||||
"FileInput",
|
||||
"ExtractConfig",
|
||||
"ExtractTarget",
|
||||
"ExtractMode",
|
||||
|
||||
@@ -2,9 +2,10 @@ import asyncio
|
||||
import base64
|
||||
import os
|
||||
import time
|
||||
from io import BufferedIOBase, TextIOWrapper
|
||||
from io import BufferedIOBase, BufferedReader, BytesIO, TextIOWrapper
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Type, Union, Coroutine, Any, TypeVar
|
||||
import secrets
|
||||
import warnings
|
||||
import httpx
|
||||
from pydantic import BaseModel
|
||||
@@ -18,12 +19,14 @@ from llama_cloud import (
|
||||
ExtractAgent as CloudExtractAgent,
|
||||
ExtractConfig,
|
||||
ExtractJob,
|
||||
ExtractJobCreate,
|
||||
ExtractRun,
|
||||
File,
|
||||
FileData,
|
||||
ExtractMode,
|
||||
StatusEnum,
|
||||
ExtractTarget,
|
||||
LlamaExtractSettings,
|
||||
PaginatedExtractRunsResponse,
|
||||
)
|
||||
from llama_cloud.client import AsyncLlamaCloud
|
||||
@@ -32,8 +35,7 @@ from llama_cloud_services.extract.utils import (
|
||||
JSONObjectType,
|
||||
ExperimentalWarning,
|
||||
)
|
||||
from llama_cloud_services.utils import augment_async_errors, SourceText, FileInput
|
||||
from llama_cloud_services.files.client import FileClient
|
||||
from llama_cloud_services.utils import augment_async_errors
|
||||
from llama_index.core.schema import BaseComponent
|
||||
from llama_index.core.async_utils import run_jobs
|
||||
from llama_index.core.bridge.pydantic import Field, PrivateAttr
|
||||
@@ -188,6 +190,46 @@ async def _wait_for_job_result(
|
||||
)
|
||||
|
||||
|
||||
class SourceText:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
file: Union[bytes, BufferedIOBase, TextIOWrapper, str, Path, None] = None,
|
||||
text_content: Optional[str] = None,
|
||||
filename: Optional[str] = None,
|
||||
):
|
||||
self.file = file
|
||||
self.filename = filename
|
||||
self.text_content = text_content
|
||||
self._validate()
|
||||
|
||||
def _validate(self) -> None:
|
||||
"""Ensure filename is provided when needed."""
|
||||
if not ((self.file is None) ^ (self.text_content is None)):
|
||||
raise ValueError("Either file or text_content must be provided.")
|
||||
if self.text_content is not None:
|
||||
if not self.filename:
|
||||
random_hex = secrets.token_hex(4)
|
||||
self.filename = f"text_input_{random_hex}.txt"
|
||||
return
|
||||
|
||||
if isinstance(self.file, (bytes, BufferedIOBase, TextIOWrapper)):
|
||||
if not self.filename and hasattr(self.file, "name"):
|
||||
self.filename = os.path.basename(str(self.file.name))
|
||||
elif not hasattr(self.file, "name") and self.filename is None:
|
||||
raise ValueError(
|
||||
"filename must be provided when file is bytes or a file-like object without a name"
|
||||
)
|
||||
elif isinstance(self.file, (str, Path)):
|
||||
if not self.filename:
|
||||
self.filename = os.path.basename(str(self.file))
|
||||
else:
|
||||
raise ValueError(f"Unsupported file type: {type(self.file)}")
|
||||
|
||||
|
||||
FileInput = Union[str, Path, BufferedIOBase, SourceText, File]
|
||||
|
||||
|
||||
def run_in_thread(
|
||||
coro: Coroutine[Any, Any, T],
|
||||
thread_pool: ThreadPoolExecutor,
|
||||
@@ -280,7 +322,6 @@ class ExtractionAgent:
|
||||
self._thread_pool = ThreadPoolExecutor(
|
||||
max_workers=min(10, (os.cpu_count() or 1) + 4)
|
||||
)
|
||||
self._file_client = FileClient(client, project_id, organization_id)
|
||||
|
||||
@property
|
||||
def id(self) -> str:
|
||||
@@ -330,11 +371,65 @@ class ExtractionAgent:
|
||||
ValueError: If filename is not provided for bytes input or for file-like objects
|
||||
without a name attribute.
|
||||
"""
|
||||
return await self._file_client.upload_content(file_input)
|
||||
file_contents: Optional[Union[BufferedIOBase, BytesIO]] = None
|
||||
try:
|
||||
if file_input.text_content is not None:
|
||||
# Handle direct text content
|
||||
file_contents = BytesIO(file_input.text_content.encode("utf-8"))
|
||||
elif isinstance(file_input.file, TextIOWrapper):
|
||||
# Handle text-based IO objects
|
||||
file_contents = BytesIO(file_input.file.read().encode("utf-8"))
|
||||
elif isinstance(file_input.file, (str, Path)):
|
||||
# Handle file paths
|
||||
file_contents = open(file_input.file, "rb")
|
||||
elif isinstance(file_input.file, bytes):
|
||||
# Handle bytes
|
||||
file_contents = BytesIO(file_input.file)
|
||||
elif isinstance(file_input.file, BufferedIOBase):
|
||||
# Handle binary IO objects
|
||||
file_contents = file_input.file
|
||||
else:
|
||||
raise ValueError(f"Unsupported file type: {type(file_input.file)}")
|
||||
|
||||
# Add name attribute to file object if needed
|
||||
if not hasattr(file_contents, "name"):
|
||||
file_contents.name = file_input.filename # type: ignore
|
||||
|
||||
return await self._client.files.upload_file(
|
||||
project_id=self._project_id, upload_file=file_contents
|
||||
)
|
||||
finally:
|
||||
if file_contents is not None and isinstance(
|
||||
file_contents, (BufferedReader, BytesIO)
|
||||
):
|
||||
file_contents.close()
|
||||
|
||||
async def _upload_file(self, file_input: FileInput) -> File:
|
||||
"""Upload a file from various input types using FileClient."""
|
||||
return await self._file_client.upload_content(file_input)
|
||||
source_text = None
|
||||
if isinstance(file_input, File):
|
||||
return file_input
|
||||
if isinstance(file_input, SourceText):
|
||||
source_text = file_input
|
||||
elif isinstance(file_input, (str, Path)):
|
||||
path = Path(file_input)
|
||||
source_text = SourceText(file=path, filename=path.name)
|
||||
else:
|
||||
# Try to get filename from the file object if not provided
|
||||
filename = None
|
||||
if hasattr(file_input, "name"):
|
||||
filename = os.path.basename(str(file_input.name))
|
||||
if filename is None:
|
||||
raise ValueError(
|
||||
"Use SourceText to provide filename when uploading bytes or file-like objects."
|
||||
)
|
||||
|
||||
warnings.warn(
|
||||
"Use SourceText instead of bytes or file-like objects",
|
||||
DeprecationWarning,
|
||||
)
|
||||
source_text = SourceText(file=file_input, filename=filename)
|
||||
|
||||
return await self.upload_file(source_text)
|
||||
|
||||
async def _wait_for_job_result(self, job_id: str) -> Optional[ExtractRun]:
|
||||
"""Wait for and return the results of an extraction job."""
|
||||
@@ -368,6 +463,56 @@ class ExtractionAgent:
|
||||
)
|
||||
)
|
||||
|
||||
async def _run_extraction_test(
|
||||
self,
|
||||
files: Union[FileInput, List[FileInput]],
|
||||
extract_settings: LlamaExtractSettings,
|
||||
) -> Union[ExtractJob, List[ExtractJob]]:
|
||||
if not isinstance(files, list):
|
||||
files = [files]
|
||||
single_file = True
|
||||
else:
|
||||
single_file = False
|
||||
|
||||
upload_tasks = [self._upload_file(file) for file in files]
|
||||
with augment_async_errors():
|
||||
uploaded_files = await run_jobs(
|
||||
upload_tasks,
|
||||
workers=self.num_workers,
|
||||
desc="Uploading files",
|
||||
show_progress=self.show_progress,
|
||||
)
|
||||
|
||||
async def run_job(file: File) -> ExtractRun:
|
||||
job_queued = await self._client.llama_extract.run_job_test_user(
|
||||
job_create=ExtractJobCreate(
|
||||
extraction_agent_id=self.id,
|
||||
file_id=file.id,
|
||||
data_schema_override=self.data_schema,
|
||||
config_override=self.config,
|
||||
),
|
||||
extract_settings=extract_settings,
|
||||
)
|
||||
return await self._wait_for_job_result(job_queued.id)
|
||||
|
||||
job_tasks = [run_job(file) for file in uploaded_files]
|
||||
with augment_async_errors():
|
||||
extract_results = await run_jobs(
|
||||
job_tasks,
|
||||
workers=self.num_workers,
|
||||
desc="Running extraction jobs",
|
||||
show_progress=self.show_progress,
|
||||
)
|
||||
|
||||
if self._verbose:
|
||||
for file, job in zip(files, extract_results):
|
||||
file_repr = (
|
||||
str(file) if isinstance(file, (str, Path)) else "<bytes/buffer>"
|
||||
)
|
||||
print(f"Running extraction for file {file_repr} under job_id {job.id}")
|
||||
|
||||
return extract_results[0] if single_file else extract_results
|
||||
|
||||
async def queue_extraction(
|
||||
self,
|
||||
files: Union[FileInput, List[FileInput]],
|
||||
@@ -399,10 +544,12 @@ class ExtractionAgent:
|
||||
|
||||
job_tasks = [
|
||||
self._client.llama_extract.run_job(
|
||||
extraction_agent_id=self.id,
|
||||
file_id=file.id,
|
||||
data_schema_override=self.data_schema,
|
||||
config_override=self.config,
|
||||
request=ExtractJobCreate(
|
||||
extraction_agent_id=self.id,
|
||||
file_id=file.id,
|
||||
data_schema_override=self.data_schema,
|
||||
config_override=self.config,
|
||||
),
|
||||
)
|
||||
for file in uploaded_files
|
||||
]
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
from io import BytesIO
|
||||
from typing import BinaryIO
|
||||
import os
|
||||
from pathlib import Path
|
||||
from llama_cloud.client import AsyncLlamaCloud
|
||||
from llama_cloud.types import File, FileCreate
|
||||
from typing import Optional
|
||||
from llama_cloud_services.utils import SourceText, FileInput
|
||||
|
||||
|
||||
class FileClient:
|
||||
@@ -97,83 +95,3 @@ class FileClient:
|
||||
project_id=self.project_id,
|
||||
organization_id=self.organization_id,
|
||||
)
|
||||
|
||||
async def upload_content(
|
||||
self, file_input: FileInput, external_file_id: Optional[str] = None
|
||||
) -> File:
|
||||
"""
|
||||
Upload content from various input types or fetch an already-uploaded file.
|
||||
|
||||
Args:
|
||||
file_input: The content to upload. Can be:
|
||||
- File: Already uploaded file (returned as-is)
|
||||
- str/Path: Path to a file on disk
|
||||
- SourceText: Text content, file, or file_id with explicit filename
|
||||
- BufferedIOBase: File-like binary object
|
||||
external_file_id: Optional external identifier for the file
|
||||
|
||||
Returns:
|
||||
File: The uploaded (or fetched) file object
|
||||
|
||||
Raises:
|
||||
ValueError: If the input type is not supported or required info is missing
|
||||
"""
|
||||
# If already a File object, return it
|
||||
if isinstance(file_input, File):
|
||||
return file_input
|
||||
|
||||
# Handle SourceText
|
||||
if isinstance(file_input, SourceText):
|
||||
# If file_id is provided, fetch the file object
|
||||
if file_input.file_id is not None:
|
||||
return await self.get_file(file_input.file_id)
|
||||
elif file_input.text_content is not None:
|
||||
# Handle direct text content
|
||||
text_bytes = file_input.text_content.encode("utf-8")
|
||||
return await self.upload_bytes(
|
||||
text_bytes, external_file_id or file_input.filename or "file"
|
||||
)
|
||||
elif isinstance(file_input.file, (str, Path)):
|
||||
# Handle file paths using the existing upload_file method
|
||||
return await self.upload_file(
|
||||
str(file_input.file), external_file_id or file_input.filename
|
||||
)
|
||||
elif isinstance(file_input.file, bytes):
|
||||
# Handle bytes
|
||||
return await self.upload_bytes(
|
||||
file_input.file, external_file_id or file_input.filename or "file"
|
||||
)
|
||||
elif hasattr(file_input.file, "read"):
|
||||
# Handle any file-like object (TextIOWrapper, BytesIO, BufferedReader, BufferedIOBase, etc.)
|
||||
content = file_input.file.read() # type: ignore
|
||||
if isinstance(content, str):
|
||||
content = content.encode("utf-8")
|
||||
return await self.upload_bytes(
|
||||
content, external_file_id or file_input.filename or "file"
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported file type: {type(file_input.file)}")
|
||||
|
||||
# Handle string/Path directly
|
||||
elif isinstance(file_input, (str, Path)):
|
||||
return await self.upload_file(str(file_input), external_file_id)
|
||||
|
||||
# Handle raw file-like objects
|
||||
elif hasattr(file_input, "read"):
|
||||
if hasattr(file_input, "name"):
|
||||
filename = os.path.basename(str(file_input.name))
|
||||
else:
|
||||
filename = external_file_id or "file"
|
||||
|
||||
# Read content to determine size
|
||||
content = file_input.read()
|
||||
if isinstance(content, str):
|
||||
content = content.encode("utf-8")
|
||||
|
||||
return await self.upload_bytes(content, external_file_id or filename)
|
||||
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported file input type: {type(file_input)}. "
|
||||
f"Supported types: str, Path, SourceText, BufferedIOBase, or File."
|
||||
)
|
||||
|
||||
@@ -333,7 +333,7 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
if file_ids:
|
||||
self._wait_for_resources(
|
||||
file_ids,
|
||||
lambda fid: self._client.pipeline_files.get_pipeline_file_status(
|
||||
lambda fid: self._client.pipelines.get_pipeline_file_status(
|
||||
pipeline_id=self.pipeline.id, file_id=fid
|
||||
),
|
||||
resource_name="file",
|
||||
@@ -420,7 +420,7 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
if file_ids:
|
||||
await self._await_for_resources(
|
||||
file_ids,
|
||||
lambda fid: self._aclient.pipeline_files.get_pipeline_file_status(
|
||||
lambda fid: self._aclient.pipelines.get_pipeline_file_status(
|
||||
pipeline_id=self.pipeline.id, file_id=fid
|
||||
),
|
||||
resource_name="file",
|
||||
@@ -489,7 +489,6 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
name: str,
|
||||
project_name: str = DEFAULT_PROJECT_NAME,
|
||||
organization_id: Optional[str] = None,
|
||||
project_id: Optional[str] = None,
|
||||
api_key: Optional[str] = None,
|
||||
base_url: Optional[str] = None,
|
||||
app_url: Optional[str] = None,
|
||||
@@ -505,15 +504,15 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
app_url = app_url or os.environ.get("LLAMA_CLOUD_APP_URL", DEFAULT_APP_URL)
|
||||
client = get_client(api_key, base_url, app_url, timeout)
|
||||
|
||||
if project_id is None:
|
||||
# create project if it doesn't exist
|
||||
project = client.projects.upsert_project(
|
||||
organization_id=organization_id,
|
||||
request=ProjectCreate(name=project_name),
|
||||
)
|
||||
project_id = project.id
|
||||
if verbose:
|
||||
print(f"Created project {project_id} with name {project_name}")
|
||||
# create project if it doesn't exist
|
||||
project = client.projects.upsert_project(
|
||||
organization_id=organization_id, request=ProjectCreate(name=project_name)
|
||||
)
|
||||
if project.id is None:
|
||||
raise ValueError(f"Failed to create/get project {project_name}")
|
||||
|
||||
if verbose:
|
||||
print(f"Created project {project.id} with name {project.name}")
|
||||
|
||||
# create pipeline
|
||||
pipeline_create = PipelineCreate(
|
||||
@@ -524,7 +523,7 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
llama_parse_parameters=llama_parse_parameters or LlamaParseParameters(),
|
||||
)
|
||||
pipeline = client.pipelines.upsert_pipeline(
|
||||
project_id=project_id, request=pipeline_create
|
||||
project_id=project.id, request=pipeline_create
|
||||
)
|
||||
if pipeline.id is None:
|
||||
raise ValueError(f"Failed to create/get pipeline {name}")
|
||||
@@ -533,7 +532,8 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
|
||||
return cls(
|
||||
name,
|
||||
project_id=project_id,
|
||||
project_name=project.name,
|
||||
organization_id=project.organization_id,
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
app_url=app_url,
|
||||
@@ -606,7 +606,6 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
name: str,
|
||||
project_name: str = DEFAULT_PROJECT_NAME,
|
||||
organization_id: Optional[str] = None,
|
||||
project_id: Optional[str] = None,
|
||||
api_key: Optional[str] = None,
|
||||
base_url: Optional[str] = None,
|
||||
app_url: Optional[str] = None,
|
||||
@@ -632,7 +631,6 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
verbose=verbose,
|
||||
embedding_config=embedding_config,
|
||||
transform_config=transform_config,
|
||||
project_id=project_id,
|
||||
)
|
||||
|
||||
app_url = app_url or os.environ.get("LLAMA_CLOUD_APP_URL", DEFAULT_APP_URL)
|
||||
@@ -919,7 +917,7 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
|
||||
# Add file to pipeline
|
||||
pipeline_file_create = PipelineFileCreate(file_id=file.id)
|
||||
self._client.pipeline_files.add_files_to_pipeline_api(
|
||||
self._client.pipelines.add_files_to_pipeline_api(
|
||||
pipeline_id=self.pipeline.id, request=[pipeline_file_create]
|
||||
)
|
||||
|
||||
@@ -946,7 +944,7 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
|
||||
# Add file to pipeline
|
||||
pipeline_file_create = PipelineFileCreate(file_id=file.id)
|
||||
await self._aclient.pipeline_files.add_files_to_pipeline_api(
|
||||
await self._aclient.pipelines.add_files_to_pipeline_api(
|
||||
pipeline_id=self.pipeline.id, request=[pipeline_file_create]
|
||||
)
|
||||
|
||||
@@ -984,7 +982,7 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
|
||||
# Add file to pipeline
|
||||
pipeline_file_create = PipelineFileCreate(file_id=file.id)
|
||||
self._client.pipeline_files.add_files_to_pipeline_api(
|
||||
self._client.pipelines.add_files_to_pipeline_api(
|
||||
pipeline_id=self.pipeline.id, request=[pipeline_file_create]
|
||||
)
|
||||
|
||||
@@ -1021,7 +1019,7 @@ class LlamaCloudIndex(BaseManagedIndex):
|
||||
|
||||
# Add file to pipeline
|
||||
pipeline_file_create = PipelineFileCreate(file_id=file.id)
|
||||
await self._aclient.pipeline_files.add_files_to_pipeline_api(
|
||||
await self._aclient.pipelines.add_files_to_pipeline_api(
|
||||
pipeline_id=self.pipeline.id, request=[pipeline_file_create]
|
||||
)
|
||||
|
||||
|
||||
@@ -188,10 +188,6 @@ class LlamaParse(BasePydanticReader):
|
||||
default=False,
|
||||
description="If set to true, LlamaParse will try to detect long table and adapt the output.",
|
||||
)
|
||||
aggressive_table_extraction: Optional[bool] = Field(
|
||||
default=False,
|
||||
description="If set to true, LlamaParse will try to extract tables aggressively, may lead to false positives.",
|
||||
)
|
||||
annotate_links: Optional[bool] = Field(
|
||||
default=False,
|
||||
description="Annotate links found in the document to extract their URL.",
|
||||
@@ -424,10 +420,6 @@ class LlamaParse(BasePydanticReader):
|
||||
default=False,
|
||||
description="If set to true, the parser will extract sub-tables from the spreadsheet when possible (more than one table per sheet).",
|
||||
)
|
||||
spreadsheet_force_formula_computation: Optional[bool] = Field(
|
||||
default=False,
|
||||
description="If set to true, the parser will re-compute values for all spreadsheet cells containing formulas.",
|
||||
)
|
||||
specialized_chart_parsing_agentic: Optional[bool] = Field(
|
||||
default=False,
|
||||
description="If set to true, the parser will use a specialized agentic chart parsing model to extract data from charts. This model is able to understand the chart type and extract the data accordingly.",
|
||||
@@ -717,9 +709,6 @@ class LlamaParse(BasePydanticReader):
|
||||
if self.adaptive_long_table:
|
||||
data["adaptive_long_table"] = self.adaptive_long_table
|
||||
|
||||
if self.aggressive_table_extraction:
|
||||
data["aggressive_table_extraction"] = self.aggressive_table_extraction
|
||||
|
||||
if self.annotate_links:
|
||||
data["annotate_links"] = self.annotate_links
|
||||
|
||||
@@ -976,11 +965,6 @@ class LlamaParse(BasePydanticReader):
|
||||
if self.spreadsheet_extract_sub_tables:
|
||||
data["spreadsheet_extract_sub_tables"] = self.spreadsheet_extract_sub_tables
|
||||
|
||||
if self.spreadsheet_force_formula_computation:
|
||||
data[
|
||||
"spreadsheet_force_formula_computation"
|
||||
] = self.spreadsheet_force_formula_computation
|
||||
|
||||
if self.specialized_chart_parsing_agentic:
|
||||
data[
|
||||
"specialized_chart_parsing_agentic"
|
||||
@@ -1146,25 +1130,6 @@ class LlamaParse(BasePydanticReader):
|
||||
)
|
||||
current_interval = self._calculate_backoff(current_interval)
|
||||
|
||||
async def _get_job_result_with_error_handling(
|
||||
self, job_id: str, result_type: str, verbose: bool = False
|
||||
) -> Dict[str, Any]:
|
||||
"""Get job result with error handling based on ignore_errors setting."""
|
||||
try:
|
||||
return await self._get_job_result(job_id, result_type, verbose=verbose)
|
||||
except JobFailedException as e:
|
||||
if self.ignore_errors:
|
||||
# Return error information when ignore_errors is True
|
||||
return {
|
||||
"pages": [],
|
||||
"job_metadata": {},
|
||||
"error": f"{e.status}: {e.error_message or 'No error message'}",
|
||||
"error_code": e.error_code,
|
||||
"status": e.status,
|
||||
}
|
||||
else:
|
||||
raise e
|
||||
|
||||
async def _parse_one(
|
||||
self,
|
||||
file_path: FileInput,
|
||||
@@ -1206,7 +1171,7 @@ class LlamaParse(BasePydanticReader):
|
||||
)
|
||||
if self.verbose:
|
||||
print("Started parsing the file under job_id %s" % job_id)
|
||||
result = await self._get_job_result_with_error_handling(
|
||||
result = await self._get_job_result(
|
||||
job_id, result_type or self.result_type.value, verbose=self.verbose
|
||||
)
|
||||
return job_id, result
|
||||
@@ -1269,15 +1234,6 @@ class LlamaParse(BasePydanticReader):
|
||||
result_type=ResultType.JSON.value,
|
||||
partition_target_pages=f"{total}-{total + size - 1}",
|
||||
)
|
||||
# Check if the result is an error result (when ignore_errors=True)
|
||||
if json_result.get("error_code") == "NO_DATA_FOUND_IN_FILE":
|
||||
raise JobFailedException(
|
||||
job_id=job_id,
|
||||
status=json_result.get("status", "ERROR"),
|
||||
error_code=json_result.get("error_code"),
|
||||
error_message=json_result.get("error"),
|
||||
)
|
||||
|
||||
result_type = result_type or self.result_type.value
|
||||
if result_type == ResultType.JSON.value:
|
||||
job_result = json_result
|
||||
@@ -1803,7 +1759,7 @@ class LlamaParse(BasePydanticReader):
|
||||
JobResult object or list of JobResult objects if multiple job IDs were provided.
|
||||
"""
|
||||
if isinstance(job_id, str):
|
||||
result = await self._get_job_result_with_error_handling(
|
||||
result = await self._get_job_result(
|
||||
job_id, ResultType.JSON.value, verbose=self.verbose
|
||||
)
|
||||
return JobResult(
|
||||
@@ -1818,9 +1774,7 @@ class LlamaParse(BasePydanticReader):
|
||||
elif isinstance(job_id, list):
|
||||
results = []
|
||||
jobs = [
|
||||
self._get_job_result_with_error_handling(
|
||||
id_, ResultType.JSON.value, verbose=self.verbose
|
||||
)
|
||||
self._get_job_result(id_, ResultType.JSON.value, verbose=self.verbose)
|
||||
for id_ in job_id
|
||||
]
|
||||
results = await run_jobs(
|
||||
|
||||
@@ -1,87 +1,17 @@
|
||||
import httpx
|
||||
import os
|
||||
import re
|
||||
from pydantic import BaseModel, ConfigDict, Field, SerializeAsAny, model_validator
|
||||
from typing import Dict, Any, List, Optional, get_origin, get_args
|
||||
from pydantic import BaseModel, Field, SerializeAsAny
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
from llama_cloud_services.parse.utils import (
|
||||
make_api_request,
|
||||
is_jupyter,
|
||||
)
|
||||
from llama_cloud_services.parse.utils import make_api_request
|
||||
from llama_index.core.async_utils import asyncio_run
|
||||
from llama_index.core.schema import Document, ImageDocument, ImageNode, TextNode
|
||||
|
||||
PAGE_REGEX = r"page[-_](\d+)\.jpg$"
|
||||
|
||||
SAFE_MODEL_CONFIGS = ConfigDict(
|
||||
extra="allow",
|
||||
validate_assignment=False,
|
||||
arbitrary_types_allowed=True,
|
||||
validate_default=False,
|
||||
)
|
||||
|
||||
|
||||
class SafeBaseModel(BaseModel):
|
||||
"""Base model that gracefully handles None values from unstable backend responses."""
|
||||
|
||||
model_config = SAFE_MODEL_CONFIGS
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def coerce_none_to_defaults(cls, data: Any) -> Any:
|
||||
"""
|
||||
Replace None values with appropriate defaults based on field type annotations.
|
||||
This prevents validation errors when the backend returns None for non-optional fields.
|
||||
"""
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
|
||||
# Process each field that has a None value
|
||||
result = {}
|
||||
for key, value in data.items():
|
||||
if value is not None or key not in cls.model_fields:
|
||||
result[key] = value
|
||||
continue
|
||||
|
||||
# Value is None and field exists in model
|
||||
field_info = cls.model_fields[key]
|
||||
|
||||
# If field has a default or default_factory, let Pydantic handle it
|
||||
from pydantic_core import PydanticUndefined
|
||||
|
||||
if (
|
||||
field_info.default is not PydanticUndefined
|
||||
or field_info.default_factory is not None
|
||||
):
|
||||
continue
|
||||
|
||||
# Otherwise, provide a sensible default based on the type annotation
|
||||
annotation = field_info.annotation
|
||||
origin = get_origin(annotation)
|
||||
|
||||
# Handle List types
|
||||
if origin is list:
|
||||
result[key] = []
|
||||
# Handle Dict types
|
||||
elif origin is dict:
|
||||
result[key] = {}
|
||||
# Handle basic types
|
||||
elif annotation == str or (origin and str in get_args(annotation)):
|
||||
result[key] = ""
|
||||
elif annotation == int or (origin and int in get_args(annotation)):
|
||||
result[key] = 0
|
||||
elif annotation == float or (origin and float in get_args(annotation)):
|
||||
result[key] = 0.0
|
||||
elif annotation == bool or (origin and bool in get_args(annotation)):
|
||||
result[key] = False
|
||||
# If we can't determine a safe default, skip (let Pydantic try)
|
||||
else:
|
||||
result[key] = value
|
||||
|
||||
return result
|
||||
|
||||
|
||||
class JobMetadata(SafeBaseModel):
|
||||
class JobMetadata(BaseModel):
|
||||
"""Metadata about the job."""
|
||||
|
||||
job_pages: int = Field(default=0, description="The number of pages in the job.")
|
||||
@@ -94,31 +24,19 @@ class JobMetadata(SafeBaseModel):
|
||||
)
|
||||
|
||||
|
||||
class BBox(SafeBaseModel):
|
||||
class BBox(BaseModel):
|
||||
"""A bounding box."""
|
||||
|
||||
x: Optional[float] = Field(
|
||||
default=None,
|
||||
description="The x-coordinate of the bounding box.",
|
||||
)
|
||||
y: Optional[float] = Field(
|
||||
default=None,
|
||||
description="The y-coordinate of the bounding box.",
|
||||
)
|
||||
w: Optional[float] = Field(
|
||||
default=None,
|
||||
description="The width of the bounding box.",
|
||||
)
|
||||
h: Optional[float] = Field(
|
||||
default=None,
|
||||
description="The height of the bounding box.",
|
||||
)
|
||||
x: float = Field(description="The x-coordinate of the bounding box.")
|
||||
y: float = Field(description="The y-coordinate of the bounding box.")
|
||||
w: float = Field(description="The width of the bounding box.")
|
||||
h: float = Field(description="The height of the bounding box.")
|
||||
|
||||
|
||||
class PageItem(SafeBaseModel):
|
||||
class PageItem(BaseModel):
|
||||
"""An item in a page."""
|
||||
|
||||
type: str = Field(default="", description="The type of the item.")
|
||||
type: str = Field(description="The type of the item.")
|
||||
lvl: Optional[int] = Field(
|
||||
default=None, description="The level of indentation of the item."
|
||||
)
|
||||
@@ -140,10 +58,10 @@ class PageItem(SafeBaseModel):
|
||||
)
|
||||
|
||||
|
||||
class ImageItem(SafeBaseModel):
|
||||
class ImageItem(BaseModel):
|
||||
"""An image in a page."""
|
||||
|
||||
name: str = Field(default="", description="The name of the image.")
|
||||
name: str = Field(description="The name of the image.")
|
||||
height: Optional[float] = Field(
|
||||
default=None, description="The height of the image."
|
||||
)
|
||||
@@ -163,28 +81,22 @@ class ImageItem(SafeBaseModel):
|
||||
type: Optional[str] = Field(default=None, description="The type of the image.")
|
||||
|
||||
|
||||
class LayoutItem(SafeBaseModel):
|
||||
class LayoutItem(BaseModel):
|
||||
"""The layout of a page."""
|
||||
|
||||
image: str = Field(
|
||||
default="", description="The name of the image containing the layout item"
|
||||
)
|
||||
confidence: float = Field(
|
||||
default=0.0, description="The confidence of the layout item."
|
||||
)
|
||||
label: str = Field(default="", description="The label of the layout item.")
|
||||
image: str = Field(description="The name of the image containing the layout item")
|
||||
confidence: float = Field(description="The confidence of the layout item.")
|
||||
label: str = Field(description="The label of the layout item.")
|
||||
bbox: Optional[BBox] = Field(
|
||||
default=None, description="The bounding box of the layout item."
|
||||
)
|
||||
isLikelyNoise: bool = Field(
|
||||
default=False, description="Whether the layout item is likely noise."
|
||||
)
|
||||
isLikelyNoise: bool = Field(description="Whether the layout item is likely noise.")
|
||||
|
||||
|
||||
class ChartItem(SafeBaseModel):
|
||||
class ChartItem(BaseModel):
|
||||
"""A chart in a page."""
|
||||
|
||||
name: str = Field(default="", description="The name of the chart.")
|
||||
name: str = Field(description="The name of the chart.")
|
||||
x: Optional[float] = Field(
|
||||
default=None, description="The x-coordinate of the chart."
|
||||
)
|
||||
@@ -197,7 +109,7 @@ class ChartItem(SafeBaseModel):
|
||||
)
|
||||
|
||||
|
||||
class Page(SafeBaseModel):
|
||||
class Page(BaseModel):
|
||||
"""A page of the document."""
|
||||
|
||||
page: int = Field(default=0, description="The page number.")
|
||||
@@ -252,7 +164,7 @@ class Page(SafeBaseModel):
|
||||
)
|
||||
|
||||
|
||||
class JobResult(SafeBaseModel):
|
||||
class JobResult(BaseModel):
|
||||
"""The raw JSON result from the LlamaParse API."""
|
||||
|
||||
pages: List[Page] = Field(
|
||||
@@ -269,13 +181,6 @@ class JobResult(SafeBaseModel):
|
||||
error: Optional[str] = Field(
|
||||
default=None, description="The error message if the job failed."
|
||||
)
|
||||
error_code: Optional[str] = Field(
|
||||
default=None, description="The error code if the job failed."
|
||||
)
|
||||
status: Optional[str] = Field(
|
||||
default=None,
|
||||
description="The job status (e.g., PENDING, SUCCESS, ERROR, CANCELED).",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -353,29 +258,6 @@ class JobResult(SafeBaseModel):
|
||||
documents = await self.aget_text_documents(split_by_page)
|
||||
return [TextNode(text=doc.text, metadata=doc.metadata) for doc in documents]
|
||||
|
||||
def _format_markdown_for_notebook(self, text: Optional[str]) -> Optional[str]:
|
||||
"""Format markdown text for Jupyter notebook display by escaping dollar signs."""
|
||||
if text is None:
|
||||
return None
|
||||
|
||||
def escape_single_dollar_signs(text: str) -> str:
|
||||
"""Escape single dollar signs in text to prevent Jupyter from interpreting them as LaTeX.
|
||||
|
||||
Preserves all strings of dollar signs greater than length 1,
|
||||
especially preserving double dollar signs ($$) which denote LaTeX equations.
|
||||
|
||||
Args:
|
||||
text: The text to escape
|
||||
|
||||
Returns:
|
||||
Text with single dollar signs escaped
|
||||
"""
|
||||
# Replace single $ with \$, but preserve $$
|
||||
# Use negative lookahead and lookbehind to match $ not preceded or followed by $
|
||||
return re.sub(r"(?<!\$)\$(?!\$)", r"\$", text)
|
||||
|
||||
return escape_single_dollar_signs(text)
|
||||
|
||||
def get_markdown_documents(self, split_by_page: bool = False) -> List[Document]:
|
||||
"""
|
||||
Get the markdown documents from the job.
|
||||
@@ -386,22 +268,17 @@ class JobResult(SafeBaseModel):
|
||||
if split_by_page:
|
||||
return [
|
||||
Document(
|
||||
text=self._format_markdown_for_notebook(page.md)
|
||||
if is_jupyter()
|
||||
else page.md,
|
||||
text=page.md,
|
||||
metadata={"page_number": page.page, "file_name": self.file_name},
|
||||
)
|
||||
for page in self.pages
|
||||
]
|
||||
else:
|
||||
text = self._page_separator.join(
|
||||
[page.md if page.md is not None else "" for page in self.pages]
|
||||
)
|
||||
return [
|
||||
Document(
|
||||
text=self._format_markdown_for_notebook(text)
|
||||
if is_jupyter()
|
||||
else text,
|
||||
text=self._page_separator.join(
|
||||
[page.md if page.md is not None else "" for page in self.pages]
|
||||
),
|
||||
metadata={"file_name": self.file_name},
|
||||
)
|
||||
]
|
||||
@@ -451,10 +328,7 @@ class JobResult(SafeBaseModel):
|
||||
"""
|
||||
url = f"{self._base_url}/api/v1/parsing/job/{self.job_id}/result/raw/markdown"
|
||||
response = await make_api_request(self._client, "GET", url)
|
||||
markdown = response.content.decode("utf-8")
|
||||
return (
|
||||
self._format_markdown_for_notebook(markdown) if is_jupyter() else markdown
|
||||
)
|
||||
return response.content.decode("utf-8")
|
||||
|
||||
def get_text(self) -> str:
|
||||
"""
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import functools
|
||||
import httpx
|
||||
import itertools
|
||||
import logging
|
||||
@@ -357,17 +356,6 @@ def partition_pages(
|
||||
return
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def is_jupyter() -> bool:
|
||||
"""Check if we're running in a Jupyter environment."""
|
||||
try:
|
||||
from IPython import get_ipython
|
||||
|
||||
return get_ipython().__class__.__name__ == "ZMQInteractiveShell"
|
||||
except (ImportError, AttributeError):
|
||||
return False
|
||||
|
||||
|
||||
def extract_tables_from_json_results(
|
||||
json_results: List[dict], download_path: str
|
||||
) -> List[str]:
|
||||
|
||||
@@ -3,14 +3,11 @@ import importlib.metadata
|
||||
from contextlib import contextmanager
|
||||
from typing import Generator
|
||||
import difflib
|
||||
from llama_cloud.types import StatusEnum, File
|
||||
from llama_cloud.types import StatusEnum
|
||||
import httpx
|
||||
import packaging.version
|
||||
from pydantic import BaseModel
|
||||
from typing import Any, Dict, List, Tuple, Type, Union, Optional
|
||||
from io import BufferedIOBase, TextIOWrapper
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
from typing import Any, Dict, List, Tuple, Type
|
||||
|
||||
# Asyncio error messages
|
||||
nest_asyncio_err = "cannot be called from a running event loop"
|
||||
@@ -107,100 +104,3 @@ def augment_async_errors() -> Generator[None, None, None]:
|
||||
if nest_asyncio_err in str(e):
|
||||
raise RuntimeError(nest_asyncio_msg)
|
||||
raise
|
||||
|
||||
|
||||
class SourceText:
|
||||
"""
|
||||
A wrapper class for providing text or file input with optional filename specification.
|
||||
|
||||
This class allows you to provide input in multiple ways:
|
||||
- Direct text content via text_content parameter
|
||||
- File paths as strings or Path objects
|
||||
- Raw bytes
|
||||
- File-like objects (BufferedIOBase, TextIOWrapper)
|
||||
- Already-uploaded file ID via file_id parameter
|
||||
|
||||
Args:
|
||||
file: The file input (bytes, file-like object, str path, or Path).
|
||||
Mutually exclusive with text_content and file_id.
|
||||
text_content: Raw text content to process. Mutually exclusive with file and file_id.
|
||||
file_id: ID of an already-uploaded file. Mutually exclusive with file and text_content.
|
||||
filename: Optional filename. Required for bytes/file-like objects without names.
|
||||
If not provided, will be auto-generated for text_content or inferred from paths.
|
||||
|
||||
Examples:
|
||||
# Direct text input
|
||||
source = SourceText(text_content="Hello world")
|
||||
|
||||
# File path
|
||||
source = SourceText(file="document.pdf")
|
||||
|
||||
# Bytes with filename
|
||||
source = SourceText(file=b"...", filename="document.pdf")
|
||||
|
||||
# File-like object (will read from current position)
|
||||
with open("document.pdf", "rb") as f:
|
||||
source = SourceText(file=f)
|
||||
|
||||
# Already-uploaded file
|
||||
source = SourceText(file_id="file_abc123")
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
file: Union[bytes, BufferedIOBase, TextIOWrapper, str, Path, None] = None,
|
||||
text_content: Optional[str] = None,
|
||||
file_id: Optional[str] = None,
|
||||
filename: Optional[str] = None,
|
||||
):
|
||||
self.file = file
|
||||
self.filename = filename
|
||||
self.text_content = text_content
|
||||
self.file_id = file_id
|
||||
self._validate()
|
||||
|
||||
def _validate(self) -> None:
|
||||
"""Ensure filename is provided when needed."""
|
||||
# Check that exactly one of file, text_content, or file_id is provided
|
||||
provided = sum(
|
||||
[
|
||||
self.file is not None,
|
||||
self.text_content is not None,
|
||||
self.file_id is not None,
|
||||
]
|
||||
)
|
||||
|
||||
if provided == 0:
|
||||
raise ValueError("One of file, text_content, or file_id must be provided.")
|
||||
elif provided > 1:
|
||||
raise ValueError(
|
||||
"Only one of file, text_content, or file_id can be provided."
|
||||
)
|
||||
|
||||
# If file_id is provided, we don't need filename validation
|
||||
if self.file_id is not None:
|
||||
return
|
||||
|
||||
if self.text_content is not None:
|
||||
if not self.filename:
|
||||
random_hex = secrets.token_hex(4)
|
||||
self.filename = f"text_input_{random_hex}.txt"
|
||||
return
|
||||
|
||||
if isinstance(self.file, (bytes, BufferedIOBase, TextIOWrapper)):
|
||||
if not self.filename and hasattr(self.file, "name"):
|
||||
self.filename = os.path.basename(str(self.file.name))
|
||||
elif self.filename is None and not hasattr(self.file, "name"):
|
||||
raise ValueError(
|
||||
"filename must be provided when file is bytes or a file-like object without a name"
|
||||
)
|
||||
elif isinstance(self.file, (str, Path)):
|
||||
if not self.filename:
|
||||
self.filename = os.path.basename(str(self.file))
|
||||
else:
|
||||
raise ValueError(f"Unsupported file type: {type(self.file)}")
|
||||
|
||||
|
||||
# Type alias for file input that can be used across services
|
||||
FileInput = Union[str, Path, BufferedIOBase, SourceText, File]
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
# llama_parse
|
||||
|
||||
## 0.6.78
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 9f1ef4e: Fix extract
|
||||
- Updated dependencies [9f1ef4e]
|
||||
- llama-cloud-services-py@0.6.78
|
||||
|
||||
## 0.6.77
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [407292b]
|
||||
- llama-cloud-services-py@0.6.77
|
||||
|
||||
## 0.6.76
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [4f24f53]
|
||||
- llama-cloud-services-py@0.6.76
|
||||
|
||||
## 0.6.75
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [f81532e]
|
||||
- llama-cloud-services-py@0.6.75
|
||||
|
||||
## 0.6.74
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [1bf5223]
|
||||
- Updated dependencies [24166dc]
|
||||
- llama-cloud-services-py@0.6.74
|
||||
|
||||
## 0.6.73
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [e6a7939]
|
||||
- llama-cloud-services-py@0.6.73
|
||||
|
||||
## 0.6.72
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- Updated dependencies [ad6734b]
|
||||
- llama-cloud-services-py@0.6.72
|
||||
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"name": "llama_parse",
|
||||
"version": "0.6.78",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"private": false,
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"dependencies": {
|
||||
"llama-cloud-services-py": "workspace:*"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"packageManager": "pnpm@10.11.1",
|
||||
"devDependencies": {
|
||||
"changesets": "^1.0.2"
|
||||
}
|
||||
}
|
||||
@@ -11,13 +11,13 @@ dev = [
|
||||
|
||||
[project]
|
||||
name = "llama-parse"
|
||||
version = "0.6.78"
|
||||
version = "0.6.65"
|
||||
description = "Parse files into RAG-Optimized formats."
|
||||
authors = [{name = "Logan Markewich", email = "logan@llamaindex.ai"}]
|
||||
requires-python = ">=3.9,<4.0"
|
||||
readme = "README.md"
|
||||
license = "MIT"
|
||||
dependencies = ["llama-cloud-services>=0.6.78"]
|
||||
dependencies = ["llama-cloud-services>=0.6.64"]
|
||||
|
||||
[project.scripts]
|
||||
llama-parse = "llama_parse.cli.main:parse"
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
{
|
||||
"name": "llama-cloud-services-py",
|
||||
"version": "0.6.78",
|
||||
"private": false,
|
||||
"license": "MIT",
|
||||
"scripts": {},
|
||||
"devDependencies": {
|
||||
"changesets": "^1.0.2"
|
||||
}
|
||||
}
|
||||
@@ -19,7 +19,7 @@ dev = [
|
||||
|
||||
[project]
|
||||
name = "llama-cloud-services"
|
||||
version = "0.6.78"
|
||||
version = "0.6.65"
|
||||
description = "Tailored SDK clients for LlamaCloud services."
|
||||
authors = [{name = "Logan Markewich", email = "logan@runllama.ai"}]
|
||||
requires-python = ">=3.9,<4.0"
|
||||
@@ -27,14 +27,14 @@ readme = "README.md"
|
||||
license = "MIT"
|
||||
dependencies = [
|
||||
"llama-index-core>=0.12.0",
|
||||
"llama-cloud==0.1.44",
|
||||
"llama-cloud==0.1.41",
|
||||
"pydantic>=2.8,!=2.10",
|
||||
"click>=8.1.7,<9",
|
||||
"python-dotenv>=1.0.1,<2",
|
||||
"eval-type-backport>=0.2.0,<0.3 ; python_version < '3.10'",
|
||||
"platformdirs>=4.3.7,<5",
|
||||
"tenacity>=8.5.0, <10.0",
|
||||
"packaging>=23.0"
|
||||
"packaging>=25.0"
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
|
||||
@@ -68,7 +68,7 @@ async def test_agent_data_crud_operations():
|
||||
client=client,
|
||||
type=ExampleData,
|
||||
collection=f"test-collection-{test_id[:8]}",
|
||||
deployment_name=LLAMA_DEPLOY_DEPLOYMENT_NAME,
|
||||
agent_url_id=LLAMA_DEPLOY_DEPLOYMENT_NAME,
|
||||
)
|
||||
|
||||
# Create test data
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
import os
|
||||
import pytest
|
||||
|
||||
from llama_cloud_services.extract import LlamaExtract
|
||||
from llama_cloud_services.extract import LlamaExtract, ExtractionAgent
|
||||
from time import perf_counter
|
||||
from collections import namedtuple
|
||||
import json
|
||||
import uuid
|
||||
from llama_cloud.types import (
|
||||
ExtractConfig,
|
||||
ExtractMode,
|
||||
LlamaParseParameters,
|
||||
LlamaExtractSettings,
|
||||
)
|
||||
from tests.extract.util import load_test_dotenv
|
||||
|
||||
@@ -119,3 +122,27 @@ def extraction_agent(test_case: BenchmarkTestCase, extractor: LlamaExtract):
|
||||
# Create new agent
|
||||
agent = extractor.create_agent(agent_name, schema, config=test_case.config)
|
||||
yield agent
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
"CI" in os.environ or not LLAMA_CLOUD_API_KEY,
|
||||
reason="LLAMA_CLOUD_API_KEY not set or CI environment not suitable for benchmarking",
|
||||
)
|
||||
@pytest.mark.parametrize("test_case", get_test_cases(), ids=lambda x: x.name)
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_extraction(
|
||||
test_case: BenchmarkTestCase, extraction_agent: ExtractionAgent
|
||||
) -> None:
|
||||
start = perf_counter()
|
||||
result = await extraction_agent._run_extraction_test(
|
||||
test_case.input_file,
|
||||
extract_settings=LlamaExtractSettings(
|
||||
llama_parse_params=LlamaParseParameters(
|
||||
invalidate_cache=True,
|
||||
do_not_cache=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
end = perf_counter()
|
||||
print(f"Time taken: {end - start} seconds")
|
||||
print(result)
|
||||
|
||||
@@ -58,8 +58,6 @@ def get_test_cases():
|
||||
settings = [
|
||||
ExtractConfig(extraction_mode=ExtractMode.FAST),
|
||||
ExtractConfig(extraction_mode=ExtractMode.BALANCED),
|
||||
ExtractConfig(extraction_mode=ExtractMode.MULTIMODAL),
|
||||
ExtractConfig(extraction_mode=ExtractMode.PREMIUM),
|
||||
]
|
||||
|
||||
for input_file in sorted(input_files):
|
||||
|
||||
@@ -7,7 +7,7 @@ from pathlib import Path
|
||||
|
||||
|
||||
def load_test_dotenv():
|
||||
load_dotenv(Path(__file__).parent.parent.parent.parent / ".env.dev", override=True)
|
||||
load_dotenv(Path(__file__).parent.parent.parent / ".env.dev", override=True)
|
||||
|
||||
|
||||
def json_subset_match_score(expected: Any, actual: Any) -> float:
|
||||
|
||||
@@ -44,7 +44,7 @@ def index_name() -> Generator[str, None, None]:
|
||||
client = LlamaCloud(token=api_key, base_url=base_url)
|
||||
pipeline = client.pipelines.search_pipelines(project_name=name)
|
||||
if pipeline:
|
||||
client.pipelines.delete_pipeline(pipeline_id=pipeline[0].id)
|
||||
client.pipelines.delete(pipeline_id=pipeline[0].id)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
@@ -83,7 +83,7 @@ def _setup_index_with_file(
|
||||
|
||||
# add file to pipeline
|
||||
pipeline_file_create = PipelineFileCreate(file_id=file.id)
|
||||
client.pipeline_files.add_files_to_pipeline_api(
|
||||
client.pipelines.add_files_to_pipeline_api(
|
||||
pipeline_id=pipeline.id, request=[pipeline_file_create]
|
||||
)
|
||||
|
||||
@@ -304,9 +304,6 @@ async def test_page_screenshot_retrieval(index_name: str, local_file: str):
|
||||
not base_url or not api_key, reason="No platform base url or api key set"
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip(
|
||||
reason="Consistently failing with FAILED tests/index/test_index.py::test_page_figure_retrieval - assert 0 > 0 + where 0 = len([])"
|
||||
)
|
||||
async def test_page_figure_retrieval(index_name: str, local_figures_file: str):
|
||||
index = await LlamaCloudIndex.acreate_index(
|
||||
name=index_name,
|
||||
|
||||
@@ -6,40 +6,6 @@ from llama_cloud_services import LlamaParse
|
||||
from llama_cloud_services.parse.types import JobResult
|
||||
|
||||
|
||||
def test_format_parse_result_markdown_for_notebook():
|
||||
"""Test the _format_markdown_for_notebook function.
|
||||
Right now, the only work it does is escape single dollar signs."""
|
||||
result = JobResult(job_id="test", file_name="test.pdf", job_result={})
|
||||
|
||||
# Test None input
|
||||
assert result._format_markdown_for_notebook(None) is None
|
||||
|
||||
# Test single dollar sign gets escaped
|
||||
assert result._format_markdown_for_notebook("This costs $5") == "This costs \\$5"
|
||||
|
||||
# Test double dollar signs are preserved (LaTeX equations)
|
||||
assert (
|
||||
result._format_markdown_for_notebook("$$x^2 + y^2 = z^2$$")
|
||||
== "$$x^2 + y^2 = z^2$$"
|
||||
)
|
||||
|
||||
# Test mixed single and double dollar signs
|
||||
text = "This costs $5, but $$E = mc^2$$ is priceless"
|
||||
expected = "This costs \\$5, but $$E = mc^2$$ is priceless"
|
||||
assert result._format_markdown_for_notebook(text) == expected
|
||||
|
||||
# Test multiple single dollar signs
|
||||
assert result._format_markdown_for_notebook("$10 and $20") == "\\$10 and \\$20"
|
||||
|
||||
# Test three or more consecutive dollar signs (preserve them)
|
||||
assert result._format_markdown_for_notebook("$$$") == "$$$"
|
||||
|
||||
# Test adjacent dollar signs with text in between
|
||||
text = "$$inline$$ and $separate"
|
||||
expected = "$$inline$$ and \\$separate"
|
||||
assert result._format_markdown_for_notebook(text) == expected
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def file_path() -> str:
|
||||
return "tests/test_files/attention_is_all_you_need.pdf"
|
||||
|
||||
@@ -1,158 +0,0 @@
|
||||
import pytest
|
||||
from typing import Any, Dict, List, Optional
|
||||
from pydantic import BaseModel
|
||||
from datetime import datetime
|
||||
|
||||
from llama_cloud.types.agent_data import AgentData
|
||||
from llama_cloud.types.aggregate_group import AggregateGroup
|
||||
|
||||
from llama_cloud_services.beta.agent_data.client import AsyncAgentDataClient
|
||||
|
||||
|
||||
class Person(BaseModel):
|
||||
name: str
|
||||
age: int
|
||||
|
||||
|
||||
class FakeBeta:
|
||||
def __init__(self) -> None:
|
||||
self._get_item_response: Optional[AgentData] = None
|
||||
self._search_items: List[AgentData] = []
|
||||
self._aggregate_items: List[AggregateGroup] = []
|
||||
self._total_size: Optional[int] = None
|
||||
self._next_page_token: Optional[str] = None
|
||||
|
||||
# Single get
|
||||
async def get_agent_data(self, item_id: str) -> AgentData:
|
||||
assert self._get_item_response is not None, "_get_item_response not set"
|
||||
return self._get_item_response
|
||||
|
||||
# Search
|
||||
async def search_agent_data_api_v_1_beta_agent_data_search_post(
|
||||
self,
|
||||
*,
|
||||
deployment_name: str,
|
||||
collection: str,
|
||||
filter: Optional[Dict[str, Any]] = None,
|
||||
order_by: Optional[str] = None,
|
||||
offset: Optional[int] = None,
|
||||
page_size: Optional[int] = None,
|
||||
include_total: bool = False,
|
||||
) -> Any:
|
||||
class Resp:
|
||||
def __init__(
|
||||
self,
|
||||
items: List[AgentData],
|
||||
total_size: Optional[int],
|
||||
next_page_token: Optional[str],
|
||||
) -> None:
|
||||
self.items = items
|
||||
self.total_size = total_size
|
||||
self.next_page_token = next_page_token
|
||||
|
||||
return Resp(self._search_items, self._total_size, self._next_page_token)
|
||||
|
||||
# Aggregate
|
||||
async def aggregate_agent_data_api_v_1_beta_agent_data_aggregate_post(
|
||||
self,
|
||||
*,
|
||||
deployment_name: str,
|
||||
collection: str,
|
||||
page_size: Optional[int] = None,
|
||||
filter: Optional[Dict[str, Any]] = None,
|
||||
order_by: Optional[str] = None,
|
||||
group_by: Optional[List[str]] = None,
|
||||
count: Optional[bool] = None,
|
||||
first: Optional[bool] = None,
|
||||
offset: Optional[int] = None,
|
||||
) -> Any:
|
||||
class Resp:
|
||||
def __init__(
|
||||
self,
|
||||
items: List[AggregateGroup],
|
||||
total_size: Optional[int],
|
||||
next_page_token: Optional[str],
|
||||
) -> None:
|
||||
self.items = items
|
||||
self.total_size = total_size
|
||||
self.next_page_token = next_page_token
|
||||
|
||||
return Resp(self._aggregate_items, self._total_size, self._next_page_token)
|
||||
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self) -> None:
|
||||
self.beta = FakeBeta()
|
||||
|
||||
|
||||
def make_agent_data(data: Dict[str, Any]) -> AgentData:
|
||||
return AgentData(
|
||||
id="id-1",
|
||||
deployment_name="dep",
|
||||
collection="col",
|
||||
data=data,
|
||||
created_at=datetime.now(),
|
||||
updated_at=datetime.now(),
|
||||
)
|
||||
|
||||
|
||||
def make_group(
|
||||
group_key: Dict[str, Any],
|
||||
first_item: Optional[Dict[str, Any]],
|
||||
count: Optional[int] = None,
|
||||
) -> AggregateGroup:
|
||||
return AggregateGroup(group_key=group_key, count=count, first_item=first_item)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_untyped_get_item_valid_to_dict() -> None:
|
||||
client = FakeClient()
|
||||
client.beta._get_item_response = make_agent_data({"name": "Alice", "age": 30})
|
||||
|
||||
adc = AsyncAgentDataClient(type=Person, client=client, deployment_name="dep")
|
||||
item = await adc.untyped_get_item("id-1")
|
||||
assert item.data == {"name": "Alice", "age": 30}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_untyped_get_item_invalid_retains_dict() -> None:
|
||||
client = FakeClient()
|
||||
# age wrong type; will fail validation and should be returned as dict
|
||||
client.beta._get_item_response = make_agent_data({"name": "Bob", "age": "x"})
|
||||
|
||||
adc = AsyncAgentDataClient(type=Person, client=client, deployment_name="dep")
|
||||
item = await adc.untyped_get_item("id-1")
|
||||
assert item.data == {"name": "Bob", "age": "x"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_untyped_search_mixed_items() -> None:
|
||||
client = FakeClient()
|
||||
client.beta._search_items = [
|
||||
make_agent_data({"name": "Carol", "age": 22}),
|
||||
make_agent_data({"name": "Dave", "age": "bad"}),
|
||||
]
|
||||
client.beta._total_size = 2
|
||||
|
||||
adc = AsyncAgentDataClient(type=Person, client=client, deployment_name="dep")
|
||||
results = await adc.untyped_search(include_total=True)
|
||||
assert len(results.items) == 2
|
||||
assert results.items[0].data == {"name": "Carol", "age": 22}
|
||||
assert results.items[1].data == {"name": "Dave", "age": "bad"}
|
||||
assert results.total_size == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_untyped_aggregate_first_item_dict() -> None:
|
||||
client = FakeClient()
|
||||
client.beta._aggregate_items = [
|
||||
make_group({"k": 1}, {"name": "Eve", "age": 40}),
|
||||
make_group({"k": 2}, {"name": "Frank", "age": "bad"}),
|
||||
]
|
||||
client.beta._total_size = 2
|
||||
|
||||
adc = AsyncAgentDataClient(type=Person, client=client, deployment_name="dep")
|
||||
results = await adc.untyped_aggregate(group_by=["k"], first=True)
|
||||
assert len(results.items) == 2
|
||||
assert results.items[0].first_item == {"name": "Eve", "age": 40}
|
||||
assert results.items[1].first_item == {"name": "Frank", "age": "bad"}
|
||||
@@ -2,7 +2,6 @@ from datetime import datetime
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from llama_cloud import ExtractRun, File
|
||||
@@ -39,7 +38,7 @@ def test_typed_agent_data_from_raw():
|
||||
"""Test TypedAgentData.from_raw class method."""
|
||||
raw_data = AgentData(
|
||||
id="456",
|
||||
deployment_name="extraction-agent",
|
||||
agent_slug="extraction-agent",
|
||||
collection="employees",
|
||||
data={"name": "Jane Smith", "age": 25, "email": "jane@company.com"},
|
||||
created_at=datetime.now(),
|
||||
@@ -49,7 +48,7 @@ def test_typed_agent_data_from_raw():
|
||||
typed_data = TypedAgentData.from_raw(raw_data, Person)
|
||||
|
||||
assert typed_data.id == "456"
|
||||
assert typed_data.deployment_name == "extraction-agent"
|
||||
assert typed_data.agent_url_id == "extraction-agent"
|
||||
assert typed_data.collection == "employees"
|
||||
assert typed_data.data.name == "Jane Smith"
|
||||
assert typed_data.data.age == 25
|
||||
@@ -57,10 +56,10 @@ def test_typed_agent_data_from_raw():
|
||||
|
||||
|
||||
def test_typed_agent_data_from_raw_validation_error():
|
||||
"""Test TypedAgentData.from_raw with invalid data now raises InvalidTypedAgentData."""
|
||||
"""Test TypedAgentData.from_raw with invalid data."""
|
||||
raw_data = AgentData(
|
||||
id="789",
|
||||
deployment_name="test-agent",
|
||||
agent_slug="test-agent",
|
||||
collection="people",
|
||||
data={"name": "Invalid Person", "age": "not_a_number"}, # Invalid age
|
||||
created_at=datetime.now(),
|
||||
@@ -435,7 +434,6 @@ def create_extract_run(
|
||||
"extraction_agent_id": "extraction-agent-123",
|
||||
"config": {},
|
||||
"status": "SUCCESS",
|
||||
"project_id": str(uuid.uuid4()),
|
||||
"from_ui": False,
|
||||
}
|
||||
)
|
||||
@@ -615,51 +613,3 @@ def test_parses_field_metadata_with_error_field():
|
||||
}
|
||||
assert parsed.metadata.get("field_errors") == "This is an error"
|
||||
assert parsed.metadata.get("job_id") == "job-123"
|
||||
|
||||
|
||||
REASONING_IN_SCHEMA = {
|
||||
"majority_opinion": {
|
||||
"type": {
|
||||
"citation": [
|
||||
{
|
||||
"page": 4,
|
||||
"matching_text": "BARRETT, J., delivered the opinion for a unanimous Court.",
|
||||
},
|
||||
{"page": 11, "matching_text": "Opinion of the Court"},
|
||||
],
|
||||
"parsing_confidence": 1.0,
|
||||
"extraction_confidence": 0.9999998919950147,
|
||||
"confidence": 0.9999998919950147,
|
||||
},
|
||||
"reasoning": {
|
||||
"citation": [
|
||||
{
|
||||
"page": 15,
|
||||
"matching_text": "We hold that §5110(b)(1) is not subject to equitable tolling and affirm the judg...",
|
||||
}
|
||||
],
|
||||
"parsing_confidence": 1.0,
|
||||
"extraction_confidence": 0.414292785946868,
|
||||
"confidence": 0.414292785946868,
|
||||
},
|
||||
},
|
||||
"reasoning": {
|
||||
"citation": [
|
||||
{
|
||||
"page": 15,
|
||||
"matching_text": "We hold that §5110(b)(1) is not subject to equitable tolling and affirm the judg...",
|
||||
}
|
||||
],
|
||||
"parsing_confidence": 1.0,
|
||||
"extraction_confidence": 0.414292785946868,
|
||||
"confidence": 0.414292785946868,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_field_conflict_in_schema():
|
||||
extracted = parse_extracted_field_metadata(REASONING_IN_SCHEMA)
|
||||
assert isinstance(extracted["reasoning"], ExtractedFieldMetadata)
|
||||
assert isinstance(
|
||||
extracted["majority_opinion"]["reasoning"], ExtractedFieldMetadata
|
||||
)
|
||||
|
||||
@@ -112,6 +112,5 @@
|
||||
"num_output_tokens": 3440
|
||||
}
|
||||
},
|
||||
"project_id": "77bdc79f-fb69-49ae-a783-fcc573eec7ce",
|
||||
"from_ui": false
|
||||
}
|
||||
|
||||
@@ -118,8 +118,10 @@ async def test_extraction_agent_aextract_accepts_llama_file(
|
||||
dummy_llama_extract_iface = SimpleNamespace()
|
||||
|
||||
async def fake_run_job(**kwargs):
|
||||
file_id = kwargs.get("file_id")
|
||||
assert file_id == llama_file.id
|
||||
# Ensure we are receiving a request with the right file_id
|
||||
request = kwargs.get("request")
|
||||
assert hasattr(request, "file_id")
|
||||
assert request.file_id == llama_file.id
|
||||
return SimpleNamespace(id="job_42")
|
||||
|
||||
dummy_llama_extract_iface.run_job = fake_run_job
|
||||
|
||||
@@ -1,155 +1,16 @@
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import llama_cloud_services.index.base as base
|
||||
from llama_cloud import (
|
||||
PipelineEmbeddingConfig_ManagedOpenaiEmbedding,
|
||||
Project,
|
||||
Pipeline,
|
||||
CloudDocument,
|
||||
)
|
||||
from llama_index.core.constants import DEFAULT_PROJECT_NAME
|
||||
from llama_index.core.indices.managed.base import BaseManagedIndex
|
||||
from llama_index.core.schema import Document
|
||||
from llama_cloud_services.index import LlamaCloudIndex
|
||||
|
||||
|
||||
# Simple test data as values, not fixtures
|
||||
TEST_PROJECT = Project(id="proj-123", name="test-project", organization_id="org-123")
|
||||
|
||||
EMBEDDING_CONFIG = PipelineEmbeddingConfig_ManagedOpenaiEmbedding(
|
||||
type="MANAGED_OPENAI_EMBEDDING"
|
||||
)
|
||||
TEST_PIPELINE = Pipeline(
|
||||
id="pipe-456",
|
||||
name="test-pipeline",
|
||||
project_id="proj-123",
|
||||
embedding_config=PipelineEmbeddingConfig_ManagedOpenaiEmbedding(
|
||||
type="MANAGED_OPENAI_EMBEDDING"
|
||||
),
|
||||
from llama_cloud_services.index import (
|
||||
LlamaCloudIndex,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_client() -> MagicMock:
|
||||
"""Mock client with sensible defaults."""
|
||||
client = MagicMock()
|
||||
client.projects.upsert_project.return_value = Project(
|
||||
id="default-proj", name=DEFAULT_PROJECT_NAME, organization_id="default-org"
|
||||
)
|
||||
client.pipelines.upsert_pipeline.return_value = Pipeline(
|
||||
id="default-pipe",
|
||||
name="default",
|
||||
project_id="default-proj",
|
||||
embedding_config=EMBEDDING_CONFIG,
|
||||
)
|
||||
client.pipelines.upsert_batch_pipeline_documents.return_value = [
|
||||
CloudDocument(id="doc-1", text="test", metadata={})
|
||||
]
|
||||
return client
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def base_patches(mock_client: MagicMock) -> None:
|
||||
"""Auto-applied patches for all tests."""
|
||||
with (
|
||||
patch.object(base, "get_client", return_value=mock_client),
|
||||
patch.object(
|
||||
base,
|
||||
"resolve_project_and_pipeline",
|
||||
return_value=(TEST_PROJECT, TEST_PIPELINE),
|
||||
),
|
||||
patch.object(base.LlamaCloudIndex, "wait_for_completion"),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
def test_class() -> None:
|
||||
def test_class():
|
||||
names_of_base_classes = [b.__name__ for b in LlamaCloudIndex.__mro__]
|
||||
assert BaseManagedIndex.__name__ in names_of_base_classes
|
||||
|
||||
|
||||
def test_conflicting_index_identifiers() -> None:
|
||||
def test_conflicting_index_identifiers():
|
||||
with pytest.raises(ValueError):
|
||||
LlamaCloudIndex(name="test", pipeline_id="test", index_id="test")
|
||||
|
||||
|
||||
def test_from_documents_uses_provided_project_id(mock_client: MagicMock) -> None:
|
||||
provided_project_id = "proj-123"
|
||||
organization_id = "org-abc"
|
||||
index_name = "my_new_index"
|
||||
|
||||
# Override resolve to return project with provided ID
|
||||
test_project = Project(
|
||||
id=provided_project_id, name="my_project", organization_id=organization_id
|
||||
)
|
||||
test_pipeline = Pipeline(
|
||||
id="pipe-xyz",
|
||||
name=index_name,
|
||||
project_id=provided_project_id,
|
||||
embedding_config=EMBEDDING_CONFIG,
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
base, "resolve_project_and_pipeline", return_value=(test_project, test_pipeline)
|
||||
):
|
||||
docs = [Document(text="hello")]
|
||||
index = LlamaCloudIndex.from_documents(
|
||||
documents=docs,
|
||||
name=index_name,
|
||||
project_id=provided_project_id,
|
||||
)
|
||||
|
||||
# Assert - project upsert not called; pipeline uses provided project_id
|
||||
mock_client.projects.upsert_project.assert_not_called()
|
||||
assert mock_client.pipelines.upsert_pipeline.call_count == 1
|
||||
assert (
|
||||
mock_client.pipelines.upsert_pipeline.call_args.kwargs["project_id"]
|
||||
== provided_project_id
|
||||
)
|
||||
assert index.project.id == provided_project_id
|
||||
|
||||
|
||||
def test_from_documents_upserts_project_when_project_id_missing(
|
||||
mock_client: MagicMock,
|
||||
) -> None:
|
||||
organization_id = "org-xyz"
|
||||
index_name = "my_new_index"
|
||||
|
||||
# Project is created when project_id is not provided
|
||||
upserted_project = Project(
|
||||
id="proj-999", name=DEFAULT_PROJECT_NAME, organization_id=organization_id
|
||||
)
|
||||
mock_client.projects.upsert_project.return_value = upserted_project
|
||||
|
||||
test_pipeline = Pipeline(
|
||||
id="pipe-xyz",
|
||||
name=index_name,
|
||||
project_id=upserted_project.id,
|
||||
embedding_config=EMBEDDING_CONFIG,
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
base,
|
||||
"resolve_project_and_pipeline",
|
||||
return_value=(upserted_project, test_pipeline),
|
||||
):
|
||||
docs = [Document(text="world")]
|
||||
index = LlamaCloudIndex.from_documents(
|
||||
documents=docs,
|
||||
name=index_name,
|
||||
organization_id=organization_id,
|
||||
)
|
||||
|
||||
# Assert - project was upserted with org id and default project name
|
||||
mock_client.projects.upsert_project.assert_called_once()
|
||||
kwargs = mock_client.projects.upsert_project.call_args.kwargs
|
||||
assert kwargs["organization_id"] == organization_id
|
||||
assert kwargs["request"].name == DEFAULT_PROJECT_NAME
|
||||
|
||||
# Pipeline created under the upserted project id
|
||||
assert (
|
||||
mock_client.pipelines.upsert_pipeline.call_args.kwargs["project_id"]
|
||||
== upserted_project.id
|
||||
)
|
||||
assert index.project.id == upserted_project.id
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
version = 1
|
||||
revision = 3
|
||||
revision = 2
|
||||
requires-python = ">=3.9, <4.0"
|
||||
resolution-markers = [
|
||||
"python_full_version >= '3.14'",
|
||||
@@ -1582,21 +1582,21 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "llama-cloud"
|
||||
version = "0.1.44"
|
||||
version = "0.1.41"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "certifi" },
|
||||
{ name = "httpx" },
|
||||
{ name = "pydantic" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/54/eb/16e31fb0fc4df91b08fa19cc3f28ac6e3c7d4df0bcbb71dd2bf596e9586f/llama_cloud-0.1.44.tar.gz", hash = "sha256:276a2b4f94463da037431ca3063331b3b6be398bbfb003113ee76b7c2a873b53", size = 120502, upload-time = "2025-11-04T00:51:58.578Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/62/6c/b2e84eebed376aea34c446cab745da5fc4e9dc53309180672299083219d5/llama_cloud-0.1.41.tar.gz", hash = "sha256:dcb741b779e3e740cd64928cfffc8ef70ed0e9bae9ef26acbe1d7e32aa737bdc", size = 109854, upload-time = "2025-09-05T22:45:13.069Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/69/0a/fabe54c21d5927d626550cb9560a20e51e42468355f5f0fb300f84806e28/llama_cloud-0.1.44-py3-none-any.whl", hash = "sha256:dfdcc4932353711fc8639f14261cbb54a88139b7790ebdd3ed4fde29bbbc0b88", size = 332779, upload-time = "2025-11-04T00:51:57.371Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1e/4d/f0af76b389310840ce3483a92560a152025b0eefe4eee0c81102bf3317e6/llama_cloud-0.1.41-py3-none-any.whl", hash = "sha256:c847f288f0d3f4b23f47345088006deae5f2cf3f223ac1819d4c1531e9aaa13e", size = 307646, upload-time = "2025-09-05T22:45:11.597Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "llama-cloud-services"
|
||||
version = "0.6.77"
|
||||
version = "0.6.65"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
|
||||
@@ -1631,9 +1631,9 @@ dev = [
|
||||
requires-dist = [
|
||||
{ name = "click", specifier = ">=8.1.7,<9" },
|
||||
{ name = "eval-type-backport", marker = "python_full_version < '3.10'", specifier = ">=0.2.0,<0.3" },
|
||||
{ name = "llama-cloud", specifier = "==0.1.44" },
|
||||
{ name = "llama-cloud", specifier = "==0.1.41" },
|
||||
{ name = "llama-index-core", specifier = ">=0.12.0" },
|
||||
{ name = "packaging", specifier = ">=23.0" },
|
||||
{ name = "packaging", specifier = ">=25.0" },
|
||||
{ name = "platformdirs", specifier = ">=4.3.7,<5" },
|
||||
{ name = "pydantic", specifier = ">=2.8,!=2.10" },
|
||||
{ name = "python-dotenv", specifier = ">=1.0.1,<2" },
|
||||
|
||||
@@ -1,290 +0,0 @@
|
||||
#!/usr/bin/env -S uv run --script
|
||||
# /// script
|
||||
# dependencies = ["click", "tomlkit", "packaging"]
|
||||
# ///
|
||||
|
||||
"""
|
||||
This is a script called by the changeset bot. Normally changeset can do the following things, but this is a mixed ts and python repo, so we need to do some extra things.
|
||||
|
||||
There's 2 things this does:
|
||||
- Versioning: Makes changes that may be committed with the newest version.
|
||||
- Releasing/Tagging: After versions are changed, we check each package to see if its released, and if not, we release it and tag it.
|
||||
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any, List, cast
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import re
|
||||
|
||||
import click
|
||||
import tomlkit
|
||||
from packaging.version import Version
|
||||
|
||||
|
||||
def _run_command(
|
||||
cmd: List[str], cwd: Path | None = None, env: dict[str, str] | None = None
|
||||
) -> None:
|
||||
"""Run a command, streaming output to the console, and raise on failure."""
|
||||
subprocess.run(cmd, check=True, text=True, cwd=cwd or Path.cwd(), env=env)
|
||||
|
||||
|
||||
def _run_and_capture(
|
||||
cmd: List[str], cwd: Path | None = None, env: dict[str, str] | None = None
|
||||
) -> str:
|
||||
"""Run a command and return stdout as text, raising on failure."""
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
check=True,
|
||||
text=True,
|
||||
cwd=cwd or Path.cwd(),
|
||||
env=env,
|
||||
capture_output=True,
|
||||
)
|
||||
return result.stdout
|
||||
|
||||
|
||||
@dataclass
|
||||
class Package:
|
||||
name: str
|
||||
version: str
|
||||
path: Path
|
||||
|
||||
def python_package_name(self) -> str | None:
|
||||
if "/py/" in str(self.path) or str(self.path).endswith("/py"):
|
||||
return self.name.removesuffix("-py")
|
||||
return None
|
||||
|
||||
|
||||
def _get_pnpm_workspace_packages() -> list[Package]:
|
||||
"""Return directories for all workspace packages from pnpm list JSON output."""
|
||||
output = _run_and_capture(["pnpm", "list", "-r", "--depth=-1", "--json"])
|
||||
|
||||
data = cast(list[dict[str, Any]], json.loads(output))
|
||||
packages: list[Package] = [
|
||||
Package(name=data["name"], version=data["version"], path=Path(data["path"]))
|
||||
for data in data
|
||||
]
|
||||
return packages
|
||||
|
||||
|
||||
def _sync_package_version_with_pyproject(
|
||||
package_dir: Path, packages: dict[str, Package], js_package_name: str
|
||||
) -> None:
|
||||
"""Sync version from package.json to pyproject.toml.
|
||||
|
||||
Returns True if pyproject was changed, else False.
|
||||
"""
|
||||
pyproject_path = package_dir / "pyproject.toml"
|
||||
if not pyproject_path.exists():
|
||||
return
|
||||
|
||||
package_version = packages[js_package_name].version
|
||||
py_doc = tomlkit.parse(pyproject_path.read_text())
|
||||
|
||||
by_python_name = {
|
||||
pkg.python_package_name(): pkg
|
||||
for pkg in packages.values()
|
||||
if pkg.python_package_name()
|
||||
}
|
||||
|
||||
current_version = py_doc["project"]["version"]
|
||||
assert isinstance(current_version, str)
|
||||
|
||||
# update workspace dependency strings by replacing the first version after == or >=
|
||||
deps = py_doc["project"]["dependencies"] or []
|
||||
changed = False
|
||||
for i, dep in enumerate(deps):
|
||||
if not isinstance(dep, str):
|
||||
continue
|
||||
pkg = (cast(str, dep).split("==")[0]).split(">=")[0]
|
||||
if pkg not in by_python_name:
|
||||
continue
|
||||
target_version = by_python_name[pkg].version
|
||||
new_dep = re.sub(
|
||||
r"(==|>=)\s*([0-9A-Za-z_.+-]+)",
|
||||
lambda m: m.group(1) + target_version,
|
||||
dep,
|
||||
count=1,
|
||||
)
|
||||
if new_dep != dep:
|
||||
deps[i] = new_dep
|
||||
changed = True
|
||||
|
||||
if current_version != package_version:
|
||||
py_doc["project"]["version"] = package_version
|
||||
changed = True
|
||||
|
||||
if changed:
|
||||
pyproject_path.write_text(tomlkit.dumps(py_doc))
|
||||
click.echo(
|
||||
f"Updated {pyproject_path} version to {package_version} and synced dependency specs"
|
||||
)
|
||||
|
||||
|
||||
def lock_python_dependencies() -> None:
|
||||
"""Lock Python dependencies."""
|
||||
try:
|
||||
_run_command(["uv", "lock"])
|
||||
click.echo("Locked Python dependencies")
|
||||
except subprocess.CalledProcessError as e:
|
||||
click.echo(f"Warning: Failed to lock Python dependencies: {e}", err=True)
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli() -> None:
|
||||
"""Changeset-based version management for llama-cloud-services."""
|
||||
pass
|
||||
|
||||
|
||||
@cli.command()
|
||||
def version() -> None:
|
||||
"""Apply changeset versions, then sync versions for co-located JS/Py packages.
|
||||
|
||||
- Runs changesets to bump package.json versions.
|
||||
- Discovers all workspace packages via pnpm.
|
||||
- For any directory containing both package.json and pyproject.toml, and with
|
||||
package.json private: false, set pyproject [project].version to match the JS version.
|
||||
- If a pyproject is updated, run `uv sync` in that directory to update its lock file.
|
||||
"""
|
||||
# Ensure we're at the repo root
|
||||
os.chdir(Path(__file__).parent.parent)
|
||||
|
||||
# First, run changeset version to update all package.json files
|
||||
_run_command(["npx", "@changesets/cli", "version"])
|
||||
|
||||
# Enumerate workspace packages and perform syncs
|
||||
packages = _get_pnpm_workspace_packages()
|
||||
version_map = {pkg.name: pkg for pkg in packages}
|
||||
for pkg in packages:
|
||||
_sync_package_version_with_pyproject(pkg.path, version_map, pkg.name)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--tag", is_flag=True, help="Tag the packages after publishing")
|
||||
@click.option("--dry-run", is_flag=True, help="Dry run the publish")
|
||||
@click.option("--js/--no-js", default=True, help="Publish the js package")
|
||||
@click.option("--py/--no-py", default=True, help="Publish the py package")
|
||||
def publish(tag: bool, dry_run: bool, js: bool, py: bool) -> None:
|
||||
"""Publish all packages."""
|
||||
# move to the root
|
||||
os.chdir(Path(__file__).parent.parent)
|
||||
|
||||
if js:
|
||||
if not os.getenv("NPM_TOKEN"):
|
||||
click.echo("NPM_TOKEN is not set, skipping publish", err=True)
|
||||
raise click.Abort("No token set")
|
||||
if py:
|
||||
if not os.getenv("LLAMA_PARSE_PYPI_TOKEN"):
|
||||
click.echo("LLAMA_PARSE_PYPI_TOKEN is not set, skipping publish", err=True)
|
||||
raise click.Abort("No token set")
|
||||
|
||||
# not general script. Just checks each of the 2 packages to see if they need to be published.
|
||||
if js:
|
||||
maybe_publish_npm(dry_run)
|
||||
if py:
|
||||
maybe_publish_pypi(dry_run)
|
||||
|
||||
if tag:
|
||||
if dry_run:
|
||||
click.echo("Dry run, skipping tag. Would run:")
|
||||
click.echo(" npx @changesets/cli tag")
|
||||
click.echo(" git push --tags")
|
||||
else:
|
||||
# Let changesets create JS-related tags as usual
|
||||
_run_command(["npx", "@changesets/cli", "tag"])
|
||||
_run_command(["git", "push", "--tags"])
|
||||
|
||||
|
||||
def maybe_publish_npm(dry_run: bool) -> None:
|
||||
"""Publish the ts package if it needs to be published."""
|
||||
target_dir = Path("ts/llama_cloud_services")
|
||||
ts_path_package = target_dir / "package.json"
|
||||
package_json = json.loads(ts_path_package.read_text())
|
||||
version = package_json["version"]
|
||||
|
||||
# Check if this version is already published on npm
|
||||
result = subprocess.run(
|
||||
["npm", "view", "llama-cloud-services", "versions", "--json"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd=target_dir,
|
||||
)
|
||||
|
||||
published_versions = json.loads(result.stdout)
|
||||
if version in published_versions:
|
||||
click.echo(
|
||||
f"npm package llama-cloud-services@{version} already published, skipping"
|
||||
)
|
||||
return
|
||||
click.echo(f"Publishing npm package llama-cloud-services@{version}")
|
||||
# defer to the package.json publish script
|
||||
if dry_run:
|
||||
click.echo("Dry run, skipping publish. Would run:")
|
||||
click.echo(" pnpm run publish")
|
||||
return
|
||||
else:
|
||||
_run_command(["pnpm", "run", "build"], cwd=target_dir)
|
||||
_run_command(["pnpm", "publish"], cwd=target_dir)
|
||||
|
||||
|
||||
def maybe_publish_pypi(dry_run: bool) -> None:
|
||||
"""Publish the py packages if they need to be published."""
|
||||
for pyproject in list(Path("py").glob("*/pyproject.toml")) + [
|
||||
Path("py/pyproject.toml")
|
||||
]:
|
||||
name, version = current_version(pyproject)
|
||||
if is_published(name, version):
|
||||
click.echo(f"PyPI package {name}@{version} already published, skipping")
|
||||
continue
|
||||
click.echo(f"Publishing PyPI package {name}@{version}")
|
||||
|
||||
# Use different tokens for different packages
|
||||
env = os.environ.copy()
|
||||
token = os.environ["LLAMA_PARSE_PYPI_TOKEN"]
|
||||
env["UV_PUBLISH_TOKEN"] = token
|
||||
if dry_run:
|
||||
summary = (token[:3] + "***") if len(token) <= 6 else token[:6] + "****"
|
||||
click.echo(
|
||||
f"Dry run, skipping publish. Would run with publish token {summary}:"
|
||||
)
|
||||
click.echo(" uv build")
|
||||
click.echo(" uv publish")
|
||||
else:
|
||||
_run_command(["uv", "build"], cwd=pyproject.parent)
|
||||
_run_command(["uv", "publish"], cwd=pyproject.parent, env=env)
|
||||
|
||||
|
||||
def current_version(pyproject: Path) -> tuple[str, str]:
|
||||
"""Return (package_name, version_str) taken from the given pyproject.toml."""
|
||||
doc = tomlkit.parse(pyproject.read_text())
|
||||
name = doc["project"]["name"]
|
||||
version = str(Version(doc["project"]["version"])) # normalise
|
||||
return name, version
|
||||
|
||||
|
||||
def is_published(
|
||||
name: str, version: str, index_url: str = "https://pypi.org/pypi"
|
||||
) -> bool:
|
||||
"""
|
||||
True → `<name>==<version>` exists on the given index
|
||||
False → package missing *or* version missing
|
||||
"""
|
||||
url = f"{index_url.rstrip('/')}/{name}/json"
|
||||
try:
|
||||
data = json.load(urllib.request.urlopen(url))
|
||||
except urllib.error.HTTPError as e: # 404 → package not published at all
|
||||
if e.code == 404:
|
||||
return False
|
||||
raise # any other error should surface
|
||||
return version in data["releases"] # keys are version strings
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
@@ -0,0 +1,227 @@
|
||||
#!/usr/bin/env -S uv run --script
|
||||
# /// script
|
||||
# dependencies = ["click", "tomlkit"]
|
||||
# ///
|
||||
|
||||
import click
|
||||
import subprocess
|
||||
import sys
|
||||
import tomlkit
|
||||
from pathlib import Path
|
||||
import json
|
||||
|
||||
|
||||
def get_current_versions() -> tuple[str, str, str, str | None]:
|
||||
"""Get current versions from both pyproject.toml files and TS package.json."""
|
||||
# Read main pyproject.toml
|
||||
main_content = Path("py/pyproject.toml").read_text()
|
||||
main_doc = tomlkit.parse(main_content)
|
||||
main_version = main_doc["project"]["version"]
|
||||
|
||||
# Read llama_parse/pyproject.toml
|
||||
llama_parse_content = Path("py/llama_parse/pyproject.toml").read_text()
|
||||
llama_parse_doc = tomlkit.parse(llama_parse_content)
|
||||
llama_parse_version = llama_parse_doc["project"]["version"]
|
||||
# Find llama-cloud-services dependency in the dependencies list
|
||||
dependency_version = None
|
||||
for dep in llama_parse_doc["project"]["dependencies"]:
|
||||
if isinstance(dep, str) and dep.startswith("llama-cloud-services"):
|
||||
dependency_version = (
|
||||
dep.split("==")[1]
|
||||
if "==" in dep
|
||||
else dep.split(">=")[1]
|
||||
if ">=" in dep
|
||||
else None
|
||||
)
|
||||
break
|
||||
|
||||
# Read TypeScript package.json version via helper
|
||||
ts_version: str = get_ts_version()
|
||||
|
||||
return (
|
||||
str(main_version),
|
||||
str(llama_parse_version),
|
||||
str(dependency_version),
|
||||
str(ts_version) if ts_version is not None else None,
|
||||
)
|
||||
|
||||
|
||||
def validate_versions(
|
||||
main_version: str,
|
||||
llama_parse_version: str,
|
||||
dependency_version: str,
|
||||
) -> list[str]:
|
||||
"""Validate that versions are consistent and return warnings."""
|
||||
warnings = []
|
||||
|
||||
if main_version != llama_parse_version:
|
||||
warnings.append(
|
||||
f"Version mismatch: main={main_version}, llama_parse={llama_parse_version}"
|
||||
)
|
||||
|
||||
# Extract version from dependency string (e.g., ">=0.6.51" -> "0.6.51")
|
||||
if dependency_version and dependency_version.startswith(">="):
|
||||
dep_ver = dependency_version[2:]
|
||||
if dep_ver != main_version:
|
||||
warnings.append(
|
||||
f"Dependency version mismatch: dependency={dep_ver}, main={main_version}"
|
||||
)
|
||||
|
||||
return warnings
|
||||
|
||||
|
||||
def set_version(version: str) -> None:
|
||||
"""Set version across Python projects (no TS change)."""
|
||||
# Update main pyproject.toml
|
||||
main_content = Path("py/pyproject.toml").read_text()
|
||||
main_doc = tomlkit.parse(main_content)
|
||||
main_doc["project"]["version"] = version
|
||||
Path("py/pyproject.toml").write_text(tomlkit.dumps(main_doc))
|
||||
|
||||
# Update llama_parse/pyproject.toml
|
||||
llama_parse_content = Path("py/llama_parse/pyproject.toml").read_text()
|
||||
llama_parse_doc = tomlkit.parse(llama_parse_content)
|
||||
llama_parse_doc["project"]["version"] = version
|
||||
for dep_index, dep in enumerate(llama_parse_doc["project"]["dependencies"]):
|
||||
if isinstance(dep, str) and dep.startswith("llama-cloud-services"):
|
||||
llama_parse_doc["project"]["dependencies"][
|
||||
dep_index
|
||||
] = f"llama-cloud-services>={version}"
|
||||
break
|
||||
Path("py/llama_parse/pyproject.toml").write_text(tomlkit.dumps(llama_parse_doc))
|
||||
|
||||
click.echo(f"Updated Python versions to {version}")
|
||||
|
||||
|
||||
def get_ts_version() -> str:
|
||||
"""Read TypeScript package.json version (if present)."""
|
||||
ts_package_path = Path("ts/llama_cloud_services/package.json")
|
||||
package_data = json.loads(ts_package_path.read_text())
|
||||
data = package_data.get("version")
|
||||
if data is None:
|
||||
raise RuntimeError("TypeScript package.json version not found")
|
||||
return data
|
||||
|
||||
|
||||
def set_ts_version(version: str) -> None:
|
||||
"""Set TypeScript package.json version only."""
|
||||
ts_package_path = Path("ts/llama_cloud_services/package.json")
|
||||
package_data = json.loads(ts_package_path.read_text())
|
||||
package_data["version"] = version
|
||||
ts_package_path.write_text(json.dumps(package_data, indent=2) + "\n")
|
||||
click.echo(f"Updated TypeScript package.json version to {version}")
|
||||
|
||||
|
||||
def get_current_branch() -> str:
|
||||
"""Get the current git branch."""
|
||||
result = subprocess.run(
|
||||
["git", "branch", "--show-current"], capture_output=True, text=True, check=True
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def create_if_not_exists(version: str) -> None:
|
||||
"""Create a git tag and push it."""
|
||||
current_branch = get_current_branch()
|
||||
if current_branch != "main":
|
||||
click.echo(
|
||||
f"Error: Not on main branch (currently on {current_branch})", err=True
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
tag_name = f"v{version}" if version[0].isdigit() else version
|
||||
if not tag_exists(version):
|
||||
# Create tag
|
||||
subprocess.run(["git", "tag", tag_name], check=True)
|
||||
click.echo(f"Created tag {tag_name}")
|
||||
else:
|
||||
click.echo(f"Tag {tag_name} already exists")
|
||||
|
||||
|
||||
def tag_exists(version: str) -> bool:
|
||||
"""Check if a git tag exists."""
|
||||
tag_name = f"v{version}"
|
||||
result = subprocess.run(
|
||||
["git", "tag", "-l", tag_name], capture_output=True, text=True, check=True
|
||||
)
|
||||
return tag_name in result.stdout.strip()
|
||||
|
||||
|
||||
def push_tag(version: str) -> None:
|
||||
"""Push a git tag."""
|
||||
tag_name = f"v{version}"
|
||||
subprocess.run(["git", "push", "origin", tag_name], check=True)
|
||||
click.echo(f"Pushed tag {tag_name}")
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli() -> None:
|
||||
"""Version management for llama-cloud-services."""
|
||||
pass
|
||||
|
||||
|
||||
@cli.command()
|
||||
def get() -> None:
|
||||
"""Get current versions and show validation warnings."""
|
||||
(
|
||||
main_version,
|
||||
llama_parse_version,
|
||||
dependency_version,
|
||||
ts_version,
|
||||
) = get_current_versions()
|
||||
|
||||
click.echo("Current versions:")
|
||||
click.echo(f" llama-cloud-services: {main_version}")
|
||||
click.echo(f" llama-parse: {llama_parse_version}")
|
||||
click.echo(f" dependency reference: {dependency_version}")
|
||||
click.echo(f" typescript package: {ts_version}")
|
||||
|
||||
warnings = validate_versions(main_version, llama_parse_version, dependency_version)
|
||||
if warnings:
|
||||
click.echo("\nValidation warnings:")
|
||||
for warning in warnings:
|
||||
click.echo(f" ⚠️ {warning}")
|
||||
else:
|
||||
click.echo("\n✅ All versions are consistent")
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.argument("version")
|
||||
@click.option("--js", is_flag=True, help="Update TypeScript package.json only")
|
||||
def set(version: str, js: bool) -> None:
|
||||
"""Set version for Python, TypeScript, or both (default: Python only)."""
|
||||
|
||||
if js:
|
||||
set_ts_version(version)
|
||||
return
|
||||
else:
|
||||
set_version(version)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option(
|
||||
"--version", help="Version to tag (uses current version if not specified)"
|
||||
)
|
||||
@click.option(
|
||||
"--push",
|
||||
is_flag=True,
|
||||
help="Push the tag to the remote repository",
|
||||
)
|
||||
@click.option(
|
||||
"--js",
|
||||
is_flag=True,
|
||||
help="tag TypeScript package.json only",
|
||||
)
|
||||
def tag(version: str | None = None, push: bool = False, js: bool = False) -> None:
|
||||
"""Create and push a git tag for the current version."""
|
||||
if not version:
|
||||
main_version, _, _, js_version = get_current_versions()
|
||||
version = f"llama-cloud-services@{js_version}" if js else main_version
|
||||
|
||||
create_if_not_exists(version)
|
||||
if push:
|
||||
push_tag(version)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
@@ -9,12 +9,10 @@ test("LlamaIndex module resolution test", async (t) => {
|
||||
const index = new LlamaCloudIndex({
|
||||
name: "test-index",
|
||||
projectName: "Default",
|
||||
apiKey: process.env.LLAMA_CLOUD_API_KEY || "test-key",
|
||||
});
|
||||
const reader = new LlamaParseReader({
|
||||
resultType: "markdown",
|
||||
verbose: false,
|
||||
apiKey: process.env.LLAMA_CLOUD_API_KEY || "test-key",
|
||||
});
|
||||
ok(index !== undefined);
|
||||
ok(reader !== undefined);
|
||||
@@ -26,7 +24,6 @@ test("LlamaIndex module resolution test", async (t) => {
|
||||
const index = new mod.LlamaCloudIndex({
|
||||
name: "test-index",
|
||||
projectName: "Default",
|
||||
apiKey: process.env.LLAMA_CLOUD_API_KEY || "test-key",
|
||||
});
|
||||
ok(index !== undefined);
|
||||
});
|
||||
|
||||
@@ -1,29 +1,5 @@
|
||||
# llama-cloud-services
|
||||
|
||||
## 0.3.10
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- fee516d: Adding LlamaClassify among the available LlamaCloud services
|
||||
|
||||
## 0.3.9
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 5d4cabd: Add ImageNode support in TypeScript
|
||||
|
||||
## 0.3.8
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- 6e0f2f4: Agent data extraction citations can be undefined
|
||||
|
||||
## 0.3.7
|
||||
|
||||
### Patch Changes
|
||||
|
||||
- d028397: Update llama-cloud api version, and integrate with agent data deletion
|
||||
|
||||
## v0.1.0
|
||||
|
||||
First release for `llama-cloud-services`.
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
{
|
||||
"type": "module",
|
||||
"main": "./dist/index.cjs",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": "./dist/index.js",
|
||||
"private": true
|
||||
}
|
||||
@@ -1,10 +1,9 @@
|
||||
{
|
||||
"name": "llama-cloud-services",
|
||||
"version": "0.3.10",
|
||||
"version": "0.3.5",
|
||||
"type": "module",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
"get-openapi": "node ./scripts/get-openapi.js",
|
||||
"generate": "./node_modules/.bin/openapi-ts",
|
||||
"build": "pnpm run generate && bunchee",
|
||||
"dev": "bunchee --watch",
|
||||
@@ -14,8 +13,7 @@
|
||||
"test": "vitest run --testTimeout=60000",
|
||||
"test:watch": "vitest --watch",
|
||||
"test:ui": "vitest --ui",
|
||||
"test:coverage": "vitest --coverage",
|
||||
"release": "pnpm run build && pnpm publish"
|
||||
"test:coverage": "vitest --coverage"
|
||||
},
|
||||
"files": [
|
||||
"openapi.json",
|
||||
@@ -24,8 +22,7 @@
|
||||
"./reader",
|
||||
"./parse",
|
||||
"./beta/agent",
|
||||
"./extract",
|
||||
"./classify"
|
||||
"./extract"
|
||||
],
|
||||
"exports": {
|
||||
"./openapi.json": "./openapi.json",
|
||||
@@ -84,17 +81,6 @@
|
||||
},
|
||||
"default": "./extract/dist/index.js"
|
||||
},
|
||||
"./classify": {
|
||||
"require": {
|
||||
"types": "./classify/dist/index.d.cts",
|
||||
"default": "./classify/dist/index.cjs"
|
||||
},
|
||||
"import": {
|
||||
"types": "./classify/dist/index.d.ts",
|
||||
"default": "./classify/dist/index.js"
|
||||
},
|
||||
"default": "./classify/dist/index.js"
|
||||
},
|
||||
".": {
|
||||
"require": {
|
||||
"types": "./dist/index.d.cts",
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
import fs from 'fs';
|
||||
|
||||
async function downloadOpenApiSpec() {
|
||||
try {
|
||||
const response = await fetch('https://api.cloud.llamaindex.ai/api/openapi.json');
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
fs.writeFileSync('openapi.json', JSON.stringify(data, null, 2));
|
||||
console.log('Successfully downloaded openapi.json');
|
||||
} catch (error) {
|
||||
console.error('Error downloading OpenAPI spec:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
downloadOpenApiSpec();
|
||||
@@ -1,69 +0,0 @@
|
||||
import { createClient, createConfig, type Client } from "@hey-api/client-fetch";
|
||||
import {
|
||||
classify,
|
||||
type ClassifyParsingConfiguration,
|
||||
type ClassifierRule,
|
||||
type ClassifyJobResults,
|
||||
} from "./classify";
|
||||
import { getUrl } from "./utils";
|
||||
import { getEnv } from "@llamaindex/env";
|
||||
import { File } from "buffer";
|
||||
|
||||
export class LlamaClassify {
|
||||
private client: Client;
|
||||
|
||||
constructor(
|
||||
apiKey: string | undefined = undefined,
|
||||
baseUrl: string | undefined = undefined,
|
||||
region: string | undefined = undefined,
|
||||
) {
|
||||
const key = apiKey ?? getEnv("LLAMA_CLOUD_API_KEY");
|
||||
if (typeof key === "undefined") {
|
||||
throw new Error(
|
||||
"No API key provided and no API key found in environment. Please pass the API key or set `LLAMA_CLOUD_API_KEY` as an environment variable.",
|
||||
);
|
||||
}
|
||||
const url = getUrl(baseUrl, region);
|
||||
this.client = createClient(
|
||||
createConfig({
|
||||
baseUrl: url,
|
||||
headers: {
|
||||
Authorization: `Bearer ${key}`,
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async classify(
|
||||
rules: ClassifierRule[],
|
||||
parsingConfiguration: ClassifyParsingConfiguration,
|
||||
fileContents:
|
||||
| Buffer<ArrayBufferLike>[]
|
||||
| File[]
|
||||
| Uint8Array<ArrayBuffer>[]
|
||||
| string[]
|
||||
| undefined = undefined,
|
||||
filePaths: string[] | undefined = undefined,
|
||||
projectId: string | null = null,
|
||||
organizationId: string | null = null,
|
||||
pollingInterval: number = 1,
|
||||
maxPollingIterations: number = 1800,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<ClassifyJobResults> {
|
||||
const result = await classify(
|
||||
rules,
|
||||
parsingConfiguration,
|
||||
fileContents,
|
||||
filePaths,
|
||||
projectId,
|
||||
organizationId,
|
||||
this.client,
|
||||
pollingInterval,
|
||||
maxPollingIterations,
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -9,16 +9,10 @@ import { DEFAULT_PROJECT_NAME } from "@llamaindex/core/global";
|
||||
import type { QueryBundle } from "@llamaindex/core/query-engine";
|
||||
import { BaseRetriever } from "@llamaindex/core/retriever";
|
||||
import type { NodeWithScore } from "@llamaindex/core/schema";
|
||||
import { jsonToNode, ObjectType, ImageNode } from "@llamaindex/core/schema";
|
||||
import { jsonToNode, ObjectType } from "@llamaindex/core/schema";
|
||||
import { extractText } from "@llamaindex/core/utils";
|
||||
import type { ClientParams, CloudConstructorParams } from "./type.js";
|
||||
import { getPipelineId, getProjectId, initService } from "./utils.js";
|
||||
import {
|
||||
type PageScreenshotNodeWithScore,
|
||||
type PageFigureNodeWithScore,
|
||||
generateFilePageScreenshotPresignedUrlApiV1FilesIdPageScreenshotsPageIndexPresignedUrlPost,
|
||||
generateFilePageFigurePresignedUrlApiV1FilesIdPageFiguresPageIndexFigureNamePresignedUrlPost,
|
||||
} from "./api";
|
||||
import { getPipelineId, initService } from "./utils.js";
|
||||
|
||||
export type CloudRetrieveParams = Omit<
|
||||
RetrievalParams,
|
||||
@@ -49,95 +43,6 @@ export class LlamaCloudRetriever extends BaseRetriever {
|
||||
});
|
||||
}
|
||||
|
||||
private async fetchBase64FromPresignedUrl(url: string): Promise<string> {
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`Failed to fetch media from presigned URL: ${response.status} ${response.statusText}`,
|
||||
);
|
||||
}
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
return buffer.toString("base64");
|
||||
}
|
||||
|
||||
private async pageScreenshotNodesToNodeWithScore(
|
||||
nodes: PageScreenshotNodeWithScore[] | undefined,
|
||||
projectId: string,
|
||||
): Promise<NodeWithScore[]> {
|
||||
if (!nodes || nodes.length === 0) return [];
|
||||
|
||||
const results = await Promise.all(
|
||||
nodes.map(async (n) => {
|
||||
const { data: presigned } =
|
||||
await generateFilePageScreenshotPresignedUrlApiV1FilesIdPageScreenshotsPageIndexPresignedUrlPost(
|
||||
{
|
||||
throwOnError: true,
|
||||
path: {
|
||||
id: n.node.file_id,
|
||||
page_index: n.node.page_index,
|
||||
},
|
||||
query: {
|
||||
project_id: projectId,
|
||||
organization_id: this.organizationId ?? null,
|
||||
},
|
||||
},
|
||||
);
|
||||
const base64 = await this.fetchBase64FromPresignedUrl(presigned.url);
|
||||
const imageNode = new ImageNode({
|
||||
image: base64,
|
||||
metadata: {
|
||||
...(n.node.metadata ?? {}),
|
||||
file_id: n.node.file_id,
|
||||
page_index: n.node.page_index,
|
||||
},
|
||||
});
|
||||
return { node: imageNode, score: n.score } satisfies NodeWithScore;
|
||||
}),
|
||||
);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private async pageFigureNodesToNodeWithScore(
|
||||
nodes: PageFigureNodeWithScore[] | undefined,
|
||||
projectId: string,
|
||||
): Promise<NodeWithScore[]> {
|
||||
if (!nodes || nodes.length === 0) return [];
|
||||
|
||||
const results = await Promise.all(
|
||||
nodes.map(async (n) => {
|
||||
const { data: presigned } =
|
||||
await generateFilePageFigurePresignedUrlApiV1FilesIdPageFiguresPageIndexFigureNamePresignedUrlPost(
|
||||
{
|
||||
throwOnError: true,
|
||||
path: {
|
||||
id: n.node.file_id,
|
||||
page_index: n.node.page_index,
|
||||
figure_name: n.node.figure_name,
|
||||
},
|
||||
query: {
|
||||
project_id: projectId,
|
||||
organization_id: this.organizationId ?? null,
|
||||
},
|
||||
},
|
||||
);
|
||||
const base64 = await this.fetchBase64FromPresignedUrl(presigned.url);
|
||||
const imageNode = new ImageNode({
|
||||
image: base64,
|
||||
metadata: {
|
||||
...(n.node.metadata ?? {}),
|
||||
file_id: n.node.file_id,
|
||||
page_index: n.node.page_index,
|
||||
figure_name: n.node.figure_name,
|
||||
},
|
||||
});
|
||||
return { node: imageNode, score: n.score } satisfies NodeWithScore;
|
||||
}),
|
||||
);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
// LlamaCloud expects null values for filters, but LlamaIndexTS uses undefined for empty values
|
||||
// This function converts the undefined values to null
|
||||
private convertFilter(filters?: MetadataFilters): MetadataFilters | null {
|
||||
@@ -171,35 +76,6 @@ export class LlamaCloudRetriever extends BaseRetriever {
|
||||
}
|
||||
|
||||
async _retrieve(query: QueryBundle): Promise<NodeWithScore[]> {
|
||||
// Handle deprecated image retrieval flag
|
||||
const retrieveImageNodes = (this.retrieveParams as RetrievalParams)
|
||||
.retrieve_image_nodes;
|
||||
if (typeof retrieveImageNodes !== "undefined") {
|
||||
console.warn(
|
||||
"The `retrieve_image_nodes` parameter is deprecated. Use `retrieve_page_screenshot_nodes` and `retrieve_page_figure_nodes` instead.",
|
||||
);
|
||||
}
|
||||
|
||||
const retrievePageScreenshotNodes = (this.retrieveParams as RetrievalParams)
|
||||
.retrieve_page_screenshot_nodes;
|
||||
const retrievePageFigureNodes = (this.retrieveParams as RetrievalParams)
|
||||
.retrieve_page_figure_nodes;
|
||||
|
||||
if (retrieveImageNodes) {
|
||||
if (
|
||||
retrievePageScreenshotNodes === false ||
|
||||
retrievePageFigureNodes === false
|
||||
) {
|
||||
throw new Error(
|
||||
"If `retrieve_image_nodes` is set to true, both `retrieve_page_screenshot_nodes` and `retrieve_page_figure_nodes` must also be set to true or omitted.",
|
||||
);
|
||||
}
|
||||
(this.retrieveParams as RetrievalParams).retrieve_page_screenshot_nodes =
|
||||
true;
|
||||
(this.retrieveParams as RetrievalParams).retrieve_page_figure_nodes =
|
||||
true;
|
||||
}
|
||||
|
||||
const pipelineId = await getPipelineId(
|
||||
this.pipelineName,
|
||||
this.projectName,
|
||||
@@ -222,34 +98,6 @@ export class LlamaCloudRetriever extends BaseRetriever {
|
||||
},
|
||||
});
|
||||
|
||||
const textNodes = this.resultNodesToNodeWithScore(results.retrieval_nodes);
|
||||
|
||||
const needScreenshots = (this.retrieveParams as RetrievalParams)
|
||||
.retrieve_page_screenshot_nodes;
|
||||
const needFigures = (this.retrieveParams as RetrievalParams)
|
||||
.retrieve_page_figure_nodes;
|
||||
|
||||
if (!needScreenshots && !needFigures) {
|
||||
return textNodes;
|
||||
}
|
||||
|
||||
const projectId = await getProjectId(this.projectName, this.organizationId);
|
||||
|
||||
const [screenshotNodes, figureNodes] = await Promise.all([
|
||||
needScreenshots
|
||||
? this.pageScreenshotNodesToNodeWithScore(
|
||||
results.image_nodes,
|
||||
projectId,
|
||||
)
|
||||
: Promise.resolve([] as NodeWithScore[]),
|
||||
needFigures
|
||||
? this.pageFigureNodesToNodeWithScore(
|
||||
results.page_figure_nodes,
|
||||
projectId,
|
||||
)
|
||||
: Promise.resolve([] as NodeWithScore[]),
|
||||
]);
|
||||
|
||||
return [...textNodes, ...screenshotNodes, ...figureNodes];
|
||||
return this.resultNodesToNodeWithScore(results.retrieval_nodes);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,25 @@ import * as extract from "./extract";
|
||||
import type { ExtractAgent, ExtractConfig } from "./extract";
|
||||
import { getEnv } from "@llamaindex/env";
|
||||
import type { ExtractResult } from "./type";
|
||||
import { getUrl } from "./utils";
|
||||
|
||||
const URLS = {
|
||||
us: "https://api.cloud.llamaindex.ai",
|
||||
eu: "https://api.cloud.eu.llamaindex.ai",
|
||||
"us-staging": "https://api.staging.llamaindex.ai",
|
||||
} as const;
|
||||
|
||||
function getUrl(baseUrl: string | undefined, region: string | undefined) {
|
||||
if (typeof baseUrl != "undefined") {
|
||||
return baseUrl;
|
||||
}
|
||||
if (typeof region === "undefined") {
|
||||
return URLS["us"];
|
||||
} else if (region === "us" || region === "eu" || region === "us-staging") {
|
||||
return URLS[region];
|
||||
} else {
|
||||
throw new Error(`Unsupported region: ${region}`);
|
||||
}
|
||||
}
|
||||
|
||||
export class LlamaExtractAgent {
|
||||
private agent: ExtractAgent;
|
||||
|
||||
@@ -4,7 +4,6 @@ import {
|
||||
aggregateAgentDataApiV1BetaAgentDataAggregatePost,
|
||||
createAgentDataApiV1BetaAgentDataPost,
|
||||
deleteAgentDataApiV1BetaAgentDataItemIdDelete,
|
||||
deleteAgentDataByQueryApiV1BetaAgentDataDeletePost,
|
||||
getAgentDataApiV1BetaAgentDataItemIdGet,
|
||||
searchAgentDataApiV1BetaAgentDataSearchPost,
|
||||
updateAgentDataApiV1BetaAgentDataItemIdPut,
|
||||
@@ -13,7 +12,6 @@ import {
|
||||
} from "../../client";
|
||||
import type {
|
||||
AggregateAgentDataOptions,
|
||||
DeleteAgentDataOptions,
|
||||
SearchAgentDataOptions,
|
||||
TypedAgentData,
|
||||
TypedAgentDataItems,
|
||||
@@ -27,23 +25,20 @@ import type {
|
||||
export class AgentClient<T = unknown> {
|
||||
private client: ReturnType<typeof createClient>;
|
||||
private collection: string;
|
||||
private deploymentName: string;
|
||||
private agentUrlId: string;
|
||||
|
||||
constructor({
|
||||
client = defaultClient,
|
||||
collection = "default",
|
||||
deploymentName = "_public",
|
||||
agentUrlId,
|
||||
agentUrlId = "_public",
|
||||
}: {
|
||||
client?: ReturnType<typeof createClient>;
|
||||
collection?: string;
|
||||
deploymentName?: string;
|
||||
// deprecated, use deploymentName instead
|
||||
agentUrlId?: string;
|
||||
}) {
|
||||
this.client = client;
|
||||
this.collection = collection;
|
||||
this.deploymentName = agentUrlId || deploymentName;
|
||||
this.agentUrlId = agentUrlId;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -53,7 +48,7 @@ export class AgentClient<T = unknown> {
|
||||
const response = await createAgentDataApiV1BetaAgentDataPost({
|
||||
throwOnError: true,
|
||||
body: {
|
||||
deployment_name: this.deploymentName,
|
||||
agent_slug: this.agentUrlId,
|
||||
collection: this.collection,
|
||||
data: data as Record<string, unknown>,
|
||||
},
|
||||
@@ -114,24 +109,6 @@ export class AgentClient<T = unknown> {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete all matching agent data, returns the total number of deleted items
|
||||
*/
|
||||
async delete(options: DeleteAgentDataOptions): Promise<number> {
|
||||
const response = await deleteAgentDataByQueryApiV1BetaAgentDataDeletePost({
|
||||
throwOnError: true,
|
||||
body: {
|
||||
deployment_name: this.deploymentName,
|
||||
...(this.collection !== undefined && {
|
||||
collection: this.collection,
|
||||
}),
|
||||
...(options.filter !== undefined && { filter: options.filter }),
|
||||
},
|
||||
client: this.client,
|
||||
});
|
||||
return response.data.deleted_count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Search agent data
|
||||
*/
|
||||
@@ -141,7 +118,7 @@ export class AgentClient<T = unknown> {
|
||||
const response = await searchAgentDataApiV1BetaAgentDataSearchPost({
|
||||
throwOnError: true,
|
||||
body: {
|
||||
deployment_name: this.deploymentName,
|
||||
agent_slug: this.agentUrlId,
|
||||
...(this.collection !== undefined && {
|
||||
collection: this.collection,
|
||||
}),
|
||||
@@ -188,7 +165,7 @@ export class AgentClient<T = unknown> {
|
||||
const response = await aggregateAgentDataApiV1BetaAgentDataAggregatePost({
|
||||
throwOnError: true,
|
||||
body: {
|
||||
deployment_name: this.deploymentName,
|
||||
agent_slug: this.agentUrlId,
|
||||
...(this.collection !== undefined && {
|
||||
collection: this.collection,
|
||||
}),
|
||||
@@ -232,7 +209,7 @@ export class AgentClient<T = unknown> {
|
||||
private transformResponse(data: AgentData): TypedAgentData<T> {
|
||||
const result: TypedAgentData<T> = {
|
||||
id: data.id!,
|
||||
deploymentName: data.deployment_name,
|
||||
agentUrlId: data.agent_slug,
|
||||
data: data.data as T,
|
||||
createdAt: new Date(data.created_at!),
|
||||
updatedAt: new Date(data.updated_at!),
|
||||
@@ -273,10 +250,10 @@ export interface AgentDataClientOptions {
|
||||
/** Base URL for the client */
|
||||
/** Base URL of the llama cloud api */
|
||||
baseUrl?: string;
|
||||
/** If running in an agent runtime, optionally provide the window url to infer the deployment name */
|
||||
/** If running in an agent runtime, optionally provide the window url to infer the agent url id */
|
||||
windowUrl?: string;
|
||||
/** Deployment name for the client, if not provided, it will be inferred from the window url, or fall back to "default" */
|
||||
deploymentName?: string;
|
||||
/** Agent URL ID for the client, if not provided, it will be inferred from the window url, or fall back to "default" */
|
||||
agentUrlId?: string;
|
||||
/** Collection name for the client, defaults to "default" */
|
||||
collection?: string;
|
||||
}
|
||||
@@ -290,25 +267,22 @@ export function createAgentDataClient<T = unknown>({
|
||||
client = defaultClient,
|
||||
windowUrl,
|
||||
env,
|
||||
deploymentName,
|
||||
agentUrlId,
|
||||
collection = "default",
|
||||
}: {
|
||||
client?: ReturnType<typeof createClient>;
|
||||
windowUrl?: string;
|
||||
env?: Record<string, string>;
|
||||
deploymentName?: string;
|
||||
// deprecated, use deploymentName instead
|
||||
agentUrlId?: string;
|
||||
collection?: string;
|
||||
} = {}): AgentClient<T> {
|
||||
if (env && !deploymentName) {
|
||||
deploymentName =
|
||||
if (env && !agentUrlId) {
|
||||
agentUrlId =
|
||||
env.LLAMA_DEPLOY_DEPLOYMENT_NAME ||
|
||||
env.NEXT_PUBLIC_LLAMA_DEPLOY_DEPLOYMENT_NAME ||
|
||||
env.VITE_LLAMA_DEPLOY_DEPLOYMENT_NAME;
|
||||
}
|
||||
if (windowUrl && !deploymentName) {
|
||||
if (windowUrl && !agentUrlId) {
|
||||
try {
|
||||
const url = new URL(windowUrl);
|
||||
const path = url.pathname;
|
||||
@@ -317,18 +291,17 @@ export function createAgentDataClient<T = unknown>({
|
||||
url.hostname.includes("127.0.0.1");
|
||||
if (path.startsWith("/deployments/") && !isLocalhost) {
|
||||
// /deployments/<agent-url-id>/ui/ -> ["", "deployments", "<agent-url-id>", "ui"]
|
||||
deploymentName = path.split("/")[2];
|
||||
agentUrlId = path.split("/")[2];
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
"Failed to infer deployment name from window url, falling back to default",
|
||||
"Failed to infer agent url id from window url, falling back to default",
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return new AgentClient({
|
||||
...(deploymentName && { deploymentName }),
|
||||
...(agentUrlId && { agentUrlId }),
|
||||
collection,
|
||||
client,
|
||||
|
||||
@@ -38,7 +38,7 @@ export interface ExtractedFieldMetadata {
|
||||
confidence?: number;
|
||||
/** The confidence score for the field based on the extracted text only */
|
||||
extraction_confidence?: number;
|
||||
citation?: FieldCitation[];
|
||||
citation: FieldCitation[];
|
||||
}
|
||||
|
||||
export interface FieldCitation {
|
||||
@@ -87,8 +87,8 @@ export interface ExtractedData<T = unknown> {
|
||||
export interface TypedAgentData<T = unknown> {
|
||||
/** The unique ID of the agent data record. */
|
||||
id: string;
|
||||
/** The deployment name of the agent that created the data. */
|
||||
deploymentName: string;
|
||||
/** The ID of the agent that created the data. */
|
||||
agentUrlId: string;
|
||||
/** The collection of the agent data. */
|
||||
collection?: string;
|
||||
/** The data of the agent data. Usually an ExtractedData<SomeOtherType> */
|
||||
@@ -127,14 +127,6 @@ export interface SearchAgentDataOptions {
|
||||
includeTotal?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for deleting agent data
|
||||
*/
|
||||
export interface DeleteAgentDataOptions {
|
||||
/** Filter options for the deletion. */
|
||||
filter?: Record<string, FilterOperation>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for aggregating agent data
|
||||
*/
|
||||
|
||||
@@ -1,289 +0,0 @@
|
||||
import type {
|
||||
Options,
|
||||
CreateClassifyJobApiV1ClassifierJobsPostData,
|
||||
ClassifyJobCreate,
|
||||
ClassifierRule,
|
||||
ClassifyParsingConfiguration,
|
||||
GetClassifyJobApiV1ClassifierJobsClassifyJobIdGetData,
|
||||
GetClassificationJobResultsApiV1ClassifierJobsClassifyJobIdResultsGetData,
|
||||
ClassifyJobResults,
|
||||
} from "./api";
|
||||
import {
|
||||
StatusEnum,
|
||||
createClassifyJobApiV1ClassifierJobsPost,
|
||||
getClassifyJobApiV1ClassifierJobsClassifyJobIdGet,
|
||||
getClassificationJobResultsApiV1ClassifierJobsClassifyJobIdResultsGet,
|
||||
} from "./api";
|
||||
import type { Client } from "@hey-api/client-fetch";
|
||||
import { sleep } from "./utils";
|
||||
import { uploadFile } from "./fileUpload";
|
||||
import { File } from "buffer";
|
||||
|
||||
async function createClassifyJob(
|
||||
fileIds: string[],
|
||||
rules: ClassifierRule[],
|
||||
parsingConfiguration: ClassifyParsingConfiguration,
|
||||
organizationId: null | string,
|
||||
projectId: null | string,
|
||||
client: Client | undefined,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<string> {
|
||||
const rawData = {
|
||||
file_ids: fileIds,
|
||||
rules: rules,
|
||||
parsing_configuration: parsingConfiguration,
|
||||
} as ClassifyJobCreate;
|
||||
const data = {
|
||||
body: rawData,
|
||||
query: {
|
||||
project_id: projectId,
|
||||
organization_id: organizationId,
|
||||
},
|
||||
} as CreateClassifyJobApiV1ClassifierJobsPostData;
|
||||
const options = data as Options<CreateClassifyJobApiV1ClassifierJobsPostData>;
|
||||
if (typeof client != "undefined") {
|
||||
options.client = client;
|
||||
}
|
||||
let retries = 0;
|
||||
while (true) {
|
||||
if (retries > maxRetriesOnError) {
|
||||
throw new Error(
|
||||
"Error while creating the classify job: Exceeded maximum number of retries, the API keeps returning errors.",
|
||||
);
|
||||
}
|
||||
const response = await createClassifyJobApiV1ClassifierJobsPost(options);
|
||||
if (!response.response.ok) {
|
||||
if ("error" in response) {
|
||||
console.log(
|
||||
`An error occurred while creating the classification job.\nDetails:\n\n${JSON.stringify(
|
||||
response.error,
|
||||
)}\n\nRetrying...`,
|
||||
);
|
||||
}
|
||||
retries++;
|
||||
await sleep(retryInterval * 1000);
|
||||
} else {
|
||||
if (typeof response.data != "undefined") {
|
||||
return response.data.id;
|
||||
} else {
|
||||
throw new Error(
|
||||
"Error while creating the classify job: the job creation succeeded but no data where returned",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function pollForJobCompletion(
|
||||
jobId: string,
|
||||
interval: number = 1,
|
||||
maxIterations: number = 1800,
|
||||
client: Client | undefined = undefined,
|
||||
): Promise<boolean> {
|
||||
let status: StatusEnum | undefined = undefined;
|
||||
const jobData = {
|
||||
path: { classify_job_id: jobId },
|
||||
} as GetClassifyJobApiV1ClassifierJobsClassifyJobIdGetData;
|
||||
const jobOptions =
|
||||
jobData as Options<GetClassifyJobApiV1ClassifierJobsClassifyJobIdGetData>;
|
||||
if (typeof client != "undefined") {
|
||||
jobOptions.client = client;
|
||||
}
|
||||
let numIterations: number = 0;
|
||||
while (true) {
|
||||
if (numIterations > maxIterations) {
|
||||
return false;
|
||||
}
|
||||
const response =
|
||||
await getClassifyJobApiV1ClassifierJobsClassifyJobIdGet(jobOptions);
|
||||
if (!response.response.ok) {
|
||||
numIterations++;
|
||||
}
|
||||
if (typeof response.data != "undefined") {
|
||||
status = response.data.status as StatusEnum;
|
||||
if (status == StatusEnum.CANCELLED || status == StatusEnum.ERROR) {
|
||||
throw new Error("There was an error during the classification job.");
|
||||
} else if (status == StatusEnum.SUCCESS) {
|
||||
return true;
|
||||
} else {
|
||||
numIterations++;
|
||||
await sleep(interval * 1000);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function getJobResult(
|
||||
jobId: string,
|
||||
client: Client | undefined = undefined,
|
||||
projectId: string | null = null,
|
||||
organizationId: string | null = null,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<ClassifyJobResults> {
|
||||
const jobData = {
|
||||
path: { classify_job_id: jobId },
|
||||
query: { organization_id: organizationId, project_id: projectId },
|
||||
} as GetClassificationJobResultsApiV1ClassifierJobsClassifyJobIdResultsGetData;
|
||||
const jobOptions =
|
||||
jobData as Options<GetClassificationJobResultsApiV1ClassifierJobsClassifyJobIdResultsGetData>;
|
||||
if (typeof client != "undefined") {
|
||||
jobOptions.client = client;
|
||||
}
|
||||
let retries: number = 0;
|
||||
while (true) {
|
||||
if (retries > maxRetriesOnError) {
|
||||
throw new Error(
|
||||
"Error while getting the result of the classification job: Exceeded maximum number of retries, the API keeps returning errors.",
|
||||
);
|
||||
}
|
||||
const response =
|
||||
await getClassificationJobResultsApiV1ClassifierJobsClassifyJobIdResultsGet(
|
||||
jobOptions,
|
||||
);
|
||||
if (!response.response.ok) {
|
||||
if ("error" in response) {
|
||||
console.log(
|
||||
"An error occurred: ",
|
||||
JSON.stringify(response.error),
|
||||
"\nRetrying...",
|
||||
);
|
||||
}
|
||||
retries++;
|
||||
await sleep(retryInterval * 1000);
|
||||
}
|
||||
if (typeof response.data != "undefined") {
|
||||
return response.data as ClassifyJobResults;
|
||||
} else {
|
||||
throw new Error(
|
||||
"Error while retrieving results for the classify job: the result was successfully obtained but no data were returned",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function classify(
|
||||
rules: ClassifierRule[],
|
||||
parsingConfiguration: ClassifyParsingConfiguration,
|
||||
fileContents:
|
||||
| Buffer<ArrayBufferLike>[]
|
||||
| File[]
|
||||
| Uint8Array<ArrayBuffer>[]
|
||||
| string[]
|
||||
| undefined = undefined,
|
||||
filePaths: string[] | undefined = undefined,
|
||||
projectId: string | null = null,
|
||||
organizationId: string | null = null,
|
||||
client: Client | undefined = undefined,
|
||||
pollingInterval: number = 1,
|
||||
maxPollingIterations: number = 1800,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<ClassifyJobResults> {
|
||||
const fileIds: string[] = [];
|
||||
if (!filePaths && !fileContents) {
|
||||
throw new Error(
|
||||
"One between filePath and fileContent needs to be provided",
|
||||
);
|
||||
}
|
||||
|
||||
if (filePaths) {
|
||||
const uploadPromises = filePaths.map(async (name) => {
|
||||
try {
|
||||
const fileId = await uploadFile(
|
||||
name,
|
||||
undefined,
|
||||
undefined,
|
||||
projectId,
|
||||
organizationId,
|
||||
client,
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
);
|
||||
if (fileId) {
|
||||
return fileId;
|
||||
} else {
|
||||
console.error(`Unable to upload ${name}, skipping...`);
|
||||
return null;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error uploading ${name}:`, error);
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
const results = await Promise.all(uploadPromises);
|
||||
fileIds.push(...results.filter((id) => id !== null));
|
||||
}
|
||||
|
||||
if (fileContents) {
|
||||
const uploadPromises = fileContents.map(async (content) => {
|
||||
try {
|
||||
const fileId = await uploadFile(
|
||||
undefined,
|
||||
content,
|
||||
undefined,
|
||||
projectId,
|
||||
organizationId,
|
||||
client,
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
);
|
||||
if (fileId) {
|
||||
return fileId;
|
||||
} else {
|
||||
console.error(`Unable to upload file (content), skipping...`);
|
||||
return null;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error uploading file (content):`, error);
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
const results = await Promise.all(uploadPromises);
|
||||
fileIds.push(...results.filter((id) => id !== null));
|
||||
}
|
||||
|
||||
if (fileIds.length == 0) {
|
||||
throw new Error(
|
||||
"None of the provided files was successfully uploaded, it is not possible to create a classification job.",
|
||||
);
|
||||
}
|
||||
|
||||
const jobId = await createClassifyJob(
|
||||
fileIds,
|
||||
rules,
|
||||
parsingConfiguration,
|
||||
organizationId,
|
||||
projectId,
|
||||
client,
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
);
|
||||
const success = await pollForJobCompletion(
|
||||
jobId,
|
||||
pollingInterval,
|
||||
maxPollingIterations,
|
||||
client,
|
||||
);
|
||||
if (!success) {
|
||||
throw new Error("Your job is taking longer than 10 minutes, timing out...");
|
||||
} else {
|
||||
return (await getJobResult(
|
||||
jobId,
|
||||
client,
|
||||
projectId,
|
||||
organizationId,
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
)) as ClassifyJobResults;
|
||||
}
|
||||
}
|
||||
|
||||
export {
|
||||
type ClassifierRule,
|
||||
type ClassifyJobResults,
|
||||
type ClassifyParsingConfiguration,
|
||||
};
|
||||