mirror of
https://github.com/run-llama/llama_cloud_services.git
synced 2026-07-21 12:05:23 -04:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3fd5127252 | |||
| 43e74c36eb | |||
| e021446901 |
@@ -1,8 +0,0 @@
|
||||
# Changesets
|
||||
|
||||
Hello and welcome! This folder has been automatically generated by `@changesets/cli`, a build tool that works
|
||||
with multi-package repos, or single-package repos to help you version and publish your code. You can
|
||||
find the full documentation for it [in our repository](https://github.com/changesets/changesets)
|
||||
|
||||
We have a quick list of common questions to get you started engaging with this project in
|
||||
[our documentation](https://github.com/changesets/changesets/blob/main/docs/common-questions.md)
|
||||
@@ -1,11 +0,0 @@
|
||||
{
|
||||
"$schema": "https://unpkg.com/@changesets/config@3.1.1/schema.json",
|
||||
"changelog": "@changesets/cli/changelog",
|
||||
"commit": false,
|
||||
"fixed": [],
|
||||
"linked": [],
|
||||
"access": "restricted",
|
||||
"baseBranch": "main",
|
||||
"updateInternalDependencies": "patch",
|
||||
"ignore": []
|
||||
}
|
||||
@@ -6,11 +6,8 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "py/**"
|
||||
pull_request:
|
||||
paths:
|
||||
- "py/**"
|
||||
|
||||
env:
|
||||
UV_VERSION: "0.7.20"
|
||||
|
||||
|
||||
@@ -1,13 +1,5 @@
|
||||
name: Build Package - TypeScript
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "ts/**"
|
||||
pull_request:
|
||||
paths:
|
||||
- "ts/**"
|
||||
on: [pull_request]
|
||||
|
||||
jobs:
|
||||
pre_release:
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
name: Claude Code
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
pull_request_review_comment:
|
||||
types: [created]
|
||||
issues:
|
||||
types: [opened, assigned]
|
||||
pull_request_review:
|
||||
types: [submitted]
|
||||
|
||||
jobs:
|
||||
claude:
|
||||
if: |
|
||||
(github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
|
||||
(github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
issues: read
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Check repository access
|
||||
id: check-access
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
# Get the user who triggered the event
|
||||
case "${{ github.event_name }}" in
|
||||
"issue_comment")
|
||||
USER="${{ github.event.comment.user.login }}"
|
||||
;;
|
||||
"pull_request_review_comment")
|
||||
USER="${{ github.event.comment.user.login }}"
|
||||
;;
|
||||
"pull_request_review")
|
||||
USER="${{ github.event.review.user.login }}"
|
||||
;;
|
||||
"issues")
|
||||
USER="${{ github.event.issue.user.login }}"
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "Checking repository access for user: $USER"
|
||||
|
||||
# Check if user has write access to the repository
|
||||
REPO="${{ github.repository }}"
|
||||
if gh api repos/$REPO/collaborators/$USER/permission --jq '.permission' | grep -E "(admin|write)" > /dev/null 2>&1; then
|
||||
echo "User $USER has write access to the repository"
|
||||
echo "authorized=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "User $USER does not have write access to the repository"
|
||||
echo "authorized=false" >> $GITHUB_OUTPUT
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Checkout repository
|
||||
if: steps.check-access.outputs.authorized == 'true'
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Run Claude Code
|
||||
if: steps.check-access.outputs.authorized == 'true'
|
||||
id: claude
|
||||
uses: anthropics/claude-code-action@beta
|
||||
with:
|
||||
anthropic_api_key: ${{ secrets.ANTHROPIC_GITHUB_API_KEY }}
|
||||
|
||||
# Optional: Specify model (defaults to Claude Sonnet 4, uncomment for Claude Opus 4)
|
||||
# model: "claude-opus-4-20250514"
|
||||
|
||||
# Optional: Customize the trigger phrase (default: @claude)
|
||||
# trigger_phrase: "/claude"
|
||||
|
||||
# Optional: Trigger when specific user is assigned to an issue
|
||||
# assignee_trigger: "claude-bot"
|
||||
|
||||
# Optional: Allow Claude to run specific commands
|
||||
# Allow bash commands to be run, for things like running tests, linting, etc.
|
||||
allowed_tools: "Bash(rg:*),Bash(find:*),Bash(grep:*),Bash(pnpm:*),Bash(npm:*),Bash(uv:*),Bash(pip:*),Bash(pipx:*),Bash(make:*),Bash(cd:*),WebFetch"
|
||||
|
||||
# Optional: Add custom instructions for Claude to customize its behavior for your project
|
||||
# custom_instructions: |
|
||||
# Follow our coding standards
|
||||
# Ensure all new code has tests
|
||||
# Use TypeScript for new files
|
||||
|
||||
# Optional: Custom environment variables for Claude
|
||||
# claude_env: |
|
||||
# NODE_ENV: test
|
||||
@@ -4,11 +4,9 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "ts/**"
|
||||
pull_request:
|
||||
paths:
|
||||
- "ts/**"
|
||||
branches:
|
||||
- main
|
||||
|
||||
env:
|
||||
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
|
||||
@@ -28,6 +26,7 @@ jobs:
|
||||
with:
|
||||
node-version-file: "ts/llama_cloud_services/.nvmrc"
|
||||
- name: Install dependencies
|
||||
working-directory: ts/llama_cloud_services/
|
||||
run: pnpm install --no-frozen-lockfile
|
||||
- name: Run lint
|
||||
working-directory: ts/llama_cloud_services/
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
name: Publish - Changeset
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
concurrency: ${{ github.workflow }}-${{ github.ref }}
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Publish
|
||||
runs-on: ubuntu-latest
|
||||
# Only run if this is a merge commit and the PR was authored by github-actions[bot]
|
||||
if: ${{ github.event.head_commit.message != null && startsWith(github.event.head_commit.message, 'Merge pull request') && contains(github.event.head_commit.message, '/changeset-release/main') }}
|
||||
steps:
|
||||
- name: Checkout Repo
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- uses: pnpm/action-setup@v3
|
||||
with:
|
||||
version: 10
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v3
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: "ts/llama_cloud_services/.nvmrc"
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --no-frozen-lockfile
|
||||
|
||||
- name: Add auth token to .npmrc file
|
||||
run: |
|
||||
cat << EOF >> ".npmrc"
|
||||
//registry.npmjs.org/:_authToken=$NPM_TOKEN
|
||||
EOF
|
||||
env:
|
||||
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
|
||||
- name: Publish packages based on changesets
|
||||
run: |
|
||||
pnpm run release
|
||||
env:
|
||||
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
PYPI_TOKEN: ${{ secrets.LLAMA_PARSE_PYPI_TOKEN }}
|
||||
@@ -22,12 +22,9 @@ jobs:
|
||||
node-version-file: "ts/llama_cloud_services/.nvmrc"
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: ts/llama_cloud_services
|
||||
run: pnpm install --no-frozen-lockfile
|
||||
|
||||
- name: Run Build
|
||||
working-directory: ts/llama_cloud_services/
|
||||
run: pnpm build
|
||||
|
||||
- name: Build tarball
|
||||
run: |
|
||||
pnpm pack
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
name: Test end-to-end - Python
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "py/**"
|
||||
|
||||
env:
|
||||
UV_VERSION: "0.7.20"
|
||||
LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
|
||||
|
||||
jobs:
|
||||
test_e2e:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
# You can use PyPy versions in python-version.
|
||||
# For example, pypy-2.7 and pypy-3.8
|
||||
matrix:
|
||||
python-version: ["3.12"]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
version: ${{ env.UV_VERSION }}
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install ${{ matrix.python-version }} && uv python pin ${{ matrix.python-version }}
|
||||
|
||||
- name: Run Tests
|
||||
working-directory: py
|
||||
run: uv run pytest unit_tests/ tests/ -v
|
||||
|
||||
- name: Remove virtual environment
|
||||
working-directory: py
|
||||
run: rm -rf .venv/
|
||||
@@ -4,14 +4,11 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "py/**"
|
||||
pull_request:
|
||||
paths:
|
||||
- "py/**"
|
||||
|
||||
env:
|
||||
UV_VERSION: "0.7.20"
|
||||
LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
@@ -35,7 +32,7 @@ jobs:
|
||||
|
||||
- name: Run Tests
|
||||
working-directory: py
|
||||
run: uv run pytest unit_tests/ -v
|
||||
run: uv run -- pytest tests/**/test_*.py
|
||||
|
||||
- name: Remove virtual environment
|
||||
working-directory: py
|
||||
|
||||
@@ -4,11 +4,9 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "ts/**"
|
||||
pull_request:
|
||||
paths:
|
||||
- "ts/**"
|
||||
branches:
|
||||
- main
|
||||
|
||||
env:
|
||||
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
|
||||
@@ -17,8 +15,7 @@ env:
|
||||
LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Test - TypeScript
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -30,13 +27,8 @@ jobs:
|
||||
with:
|
||||
node-version-file: "ts/llama_cloud_services/.nvmrc"
|
||||
- name: Install dependencies
|
||||
working-directory: ts/llama_cloud_services/
|
||||
run: pnpm install --no-frozen-lockfile
|
||||
- name: Run Build
|
||||
- name: Run tests
|
||||
working-directory: ts/llama_cloud_services/
|
||||
run: pnpm build
|
||||
- name: Run Tests
|
||||
working-directory: ts/llama_cloud_services/
|
||||
run: pnpm test
|
||||
- name: Run e2e tests
|
||||
working-directory: ts/e2e-tests/
|
||||
run: pnpm test
|
||||
run: pnpm test --run
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
name: Version Bump - Changeset
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
concurrency: ${{ github.workflow }}-${{ github.ref }}
|
||||
|
||||
jobs:
|
||||
version:
|
||||
name: Version Bump
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout Repo
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Need to fetch full history for changesets
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: pnpm/action-setup@v3
|
||||
with:
|
||||
version: 10
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v3
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: "ts/cute_cats_terminal/.nvmrc"
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --no-frozen-lockfile
|
||||
|
||||
- name: Create Release Pull Request
|
||||
id: changesets
|
||||
uses: changesets/action@v1
|
||||
with:
|
||||
version: pnpm run version
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -15,7 +15,6 @@ repos:
|
||||
- id: end-of-file-fixer
|
||||
- id: mixed-line-ending
|
||||
- id: trailing-whitespace
|
||||
exclude: ^ts/llama_cloud_services/src/client/
|
||||
- repo: https://github.com/charliermarsh/ruff-pre-commit
|
||||
rev: v0.1.5
|
||||
|
||||
@@ -34,7 +33,7 @@ repos:
|
||||
rev: v1.0.1
|
||||
hooks:
|
||||
- id: mypy
|
||||
exclude: ^py/tests|^py/unit_tests
|
||||
exclude: ^py/tests/
|
||||
additional_dependencies:
|
||||
[
|
||||
"types-requests",
|
||||
@@ -64,13 +63,13 @@ repos:
|
||||
rev: v3.0.3
|
||||
hooks:
|
||||
- id: prettier
|
||||
exclude: ^(uv.lock|ts/llama_cloud_services/pnpm-lock.yaml|ts/e2e-tests)
|
||||
exclude: uv.lock
|
||||
- repo: https://github.com/codespell-project/codespell
|
||||
rev: v2.2.6
|
||||
hooks:
|
||||
- id: codespell
|
||||
additional_dependencies: [tomli]
|
||||
exclude: ^(uv.lock|docs|ts|examples|pnpm-lock.yaml)
|
||||
exclude: ^(uv.lock|examples|ts)
|
||||
args:
|
||||
[
|
||||
"--ignore-words-list",
|
||||
@@ -87,4 +86,4 @@ repos:
|
||||
- id: toml-sort-fix
|
||||
exclude: ".*uv.lock"
|
||||
|
||||
exclude: ^(.github/ISSUE_TEMPLATE|ts/llama_cloud_services/src/client|pnpm-lock.yaml)
|
||||
exclude: .github/ISSUE_TEMPLATE
|
||||
|
||||
+18
-151
@@ -1,166 +1,33 @@
|
||||
# Contribute to `llama-cloud-services`
|
||||
# Python
|
||||
|
||||
## Common Patterns
|
||||
## Installation
|
||||
|
||||
### Issues
|
||||
This project uses uv. Create a virtual environment, and run `uv sync`
|
||||
|
||||
One of the forms of contribution can be issues.
|
||||
## Versioning (Maintainers only)
|
||||
|
||||
Issues should be used when there are bugs or feature request you would like to bring to the attention of the maintainers.
|
||||
Before merging your changes, make sure to bump the versions.
|
||||
|
||||
When opening an issue:
|
||||
Make a version bump to `pyproject.toml`. If the underlying dependency on the llamacloud platform OpenAPI
|
||||
sdk needs bumping, make sure to bring that in as well. If updating dependencies, run `uv lock`.
|
||||
|
||||
- preferably, use the provided templates
|
||||
- check for other issues (closed and open) to avoid duplicates
|
||||
- try to be detailed and specific, reporting all the pieces the maintainer would need to have in order to reproduce your issue.
|
||||
The legacy `llama_parse` package re-exports some of `llama_cloud_services` in the old namespace. The
|
||||
versions need to be kept consistent to sidecar it with `llama_cloud_services`. Bump it's version in `llama_parse/pyproject.toml`, and also bump it's dependency version of `llama-cloud-services` to match.
|
||||
|
||||
### Pull requests
|
||||
**Note**: Don't worry about updating the `llama_parse/poetry.lock` file when bumping versions. The GitHub action will automatically run `poetry lock` for the llama_parse package during the build process (though it doesn't commit the updated lockfile back to the repo).
|
||||
|
||||
In order to open a valid pull request:
|
||||
You can also do this with `./scripts/version-bump.py set 0.x.x` if you have `uv` installed.
|
||||
|
||||
- Fork the repository
|
||||
- Checkout a secondary branch (common prefixes for secondary branches include: `fix`, `feat`, `chore`, `docs`). We tend to prefer the naming convention that uses `/`, such as: `fix/your-awesome-bug-fix`.
|
||||
- Add and commit the changes to the secondary branch, following language-specific logic (see below)
|
||||
- When the changes are pushed to your branch, open a pull request
|
||||
Once the change is merged, push a tag `git tag -a v0.x.x -m 0.x.x` and `git push origin 0.x.x`.
|
||||
|
||||
## Python
|
||||
This tagging step can be done with `./scripts/version-bump tag`.
|
||||
|
||||
### Set Up
|
||||
# Typescript
|
||||
|
||||
The two python packages, which can be found under `py/`, are:
|
||||
## Installation
|
||||
|
||||
- `llama-cloud-services`
|
||||
- `llama-parse`
|
||||
...
|
||||
|
||||
> [!NOTE]
|
||||
>
|
||||
> `llama-parse` mostly re-exports from `llama-cloud-services`, so you should not modify that directly.
|
||||
## Versioning
|
||||
|
||||
These packages are managed through [uv](https://docs.astral.sh/uv/), so make sure to have uv [installed](https://docs.astral.sh/uv/getting-started/installation/).
|
||||
|
||||
### Tests
|
||||
|
||||
It is important to make sure all tests pass after your changes, and cover new features with suitable unit tests.
|
||||
|
||||
Tests are found in `py/tests/` (end to end) and `py/unit_tests/` (unit tests, no API key required) and you can execute them with:
|
||||
|
||||
```bash
|
||||
pytest tests/ unit_tests/
|
||||
```
|
||||
|
||||
### Pre-Commit Versioning
|
||||
|
||||
Once you made your changes and tested them, **prior to committing** you should run (from the root folder) this command to automatically bump the version of python packages:
|
||||
|
||||
```bash
|
||||
pnpm pre-commit-version
|
||||
```
|
||||
|
||||
this will prompt you to choose what package's version you want to bump and what kind of bump you want to perform. Choose `@llama_cloud_services/py` for python and choose the version bump according to the type of changing you made.
|
||||
|
||||
### Pre-commit checks
|
||||
|
||||
Before you commit, your files should pass the linting and formatting requirements. In order to do that, you should have `pre-commit` installed and set-up in your repository:
|
||||
|
||||
```bash
|
||||
pip install pre-commit
|
||||
pre-commit install
|
||||
```
|
||||
|
||||
Once you have that set up, the files will be automatically linted and formatted according to the requirements.
|
||||
|
||||
## TypeScript
|
||||
|
||||
### Set Up
|
||||
|
||||
The TypeScript package, which can be found under `ts/llama_cloud_services/`, is managed through [`pnpm`](https://pnpm.io), so make sure to have it [installed](https://pnpm.io/installation).
|
||||
|
||||
In order to be able to run and test the package, make sure to install all the dependencies:
|
||||
|
||||
```bash
|
||||
pnpm install
|
||||
```
|
||||
|
||||
### Activate Test Mode
|
||||
|
||||
In order to activate test mode (to dynamically test your changes while you are performing them) you can use:
|
||||
|
||||
```bash
|
||||
pnpm turbo run dev
|
||||
```
|
||||
|
||||
### Test
|
||||
|
||||
It is important to make sure all tests pass after your changes, and cover new features with suitable unit tests.
|
||||
|
||||
Tests are found in `ts/llama_cloud_services/tests/` and you can execute them with:
|
||||
|
||||
```bash
|
||||
pnpm test
|
||||
```
|
||||
|
||||
### Pre-Commit Versioning
|
||||
|
||||
Once you made your changes and tested them, **prior to committing** you should run (from the root folder) two commands to automatically bump the version of python packages:
|
||||
|
||||
```bash
|
||||
pnpm pre-commit-version
|
||||
```
|
||||
|
||||
This will prompt you to choose what package's version you want to bump and what kind of bump you want to perform. Choose `llama-cloud-services` for TypeScript and choose the version bump according to the type of changing you made.
|
||||
|
||||
### Pre-commit checks
|
||||
|
||||
Before you commit, your files should pass the linting and formatting requirements. In order to do that, run (from the root folder):
|
||||
|
||||
```bash
|
||||
pnpm pre-commit
|
||||
```
|
||||
|
||||
The files will be then automatically linted and formatted according to the requirements.
|
||||
|
||||
## TypeScript _and_ Python
|
||||
|
||||
If you change **both PY and TS**, for versioning run:
|
||||
|
||||
```bash
|
||||
pnpm pre-commit-version # choose both packages
|
||||
```
|
||||
|
||||
## Release (maintainers only)
|
||||
|
||||
Every push to main might trigger a release.
|
||||
|
||||
Whether a release is pushed out or not depends on the presence of versioning files in `.changesets`: if you want a release to be packaged, then, you need to always run `pnpm pre-commit-version` prior to merging a pull request into main. See [version_bump_and_release](./.github/workflows/version_bump_and_release.yml) action to better understand the process.
|
||||
|
||||
You can, nevertheless, manually set up language-specific releases, using the logic reported below.
|
||||
|
||||
### Python
|
||||
|
||||
To release `llama-cloud-services` and `llama-parse` in Python, run:
|
||||
|
||||
```bash
|
||||
git checkout main
|
||||
git pull
|
||||
git tag <your-version> # e.g. v0.7.0
|
||||
git push <your-version>
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
>
|
||||
> The tag must start with `v`
|
||||
|
||||
This will trigger the release workflow automatically.
|
||||
|
||||
### TypeScript
|
||||
|
||||
To release `llama-cloud-services` in TypeScript, run:
|
||||
|
||||
```bash
|
||||
git checkout main
|
||||
git pull
|
||||
git tag llama-cloud-services@<your-version> # e.g. llama-cloud-services@0.3.0
|
||||
git push origin llama-cloud-services@<your-version>
|
||||
```
|
||||
|
||||
This will trigger the release workflow automatically.
|
||||
...
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
# LlamaCloud Services Examples - Python
|
||||
|
||||
In this folder you will find several TypeScript end-to-end applications that contain examples regarding:
|
||||
|
||||
- [LlamaParse](./parse/)
|
||||
- [LlamaCloud Index](./index/)
|
||||
|
||||
Follow the instructions in each example folder to get started!
|
||||
+8
-6
@@ -1,9 +1,11 @@
|
||||
# LlamaCloud Services Examples - Python
|
||||
# LlamaCloud Services Examples
|
||||
|
||||
In this folder you will find several python notebooks that contain examples regarding:
|
||||
In this folder you will find several python notebooks and two end-to-end typescript applications that contain examples regarding:
|
||||
|
||||
- [LlamaParse](./parse/)
|
||||
- [LlamaExtract](./extract/)
|
||||
- [LlamaReport](./report/)
|
||||
- [LlamaParse - Python](./parse/)
|
||||
- [LlamaParse - TypeScript](./parse-ts/)
|
||||
- [LlamaExtract - Python](./extract/)
|
||||
- [LlamaReport - Python](./report/)
|
||||
- [LlamaCloud Index - TypeScript](./index-ts/)
|
||||
|
||||
Follow the instructions in each notebook to get started!
|
||||
Follow the instructions of each notebook/application to get started!
|
||||
|
||||
@@ -147,7 +147,7 @@
|
||||
"documents = []\n",
|
||||
"\n",
|
||||
"for i, page in enumerate(pages):\n",
|
||||
" # loop through items of the page\n",
|
||||
" # loop trough items of the page\n",
|
||||
" for item in page[\"items\"]:\n",
|
||||
" document = Document(\n",
|
||||
" text=item[\"md\"], extra_info={\"bbox\": item[\"bBox\"], \"page\": i}\n",
|
||||
|
||||
+28
-146
@@ -4,11 +4,8 @@ LlamaExtract provides a simple API for extracting structured data from unstructu
|
||||
|
||||
## Quick Start
|
||||
|
||||
The simplest way to get started is to use the stateless API with the extraction configuration and the file/text to extract from:
|
||||
|
||||
```python
|
||||
from llama_cloud_services import LlamaExtract
|
||||
from llama_cloud import ExtractConfig, ExtractMode
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# Initialize client
|
||||
@@ -22,87 +19,29 @@ class Resume(BaseModel):
|
||||
skills: list[str] = Field(description="Technical skills and technologies")
|
||||
|
||||
|
||||
# Configure extraction settings
|
||||
config = ExtractConfig(extraction_mode=ExtractMode.FAST)
|
||||
# Create extraction agent
|
||||
agent = extractor.create_agent(name="resume-parser", data_schema=Resume)
|
||||
|
||||
# Extract data directly from document - no agent needed!
|
||||
result = extractor.extract(Resume, config, "resume.pdf")
|
||||
# Extract data from document
|
||||
result = agent.extract("resume.pdf")
|
||||
print(result.data)
|
||||
```
|
||||
|
||||
### Supported File Types
|
||||
|
||||
LlamaExtract supports the following file formats:
|
||||
|
||||
- **Documents**: PDF (.pdf), Word (.docx)
|
||||
- **Text files**: Plain text (.txt), CSV (.csv), JSON (.json), HTML (.html, .htm), Markdown (.md)
|
||||
- **Images**: PNG (.png), JPEG (.jpg, .jpeg)
|
||||
|
||||
### Different Input Types
|
||||
|
||||
```python
|
||||
# From file path (string or Path)
|
||||
result = extractor.extract(Resume, config, "resume.pdf")
|
||||
|
||||
# From file handle
|
||||
with open("resume.pdf", "rb") as f:
|
||||
result = extractor.extract(Resume, config, f)
|
||||
|
||||
# From bytes with filename
|
||||
with open("resume.pdf", "rb") as f:
|
||||
file_bytes = f.read()
|
||||
from llama_cloud_services.extract import SourceText
|
||||
|
||||
result = extractor.extract(
|
||||
Resume, config, SourceText(file=file_bytes, filename="resume.pdf")
|
||||
)
|
||||
|
||||
# From text content
|
||||
text = "Name: John Doe\nEmail: john@example.com\nSkills: Python, AI"
|
||||
result = extractor.extract(Resume, config, SourceText(text_content=text))
|
||||
```
|
||||
|
||||
### Async Extraction
|
||||
|
||||
For better performance with multiple files or when integrating with async applications:
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
|
||||
|
||||
async def extract_resumes():
|
||||
# Async extraction
|
||||
result = await extractor.aextract(Resume, config, "resume.pdf")
|
||||
print(result.data)
|
||||
|
||||
# Queue extraction jobs (returns immediately)
|
||||
jobs = await extractor.queue_extraction(
|
||||
Resume, config, ["resume1.pdf", "resume2.pdf"]
|
||||
)
|
||||
print(f"Queued {len(jobs)} extraction jobs")
|
||||
|
||||
|
||||
# Run async function
|
||||
asyncio.run(extract_resumes())
|
||||
```
|
||||
|
||||
## Core Concepts
|
||||
|
||||
- **Extraction Agents**: Reusable extractors configured with a specific schema and extraction settings.
|
||||
- **Data Schema**: Structure definition for the data you want to extract in the form of a JSON schema or a Pydantic model.
|
||||
- **Extraction Config**: Settings that control how extraction is performed (e.g., speed vs accuracy trade-offs).
|
||||
- **Extraction Jobs**: Asynchronous extraction tasks that can be monitored.
|
||||
- **Extraction Agents** (Advanced): Reusable extractors configured with a specific schema and extraction settings.
|
||||
|
||||
## Defining Schemas
|
||||
|
||||
Schemas define the structure of data you want to extract. You can use either Pydantic models or JSON Schema:
|
||||
Schemas can be defined using either Pydantic models or JSON Schema:
|
||||
|
||||
### Using Pydantic (Recommended)
|
||||
|
||||
```python
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List, Optional
|
||||
from llama_cloud import ExtractConfig, ExtractMode
|
||||
|
||||
|
||||
class Experience(BaseModel):
|
||||
@@ -115,11 +54,6 @@ class Experience(BaseModel):
|
||||
class Resume(BaseModel):
|
||||
name: str = Field(description="Candidate name")
|
||||
experience: List[Experience] = Field(description="Work history")
|
||||
|
||||
|
||||
# Use the schema for extraction
|
||||
config = ExtractConfig(extraction_mode=ExtractMode.FAST)
|
||||
result = extractor.extract(Resume, config, "resume.pdf")
|
||||
```
|
||||
|
||||
### Using JSON Schema
|
||||
@@ -154,27 +88,7 @@ schema = {
|
||||
},
|
||||
}
|
||||
|
||||
# Use the schema for extraction
|
||||
config = ExtractConfig(extraction_mode=ExtractMode.FAST)
|
||||
result = extractor.extract(schema, config, "resume.pdf")
|
||||
```
|
||||
|
||||
## Extraction Configuration
|
||||
|
||||
Configure how extraction is performed using `ExtractConfig`:
|
||||
|
||||
```python
|
||||
from llama_cloud import ExtractConfig, ExtractMode
|
||||
|
||||
# Fast extraction (lower accuracy, faster processing)
|
||||
fast_config = ExtractConfig(extraction_mode=ExtractMode.FAST)
|
||||
|
||||
# Balanced extraction (good balance of speed and accuracy)
|
||||
balanced_config = ExtractConfig(extraction_mode=ExtractMode.BALANCED)
|
||||
|
||||
# Use different configs for different needs
|
||||
result = extractor.extract(schema, fast_config, "simple_document.pdf")
|
||||
result = extractor.extract(schema, balanced_config, "complex_document.pdf")
|
||||
agent = extractor.create_agent(name="resume-parser", data_schema=schema)
|
||||
```
|
||||
|
||||
### Important restrictions on JSON/Pydantic Schema
|
||||
@@ -194,44 +108,28 @@ be sufficient for a wide variety of use-cases.
|
||||
your extraction workflow to fit within these constraints, e.g. by extracting subset of fields
|
||||
and later merging them together.
|
||||
|
||||
## Extraction Agents (Advanced)
|
||||
## Other Extraction APIs
|
||||
|
||||
For reusable extraction workflows, you can create extraction agents that encapsulate both schema and configuration:
|
||||
### Extraction over bytes or text
|
||||
|
||||
### Creating Agents
|
||||
You can use the `SourceText` class to extract from bytes or text directly without using a file. If passing the file bytes,
|
||||
you will need to pass the filename to the `SourceText` class.
|
||||
|
||||
```python
|
||||
from llama_cloud_services import LlamaExtract
|
||||
from llama_cloud import ExtractConfig, ExtractMode
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# Initialize client
|
||||
extractor = LlamaExtract()
|
||||
|
||||
|
||||
# Define schema
|
||||
class Resume(BaseModel):
|
||||
name: str = Field(description="Full name of candidate")
|
||||
email: str = Field(description="Email address")
|
||||
skills: list[str] = Field(description="Technical skills and technologies")
|
||||
|
||||
|
||||
# Configure extraction settings
|
||||
config = ExtractConfig(extraction_mode=ExtractMode.FAST)
|
||||
|
||||
# Create extraction agent
|
||||
agent = extractor.create_agent(
|
||||
name="resume-parser", data_schema=Resume, config=config
|
||||
)
|
||||
|
||||
# Use the agent
|
||||
result = agent.extract("resume.pdf")
|
||||
print(result.data)
|
||||
with open("resume.pdf", "rb") as f:
|
||||
file_bytes = f.read()
|
||||
result = test_agent.extract(SourceText(file=file_bytes, filename="resume.pdf"))
|
||||
```
|
||||
|
||||
### Agent Batch Processing
|
||||
```python
|
||||
result = test_agent.extract(
|
||||
SourceText(text_content="Candidate Name: Jane Doe")
|
||||
)
|
||||
```
|
||||
|
||||
Process multiple files with an agent:
|
||||
### Batch Processing
|
||||
|
||||
Process multiple files asynchronously:
|
||||
|
||||
```python
|
||||
# Queue multiple files for extraction
|
||||
@@ -246,7 +144,7 @@ for job in jobs:
|
||||
results = [agent.get_extraction_run_for_job(job.id) for job in jobs]
|
||||
```
|
||||
|
||||
### Updating Agent Schemas
|
||||
### Updating Schemas
|
||||
|
||||
Schemas can be modified and updated after creation:
|
||||
|
||||
@@ -271,26 +169,10 @@ agent = extractor.get_agent(name="resume-parser")
|
||||
extractor.delete_agent(agent.id)
|
||||
```
|
||||
|
||||
### When to Use Agents vs Direct Extraction
|
||||
|
||||
**Use Direct Extraction When:**
|
||||
|
||||
- One-off extractions
|
||||
- Different schemas for different documents
|
||||
- Simple workflows
|
||||
- Getting started quickly
|
||||
|
||||
**Use Extraction Agents When:**
|
||||
|
||||
- Repeated extractions with the same schema
|
||||
- Team collaboration (shared, named extractors)
|
||||
- Complex workflows requiring state management
|
||||
- Production systems with consistent extraction patterns
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
pip install llama-cloud-services
|
||||
pip install llama-extract==0.1.0
|
||||
```
|
||||
|
||||
## Tips & Best Practices
|
||||
@@ -311,9 +193,9 @@ At the core of LlamaExtract is the schema, which defines the structure of the da
|
||||
2. **Running Extractions**:
|
||||
- Note that resetting `agent.schema` will not save the schema to the database,
|
||||
until you call `agent.save`, but it will be used for running extractions.
|
||||
- Check extraction results for any errors. Error information is available in the `result.error` field for debugging.
|
||||
- Consider async operations (`aextract` or `queue_extraction`) for large-scale extraction or when processing multiple files.
|
||||
- For repeated extractions with the same schema, consider creating an extraction agent to avoid redefining the schema each time.
|
||||
- Check job status prior to accessing results. Any extraction error should be available as
|
||||
part of `job.error` or `extraction_run.error` fields for debugging.
|
||||
- Consider async operations (`queue_extraction`) for large-scale extraction once you have finalized your schema.
|
||||
|
||||
### Hitting "The response was too long to be processed" Error
|
||||
|
||||
@@ -326,5 +208,5 @@ Another option (orthogonal to the above) is to break the document into smaller s
|
||||
|
||||
## Additional Resources
|
||||
|
||||
- [Example Notebook](docs/examples-py/extract/resume_screening.ipynb) - Detailed walkthrough of resume parsing
|
||||
- [Example Notebook](examples/extract/resume_screening.ipynb) - Detailed walkthrough of resume parsing
|
||||
- [Discord Community](https://discord.com/invite/eN6D2HQ4aX) - Get help and share feedback
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
{
|
||||
"name": "llama_cloud_services",
|
||||
"version": "1.0.0",
|
||||
"private": "true",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
"pre-commit-version": "pnpm changeset",
|
||||
"version": "pnpm version-ts && pnpm version-py",
|
||||
"release": "pnpm release-ts && pnpm release-py",
|
||||
"release-ts": "pnpm run --filter llama-cloud-services release",
|
||||
"release-py": "pnpm run --filter @llama_cloud_services/py release",
|
||||
"version-ts": "pnpm run --filter llama-cloud-services version",
|
||||
"version-py": "pnpm run --filter @llama_cloud_services/py version"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@changesets/cli": "^2.29.5",
|
||||
"changesets": "^1.0.2"
|
||||
}
|
||||
}
|
||||
@@ -97,7 +97,7 @@ for page in result.pages:
|
||||
print(page.structuredData)
|
||||
```
|
||||
|
||||
See more details about the result object in the [example notebook](./docs/examples-py/parse/demo_json_tour.ipynb).
|
||||
See more details about the result object in the [example notebook](./examples/parse/demo_json_tour.ipynb).
|
||||
|
||||
### Using with file object / bytes
|
||||
|
||||
@@ -153,10 +153,10 @@ Full documentation for `SimpleDirectoryReader` can be found on the [LlamaIndex D
|
||||
|
||||
Several end-to-end indexing examples can be found in the examples folder
|
||||
|
||||
- [Getting Started](docs/examples-py/parse/demo_basic.ipynb)
|
||||
- [Advanced RAG Example](docs/examples-py/parse/demo_advanced.ipynb)
|
||||
- [Raw API Usage](docs/examples-py/parse/demo_api.ipynb)
|
||||
- [Result Object Tour](docs/examples-py/parse/demo_json_tour.ipynb)
|
||||
- [Getting Started](./examples/parse/demo_basic.ipynb)
|
||||
- [Advanced RAG Example](./examples/parse/demo_advanced.ipynb)
|
||||
- [Raw API Usage](./examples/parse/demo_api.ipynb)
|
||||
- [Result Object Tour](./examples/parse/demo_json_tour.ipynb)
|
||||
|
||||
## Documentation
|
||||
|
||||
|
||||
Generated
+598
-3619
File diff suppressed because it is too large
Load Diff
@@ -1,3 +0,0 @@
|
||||
packages:
|
||||
- "ts/*"
|
||||
- "py/"
|
||||
@@ -181,15 +181,11 @@ class ExtractedFieldMetadata(BaseModel):
|
||||
Metadata for an extracted data field, such as confidence, and citation information.
|
||||
"""
|
||||
|
||||
reasoning: Optional[str] = Field(
|
||||
None,
|
||||
description="symbol for how the citation/confidence was derived: 'INFERRED FROM TEXT', 'VERBATIM EXTRACTION'",
|
||||
)
|
||||
confidence: Optional[float] = Field(
|
||||
None,
|
||||
description="The confidence score for the field, combined with parsing confidence if applicable",
|
||||
)
|
||||
extraction_confidence: Optional[float] = Field(
|
||||
extracted_confidence: Optional[float] = Field(
|
||||
None,
|
||||
description="The confidence score for the field based on the extracted text only",
|
||||
)
|
||||
@@ -210,66 +206,42 @@ ExtractedFieldMetaDataDict = Dict[
|
||||
def parse_extracted_field_metadata(
|
||||
field_metadata: dict[str, Any],
|
||||
) -> ExtractedFieldMetaDataDict:
|
||||
return {
|
||||
k: _parse_extracted_field_metadata_recursive(v)
|
||||
for k, v in field_metadata.items()
|
||||
if k not in _METADATA_FIELDS_SIBLING_TO_LEAF
|
||||
}
|
||||
|
||||
|
||||
_METADATA_FIELDS_SIBLING_TO_LEAF = {"reasoning"}
|
||||
|
||||
|
||||
def _parse_extracted_field_metadata_recursive(
|
||||
field_value: Any,
|
||||
additional_fields: dict[str, Any] = {},
|
||||
) -> Union[ExtractedFieldMetadata, Dict[str, Any], list[Any]]:
|
||||
"""
|
||||
Parse the extracted field metadata into a dictionary of field names to field metadata.
|
||||
"""
|
||||
result: ExtractedFieldMetaDataDict = {}
|
||||
for field_name, field_value in field_metadata.items():
|
||||
if isinstance(field_value, ExtractedFieldMetadata):
|
||||
# support running this multiple times
|
||||
result[field_name] = field_value
|
||||
elif isinstance(field_value, dict):
|
||||
if "confidence" in field_value or "citations" in field_value:
|
||||
try:
|
||||
validated = ExtractedFieldMetadata.model_validate(field_value)
|
||||
|
||||
if isinstance(field_value, ExtractedFieldMetadata):
|
||||
# support running this multiple times
|
||||
return field_value
|
||||
elif isinstance(field_value, dict):
|
||||
# reasoning explicitly excluded, as it is included next to subfields, for example
|
||||
# "dimensions.width" is a leaf, but there will still potentially be a "dimensions.reasoning"
|
||||
indicator_fields = {"confidence", "extraction_confidence", "citation"}
|
||||
if len(indicator_fields.intersection(field_value.keys())) > 0:
|
||||
try:
|
||||
merged = {**field_value, **additional_fields}
|
||||
validated = ExtractedFieldMetadata.model_validate(merged)
|
||||
|
||||
# grab the citation from the array. This is just an array for backwards compatibility.
|
||||
if "citation" in field_value and len(field_value["citation"]) > 0:
|
||||
first_citation = field_value["citation"][0]
|
||||
if "page" in first_citation and isinstance(
|
||||
first_citation["page"], numbers.Number
|
||||
):
|
||||
validated.page_number = int(first_citation["page"]) # type: ignore
|
||||
if "matching_text" in first_citation and isinstance(
|
||||
first_citation["matching_text"], str
|
||||
):
|
||||
validated.matching_text = first_citation["matching_text"]
|
||||
return validated
|
||||
except ValidationError:
|
||||
pass
|
||||
additional_fields = {
|
||||
k: v
|
||||
for k, v in field_value.items()
|
||||
if k in _METADATA_FIELDS_SIBLING_TO_LEAF
|
||||
}
|
||||
return {
|
||||
k: _parse_extracted_field_metadata_recursive(v, additional_fields)
|
||||
for k, v in field_value.items()
|
||||
if k not in _METADATA_FIELDS_SIBLING_TO_LEAF
|
||||
}
|
||||
elif isinstance(field_value, list):
|
||||
return [_parse_extracted_field_metadata_recursive(item) for item in field_value]
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Invalid field value: {field_value}. Expected ExtractedFieldMetadata, dict, or list"
|
||||
)
|
||||
# grab the citation from the array. This is just an array for backwards compatibility.
|
||||
if "citations" in field_value and len(field_value["citations"]) > 0:
|
||||
first_citation = field_value["citations"][0]
|
||||
if "page_number" in first_citation and isinstance(
|
||||
first_citation["page_number"], numbers.Number
|
||||
):
|
||||
validated.page_number = int(first_citation["page_number"]) # type: ignore
|
||||
if "matching_text" in first_citation and isinstance(
|
||||
first_citation["matching_text"], str
|
||||
):
|
||||
validated.matching_text = first_citation["matching_text"]
|
||||
result[field_name] = validated
|
||||
continue
|
||||
except ValidationError:
|
||||
pass
|
||||
result[field_name] = parse_extracted_field_metadata(field_value)
|
||||
elif isinstance(field_value, list):
|
||||
result[field_name] = [
|
||||
parse_extracted_field_metadata(item) for item in field_value
|
||||
]
|
||||
else:
|
||||
result[field_name] = field_value
|
||||
return result
|
||||
|
||||
|
||||
class ExtractedData(BaseModel, Generic[ExtractedT]):
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import os
|
||||
import time
|
||||
from io import BufferedIOBase, BufferedReader, BytesIO, TextIOWrapper
|
||||
@@ -22,7 +21,6 @@ from llama_cloud import (
|
||||
ExtractJobCreate,
|
||||
ExtractRun,
|
||||
File,
|
||||
FileData,
|
||||
ExtractMode,
|
||||
StatusEnum,
|
||||
ExtractTarget,
|
||||
@@ -49,7 +47,7 @@ SchemaInput = Union[JSONObjectType, Type[BaseModel]]
|
||||
|
||||
DEFAULT_EXTRACT_CONFIG = ExtractConfig(
|
||||
extraction_target=ExtractTarget.PER_DOC,
|
||||
extraction_mode=ExtractMode.MULTIMODAL,
|
||||
extraction_mode=ExtractMode.BALANCED,
|
||||
)
|
||||
|
||||
|
||||
@@ -64,132 +62,6 @@ def _is_retryable_error(exception: BaseException) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
async def _validate_schema(
|
||||
client: AsyncLlamaCloud, data_schema: SchemaInput
|
||||
) -> JSONObjectType:
|
||||
"""Convert SchemaInput to a validated JSON schema dictionary."""
|
||||
processed_schema: JSONObjectType
|
||||
if isinstance(data_schema, dict):
|
||||
# TODO: if we expose a get_validated JSON schema method, we can use it here
|
||||
processed_schema = data_schema # type: ignore
|
||||
elif isinstance(data_schema, type) and issubclass(data_schema, BaseModel):
|
||||
processed_schema = data_schema.model_json_schema()
|
||||
else:
|
||||
raise ValueError("data_schema must be either a dictionary or a Pydantic model")
|
||||
|
||||
# Validate schema via API
|
||||
validated_schema = await client.llama_extract.validate_extraction_schema(
|
||||
data_schema=processed_schema
|
||||
)
|
||||
return validated_schema.data_schema
|
||||
|
||||
|
||||
async def _get_job_with_retry(
|
||||
client: AsyncLlamaCloud,
|
||||
job_id: str,
|
||||
max_attempts: int = 5,
|
||||
initial_wait: float = 1,
|
||||
max_wait: float = 60,
|
||||
jitter: float = 5,
|
||||
) -> ExtractJob:
|
||||
"""Get extraction job with retry logic."""
|
||||
async for attempt in AsyncRetrying(
|
||||
retry=retry_if_exception(_is_retryable_error),
|
||||
stop=stop_after_attempt(max_attempts),
|
||||
wait=wait_exponential_jitter(initial=initial_wait, max=max_wait, jitter=jitter),
|
||||
reraise=True,
|
||||
):
|
||||
with attempt:
|
||||
return await client.llama_extract.get_job(job_id=job_id)
|
||||
|
||||
|
||||
async def _get_run_with_retry(
|
||||
client: AsyncLlamaCloud,
|
||||
job_id: str,
|
||||
project_id: Optional[str] = None,
|
||||
organization_id: Optional[str] = None,
|
||||
max_attempts: int = 3,
|
||||
initial_wait: float = 1,
|
||||
max_wait: float = 20,
|
||||
jitter: float = 3,
|
||||
) -> ExtractRun:
|
||||
"""Get extraction run with retry logic."""
|
||||
async for attempt in AsyncRetrying(
|
||||
retry=retry_if_exception(_is_retryable_error),
|
||||
stop=stop_after_attempt(max_attempts),
|
||||
wait=wait_exponential_jitter(initial=initial_wait, max=max_wait, jitter=jitter),
|
||||
reraise=True,
|
||||
):
|
||||
with attempt:
|
||||
return await client.llama_extract.get_run_by_job_id(
|
||||
job_id=job_id,
|
||||
project_id=project_id,
|
||||
organization_id=organization_id,
|
||||
)
|
||||
|
||||
|
||||
async def _wait_for_job_result(
|
||||
client: AsyncLlamaCloud,
|
||||
job_id: str,
|
||||
check_interval: int = 1,
|
||||
max_timeout: int = 2000,
|
||||
verbose: bool = False,
|
||||
project_id: Optional[str] = None,
|
||||
organization_id: Optional[str] = None,
|
||||
job_retry_attempts: int = 5,
|
||||
job_max_wait: float = 60,
|
||||
job_jitter: float = 5,
|
||||
run_retry_attempts: int = 3,
|
||||
run_max_wait: float = 20,
|
||||
run_jitter: float = 3,
|
||||
) -> Optional[ExtractRun]:
|
||||
"""Wait for and return the results of an extraction job."""
|
||||
start = time.perf_counter()
|
||||
poll_count = 0
|
||||
|
||||
while True:
|
||||
await asyncio.sleep(check_interval)
|
||||
poll_count += 1
|
||||
job = await _get_job_with_retry(
|
||||
client,
|
||||
job_id,
|
||||
max_attempts=job_retry_attempts,
|
||||
max_wait=job_max_wait,
|
||||
jitter=job_jitter,
|
||||
)
|
||||
|
||||
if job.status == StatusEnum.SUCCESS:
|
||||
return await _get_run_with_retry(
|
||||
client,
|
||||
job_id,
|
||||
project_id,
|
||||
organization_id,
|
||||
max_attempts=run_retry_attempts,
|
||||
max_wait=run_max_wait,
|
||||
jitter=run_jitter,
|
||||
)
|
||||
elif job.status == StatusEnum.PENDING:
|
||||
end = time.perf_counter()
|
||||
if end - start > max_timeout:
|
||||
raise Exception(f"Timeout while extracting the file: {job_id}")
|
||||
if verbose and poll_count % 10 == 0:
|
||||
print(".", end="", flush=True)
|
||||
continue
|
||||
else:
|
||||
warnings.warn(
|
||||
f"Failure in job: {job_id}, status: {job.status}, error: {job.error}"
|
||||
)
|
||||
return await _get_run_with_retry(
|
||||
client,
|
||||
job_id,
|
||||
project_id,
|
||||
organization_id,
|
||||
max_attempts=run_retry_attempts,
|
||||
max_wait=run_max_wait,
|
||||
jitter=run_jitter,
|
||||
)
|
||||
|
||||
|
||||
class SourceText:
|
||||
def __init__(
|
||||
self,
|
||||
@@ -272,21 +144,6 @@ def _extraction_config_warning(config: ExtractConfig) -> None:
|
||||
"available in the `extraction_metadata` field for the extraction run.",
|
||||
ExperimentalWarning,
|
||||
)
|
||||
if config.use_reasoning:
|
||||
if config.extraction_mode == ExtractMode.FAST:
|
||||
raise ValueError(
|
||||
"`reasoning` is only supported with BALANCED, MULTIMODAL, or PREMIUM extraction modes."
|
||||
)
|
||||
if config.cite_sources:
|
||||
if config.extraction_mode in (ExtractMode.FAST, ExtractMode.BALANCED):
|
||||
raise ValueError(
|
||||
"`cite_sources` is only supported with MULTIMODAL or PREMIUM extraction modes."
|
||||
)
|
||||
if config.confidence_scores:
|
||||
if config.extraction_mode in (ExtractMode.FAST, ExtractMode.BALANCED):
|
||||
raise ValueError(
|
||||
"`confidence_scores` is only supported with MULTIMODAL or PREMIUM extraction modes."
|
||||
)
|
||||
|
||||
|
||||
class ExtractionAgent:
|
||||
@@ -337,10 +194,22 @@ class ExtractionAgent:
|
||||
|
||||
@data_schema.setter
|
||||
def data_schema(self, data_schema: SchemaInput) -> None:
|
||||
# Use the shared schema processing and validation function
|
||||
self._data_schema = self._run_in_thread(
|
||||
_validate_schema(self._client, data_schema)
|
||||
processed_schema: JSONObjectType
|
||||
if isinstance(data_schema, dict):
|
||||
# TODO: if we expose a get_validated JSON schema method, we can use it here
|
||||
processed_schema = data_schema # type: ignore
|
||||
elif isinstance(data_schema, type) and issubclass(data_schema, BaseModel):
|
||||
processed_schema = data_schema.model_json_schema()
|
||||
else:
|
||||
raise ValueError(
|
||||
"data_schema must be either a dictionary or a Pydantic model"
|
||||
)
|
||||
validated_schema = self._run_in_thread(
|
||||
self._client.llama_extract.validate_extraction_schema(
|
||||
data_schema=processed_schema
|
||||
)
|
||||
)
|
||||
self._data_schema = validated_schema.data_schema
|
||||
|
||||
@property
|
||||
def config(self) -> ExtractConfig:
|
||||
@@ -399,9 +268,7 @@ class ExtractionAgent:
|
||||
project_id=self._project_id, upload_file=file_contents
|
||||
)
|
||||
finally:
|
||||
if file_contents is not None and isinstance(
|
||||
file_contents, (BufferedReader, BytesIO)
|
||||
):
|
||||
if file_contents is not None and isinstance(file_contents, BufferedReader):
|
||||
file_contents.close()
|
||||
|
||||
async def _upload_file(self, file_input: FileInput) -> File:
|
||||
@@ -429,23 +296,60 @@ class ExtractionAgent:
|
||||
|
||||
return await self.upload_file(source_text)
|
||||
|
||||
async def _get_job_with_retry(self, job_id: str) -> ExtractJob:
|
||||
"""Get job with retry logic for transient errors."""
|
||||
async for attempt in AsyncRetrying(
|
||||
retry=retry_if_exception(_is_retryable_error),
|
||||
stop=stop_after_attempt(5),
|
||||
wait=wait_exponential_jitter(initial=1, max=60, jitter=5),
|
||||
reraise=True,
|
||||
):
|
||||
with attempt:
|
||||
return await self._client.llama_extract.get_job(job_id=job_id)
|
||||
|
||||
async def _get_run_with_retry(self, job_id: str) -> ExtractRun:
|
||||
"""Get extraction run with retry logic for transient errors."""
|
||||
async for attempt in AsyncRetrying(
|
||||
retry=retry_if_exception(_is_retryable_error),
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_exponential_jitter(initial=1, max=20, jitter=3),
|
||||
reraise=True,
|
||||
):
|
||||
with attempt:
|
||||
return await self._client.llama_extract.get_run_by_job_id(job_id=job_id)
|
||||
|
||||
async def _wait_for_job_result(self, job_id: str) -> Optional[ExtractRun]:
|
||||
"""Wait for and return the results of an extraction job."""
|
||||
return await _wait_for_job_result(
|
||||
client=self._client,
|
||||
job_id=job_id,
|
||||
check_interval=self.check_interval,
|
||||
max_timeout=self.max_timeout,
|
||||
verbose=self._verbose,
|
||||
project_id=self._project_id,
|
||||
organization_id=self._organization_id,
|
||||
job_retry_attempts=5,
|
||||
job_max_wait=60,
|
||||
job_jitter=5,
|
||||
run_retry_attempts=3,
|
||||
run_max_wait=20,
|
||||
run_jitter=3,
|
||||
)
|
||||
start = time.perf_counter()
|
||||
tries = 0
|
||||
|
||||
while True:
|
||||
await asyncio.sleep(self.check_interval)
|
||||
tries += 1
|
||||
|
||||
try:
|
||||
job = await self._get_job_with_retry(job_id)
|
||||
|
||||
if job.status == StatusEnum.SUCCESS:
|
||||
return await self._get_run_with_retry(job_id)
|
||||
elif job.status == StatusEnum.PENDING:
|
||||
end = time.perf_counter()
|
||||
if end - start > self.max_timeout:
|
||||
raise Exception(f"Timeout while extracting the file: {job_id}")
|
||||
if self._verbose and tries % 10 == 0:
|
||||
print(".", end="", flush=True)
|
||||
continue
|
||||
else:
|
||||
warnings.warn(
|
||||
f"Failure in job: {job_id}, status: {job.status}, error: {job.error}"
|
||||
)
|
||||
return await self._get_run_with_retry(job_id)
|
||||
|
||||
except Exception as e:
|
||||
# If we get a non-retryable error or all retries are exhausted, re-raise
|
||||
if self._verbose:
|
||||
print(f"\nError in job polling for {job_id}: {e}")
|
||||
raise e
|
||||
|
||||
def save(self) -> None:
|
||||
"""Persist the extraction agent's schema and config to the database.
|
||||
@@ -739,14 +643,12 @@ class LlamaExtract(BaseComponent):
|
||||
base_url = os.getenv("LLAMA_CLOUD_BASE_URL", None) or DEFAULT_BASE_URL
|
||||
|
||||
super().__init__(
|
||||
api_key=api_key, # type: ignore
|
||||
base_url=base_url, # type: ignore
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
check_interval=check_interval,
|
||||
max_timeout=max_timeout,
|
||||
num_workers=num_workers,
|
||||
show_progress=show_progress,
|
||||
project_id=project_id,
|
||||
organization_id=organization_id,
|
||||
verify=verify,
|
||||
httpx_timeout=httpx_timeout,
|
||||
verbose=verbose,
|
||||
@@ -800,7 +702,7 @@ class LlamaExtract(BaseComponent):
|
||||
config = DEFAULT_EXTRACT_CONFIG
|
||||
|
||||
if isinstance(data_schema, dict):
|
||||
pass
|
||||
data_schema = data_schema
|
||||
elif issubclass(data_schema, BaseModel):
|
||||
data_schema = data_schema.model_json_schema()
|
||||
else:
|
||||
@@ -901,8 +803,6 @@ class LlamaExtract(BaseComponent):
|
||||
num_workers=self.num_workers,
|
||||
show_progress=self.show_progress,
|
||||
verbose=self.verbose,
|
||||
verify=self.verify,
|
||||
httpx_timeout=self.httpx_timeout,
|
||||
)
|
||||
for agent in agents
|
||||
]
|
||||
@@ -919,245 +819,6 @@ class LlamaExtract(BaseComponent):
|
||||
)
|
||||
)
|
||||
|
||||
async def _wait_for_job_result(self, job_id: str) -> Optional[ExtractRun]:
|
||||
"""Wait for and return the results of an extraction job."""
|
||||
return await _wait_for_job_result(
|
||||
client=self._async_client,
|
||||
job_id=job_id,
|
||||
check_interval=self.check_interval,
|
||||
max_timeout=self.max_timeout,
|
||||
verbose=self.verbose,
|
||||
project_id=self._project_id,
|
||||
organization_id=self._organization_id,
|
||||
job_retry_attempts=3,
|
||||
job_max_wait=4,
|
||||
job_jitter=5,
|
||||
run_retry_attempts=3,
|
||||
run_max_wait=4,
|
||||
run_jitter=3,
|
||||
)
|
||||
|
||||
def _get_mime_type(
|
||||
self,
|
||||
filename: Optional[str] = None,
|
||||
file_path: Optional[Union[str, Path]] = None,
|
||||
) -> str:
|
||||
"""Determine MIME type for a file based on filename or path."""
|
||||
# MIME type mappings for supported formats
|
||||
MIME_TYPE_MAP = {
|
||||
# Text files
|
||||
".txt": "text/plain",
|
||||
".csv": "text/csv",
|
||||
".json": "application/json",
|
||||
".html": "text/html",
|
||||
".htm": "text/html",
|
||||
".md": "text/markdown",
|
||||
# Document files
|
||||
".pdf": "application/pdf",
|
||||
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
# Image files
|
||||
".png": "image/png",
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
}
|
||||
|
||||
# Try to get extension from filename or file_path
|
||||
extension = None
|
||||
if filename:
|
||||
extension = Path(filename).suffix.lower()
|
||||
elif file_path:
|
||||
extension = Path(file_path).suffix.lower()
|
||||
|
||||
# Check if the extension is supported
|
||||
if extension and extension in MIME_TYPE_MAP:
|
||||
return MIME_TYPE_MAP[extension]
|
||||
|
||||
# If we don't have a supported extension, provide helpful error message
|
||||
supported_extensions = [ext[1:] for ext in MIME_TYPE_MAP.keys()] # Remove dots
|
||||
supported_list = ", ".join(sorted(supported_extensions))
|
||||
|
||||
if extension:
|
||||
ext_without_dot = extension[1:] # Remove the leading dot
|
||||
raise ValueError(
|
||||
f"Unsupported file type: '{ext_without_dot}'. "
|
||||
f"Supported formats are: {supported_list}"
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Could not determine file type. Please provide a filename with one of these supported extensions: {supported_list}"
|
||||
)
|
||||
|
||||
def _convert_file_to_file_data(self, file_input: FileInput) -> Union[FileData, str]:
|
||||
"""Convert FileInput to FileData or text string for stateless extraction."""
|
||||
if isinstance(file_input, SourceText):
|
||||
if file_input.text_content is not None:
|
||||
return file_input.text_content
|
||||
elif file_input.file is not None:
|
||||
if isinstance(file_input.file, bytes):
|
||||
data = file_input.file
|
||||
filename = file_input.filename
|
||||
elif isinstance(file_input.file, (str, Path)):
|
||||
with open(file_input.file, "rb") as f:
|
||||
data = f.read()
|
||||
filename = file_input.filename or str(file_input.file)
|
||||
elif isinstance(file_input.file, (BufferedIOBase, TextIOWrapper)):
|
||||
if hasattr(file_input.file, "read"):
|
||||
content = file_input.file.read()
|
||||
if isinstance(content, str):
|
||||
data = content.encode("utf-8")
|
||||
else:
|
||||
data = content
|
||||
else:
|
||||
raise ValueError("File object must have a read method")
|
||||
filename = file_input.filename or getattr(
|
||||
file_input.file, "name", None
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported file type: {type(file_input.file)}")
|
||||
|
||||
# Encode as base64
|
||||
encoded_data = base64.b64encode(data).decode("utf-8")
|
||||
|
||||
# Determine mime type
|
||||
mime_type = self._get_mime_type(filename=filename)
|
||||
|
||||
return FileData(data=encoded_data, mime_type=mime_type)
|
||||
else:
|
||||
raise ValueError("SourceText must have either text_content or file")
|
||||
|
||||
elif isinstance(file_input, (str, Path)):
|
||||
with open(file_input, "rb") as f:
|
||||
data = f.read()
|
||||
encoded_data = base64.b64encode(data).decode("utf-8")
|
||||
mime_type = self._get_mime_type(file_path=file_input)
|
||||
return FileData(data=encoded_data, mime_type=mime_type)
|
||||
|
||||
elif isinstance(file_input, bytes):
|
||||
# For raw bytes, we can't determine the file type, so we need to raise an error
|
||||
raise ValueError(
|
||||
"Cannot determine file type from raw bytes. Please use SourceText with a filename, or provide a file path."
|
||||
)
|
||||
|
||||
elif isinstance(file_input, (BufferedIOBase, TextIOWrapper)):
|
||||
if hasattr(file_input, "read"):
|
||||
content = file_input.read()
|
||||
if isinstance(content, str):
|
||||
data = content.encode("utf-8")
|
||||
else:
|
||||
data = content
|
||||
encoded_data = base64.b64encode(data).decode("utf-8")
|
||||
|
||||
# Try to get filename from the file object
|
||||
filename = getattr(file_input, "name", None)
|
||||
mime_type = self._get_mime_type(filename=filename)
|
||||
|
||||
return FileData(data=encoded_data, mime_type=mime_type)
|
||||
else:
|
||||
raise ValueError("File object must have a read method")
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unsupported file input type: {type(file_input)}")
|
||||
|
||||
async def queue_extraction(
|
||||
self,
|
||||
data_schema: SchemaInput,
|
||||
config: ExtractConfig,
|
||||
files: Union[FileInput, List[FileInput]],
|
||||
) -> Union[ExtractJob, List[ExtractJob]]:
|
||||
"""Queue extraction jobs using stateless extraction (no agent required).
|
||||
|
||||
Args:
|
||||
data_schema: The schema defining what data to extract
|
||||
config: The extraction configuration
|
||||
files: File(s) to extract from
|
||||
|
||||
Returns:
|
||||
ExtractJob or list of ExtractJobs
|
||||
"""
|
||||
_extraction_config_warning(config)
|
||||
processed_schema = await _validate_schema(self._async_client, data_schema)
|
||||
|
||||
if not isinstance(files, list):
|
||||
files = [files]
|
||||
|
||||
jobs = []
|
||||
for file_input in files:
|
||||
file_data_or_text = self._convert_file_to_file_data(file_input)
|
||||
|
||||
if isinstance(file_data_or_text, str):
|
||||
# It's text content
|
||||
job = await self._async_client.llama_extract.extract_stateless(
|
||||
project_id=self._project_id,
|
||||
organization_id=self._organization_id,
|
||||
data_schema=processed_schema,
|
||||
config=config,
|
||||
text=file_data_or_text,
|
||||
)
|
||||
else:
|
||||
# It's FileData
|
||||
job = await self._async_client.llama_extract.extract_stateless(
|
||||
project_id=self._project_id,
|
||||
organization_id=self._organization_id,
|
||||
data_schema=processed_schema,
|
||||
config=config,
|
||||
file=file_data_or_text,
|
||||
)
|
||||
jobs.append(job)
|
||||
|
||||
return jobs[0] if len(jobs) == 1 else jobs
|
||||
|
||||
async def aextract(
|
||||
self,
|
||||
data_schema: SchemaInput,
|
||||
config: ExtractConfig,
|
||||
files: Union[FileInput, List[FileInput]],
|
||||
) -> Union[ExtractRun, List[ExtractRun]]:
|
||||
"""Run stateless extraction and wait for results.
|
||||
|
||||
Args:
|
||||
data_schema: The schema defining what data to extract
|
||||
config: The extraction configuration
|
||||
files: File(s) to extract from
|
||||
|
||||
Returns:
|
||||
ExtractRun or list of ExtractRuns with the extraction results
|
||||
"""
|
||||
jobs = await self.queue_extraction(data_schema, config, files)
|
||||
|
||||
if isinstance(jobs, list):
|
||||
runs = []
|
||||
for job in jobs:
|
||||
run = await self._wait_for_job_result(job.id)
|
||||
if run is None:
|
||||
raise RuntimeError(
|
||||
f"Failed to get extraction result for job {job.id}"
|
||||
)
|
||||
runs.append(run)
|
||||
return runs
|
||||
else:
|
||||
run = await self._wait_for_job_result(jobs.id)
|
||||
if run is None:
|
||||
raise RuntimeError(f"Failed to get extraction result for job {jobs.id}")
|
||||
return run
|
||||
|
||||
def extract(
|
||||
self,
|
||||
data_schema: SchemaInput,
|
||||
config: ExtractConfig,
|
||||
files: Union[FileInput, List[FileInput]],
|
||||
) -> Union[ExtractRun, List[ExtractRun]]:
|
||||
"""Run stateless extraction and wait for results (synchronous version).
|
||||
|
||||
Args:
|
||||
data_schema: The schema defining what data to extract
|
||||
config: The extraction configuration
|
||||
files: File(s) to extract from
|
||||
|
||||
Returns:
|
||||
ExtractRun or list of ExtractRuns with the extraction results
|
||||
"""
|
||||
return self._run_in_thread(self.aextract(data_schema, config, files))
|
||||
|
||||
def __del__(self) -> None:
|
||||
"""Cleanup resources properly."""
|
||||
try:
|
||||
@@ -1172,20 +833,6 @@ if __name__ == "__main__":
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# Example usage:
|
||||
#
|
||||
# # Basic usage with stateless extraction (no agent required)
|
||||
# extractor = LlamaExtract()
|
||||
# schema = {"name": {"type": "string"}, "email": {"type": "string"}}
|
||||
# config = ExtractConfig(extraction_mode=ExtractMode.FAST)
|
||||
# files = ["path/to/document.pdf"]
|
||||
#
|
||||
# # Queue extraction jobs
|
||||
# jobs = extractor.queue_extraction(schema, config, files)
|
||||
#
|
||||
# # Or run extraction and wait for results
|
||||
# results = extractor.extract(schema, config, files)
|
||||
|
||||
data_dir = Path(__file__).parent.parent / "tests" / "data"
|
||||
extractor = LlamaExtract()
|
||||
try:
|
||||
|
||||
@@ -25,7 +25,7 @@ from llama_index.core.readers.base import BasePydanticReader
|
||||
from llama_index.core.readers.file.base import get_default_fs
|
||||
from llama_index.core.schema import Document
|
||||
|
||||
from llama_cloud_services.utils import check_extra_params, check_for_updates
|
||||
from llama_cloud_services.utils import check_extra_params
|
||||
from llama_cloud_services.parse.types import JobResult
|
||||
from llama_cloud_services.parse.utils import (
|
||||
SUPPORTED_FILE_TYPES,
|
||||
@@ -541,11 +541,6 @@ class LlamaParse(BasePydanticReader):
|
||||
description="Whether to use the vendor multimodal API.",
|
||||
)
|
||||
|
||||
check_for_updates: Optional[bool] = Field(
|
||||
default=False,
|
||||
description="Automatically check for Python SDK updates.",
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def warn_extra_params(cls, data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
@@ -601,16 +596,6 @@ class LlamaParse(BasePydanticReader):
|
||||
|
||||
return self._aclient
|
||||
|
||||
_update_checked = False
|
||||
|
||||
async def _check_for_updates(self) -> None:
|
||||
if self.check_for_updates and not self._update_checked:
|
||||
try:
|
||||
await check_for_updates(self.aclient, quiet=False)
|
||||
self._update_checked = True
|
||||
except (ValueError, httpx.HTTPStatusError):
|
||||
pass
|
||||
|
||||
@asynccontextmanager
|
||||
async def client_context(self) -> AsyncGenerator[httpx.AsyncClient, None]:
|
||||
"""Create a context for the HTTPX client."""
|
||||
@@ -660,7 +645,6 @@ class LlamaParse(BasePydanticReader):
|
||||
fs: Optional[AbstractFileSystem] = None,
|
||||
partition_target_pages: Optional[str] = None,
|
||||
) -> str:
|
||||
await self._check_for_updates()
|
||||
files = None
|
||||
file_handle = None
|
||||
input_url = file_input if self._is_input_url(file_input) else None
|
||||
@@ -1550,7 +1534,6 @@ class LlamaParse(BasePydanticReader):
|
||||
self, json_result: List[dict], download_path: str, asset_key: str
|
||||
) -> List[dict]:
|
||||
"""Download assets (images or charts) from the parsed result."""
|
||||
await self._check_for_updates()
|
||||
# Make the download path
|
||||
if not os.path.exists(download_path):
|
||||
os.makedirs(download_path)
|
||||
@@ -1659,7 +1642,6 @@ class LlamaParse(BasePydanticReader):
|
||||
self, json_result: List[dict], download_path: str
|
||||
) -> List[dict]:
|
||||
"""Download xlsx from the parsed result."""
|
||||
await self._check_for_updates()
|
||||
# make the download path
|
||||
if not os.path.exists(download_path):
|
||||
os.makedirs(download_path)
|
||||
|
||||
@@ -1,9 +1,4 @@
|
||||
import os
|
||||
import importlib.metadata
|
||||
|
||||
import difflib
|
||||
import httpx
|
||||
import packaging.version
|
||||
from pydantic import BaseModel
|
||||
from typing import Any, Dict, List, Tuple, Type
|
||||
|
||||
@@ -32,36 +27,3 @@ def check_extra_params(
|
||||
)
|
||||
|
||||
return extra_params, suggestions
|
||||
|
||||
|
||||
async def check_for_updates(client: httpx.AsyncClient, quiet: bool = True) -> bool:
|
||||
"""Check if an SDK update is available.
|
||||
|
||||
Args:
|
||||
client: HTTPX client to use.
|
||||
quiet: If False, update availability will also be printed to stdout.
|
||||
|
||||
Returns: True if an update is available.
|
||||
|
||||
Raises:
|
||||
ValueError: Failed to get a valid release version from PyPI.
|
||||
"""
|
||||
package_name = "llama-cloud-services"
|
||||
r = await client.get(f"https://pypi.org/pypi/{package_name}/json")
|
||||
version = r.json().get("info", {}).get("version", "")
|
||||
if not version:
|
||||
raise ValueError("Failed to fetch package info from PyPI")
|
||||
latest = packaging.version.parse(version)
|
||||
current = packaging.version.parse(importlib.metadata.version(package_name))
|
||||
if current < latest:
|
||||
if not quiet:
|
||||
msg = [
|
||||
f"\u26A0\uFE0F {package_name} is out of date",
|
||||
f"Current version: {current} | Latest: {latest}",
|
||||
"To upgrade: pip install -U --force-reinstall llama-cloud-services",
|
||||
]
|
||||
print(os.linesep.join(msg))
|
||||
return True
|
||||
elif not quiet:
|
||||
print(f"{package_name} is up to date")
|
||||
return False
|
||||
|
||||
@@ -11,13 +11,13 @@ dev = [
|
||||
|
||||
[project]
|
||||
name = "llama-parse"
|
||||
version = "0.6.55"
|
||||
version = "0.6.53"
|
||||
description = "Parse files into RAG-Optimized formats."
|
||||
authors = [{name = "Logan Markewich", email = "logan@llamaindex.ai"}]
|
||||
requires-python = ">=3.9,<4.0"
|
||||
readme = "README.md"
|
||||
license = "MIT"
|
||||
dependencies = ["llama-cloud-services>=0.6.55"]
|
||||
dependencies = ["llama-cloud-services>=0.6.53"]
|
||||
|
||||
[project.scripts]
|
||||
llama-parse = "llama_parse.cli.main:parse"
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"name": "@llama_cloud_services/py",
|
||||
"version": "0.6.55",
|
||||
"private": "true",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
"version": "cd .. && pnpm changeset version && bash py/scripts/new_version.sh",
|
||||
"release": "bash scripts/release.sh"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@changesets/cli": "^2.29.5",
|
||||
"changesets": "^1.0.2"
|
||||
}
|
||||
}
|
||||
+3
-4
@@ -17,7 +17,7 @@ dev = [
|
||||
|
||||
[project]
|
||||
name = "llama-cloud-services"
|
||||
version = "0.6.55"
|
||||
version = "0.6.53"
|
||||
description = "Tailored SDK clients for LlamaCloud services."
|
||||
authors = [{name = "Logan Markewich", email = "logan@runllama.ai"}]
|
||||
requires-python = ">=3.9,<4.0"
|
||||
@@ -25,14 +25,13 @@ readme = "README.md"
|
||||
license = "MIT"
|
||||
dependencies = [
|
||||
"llama-index-core>=0.12.0",
|
||||
"llama-cloud==0.1.37",
|
||||
"llama-cloud==0.1.35",
|
||||
"pydantic>=2.8,!=2.10",
|
||||
"click>=8.1.7,<9",
|
||||
"python-dotenv>=1.0.1,<2",
|
||||
"eval-type-backport>=0.2.0,<0.3 ; python_version < '3.10'",
|
||||
"platformdirs>=4.3.7,<5",
|
||||
"tenacity>=8.5.0, <10.0",
|
||||
"packaging>=25.0"
|
||||
"tenacity>=8.5.0, <10.0"
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
import toml # type: ignore [import]
|
||||
import json
|
||||
|
||||
|
||||
def change_parse_dep(data: dict, version: str) -> dict:
|
||||
dep, ver = (
|
||||
data["project"]["dependencies"][0].split(">=")[0],
|
||||
data["project"]["dependencies"][0].split(">=")[1],
|
||||
)
|
||||
ver = version
|
||||
dependency = dep + ">=" + ver
|
||||
data["project"]["dependencies"].insert(0, dependency)
|
||||
data["project"]["dependencies"].pop(1)
|
||||
return data
|
||||
|
||||
|
||||
with open("py/package.json", "r") as p:
|
||||
package_data = json.load(p)
|
||||
|
||||
with open("py/pyproject.toml", "r") as t:
|
||||
toml_data = toml.load(t)
|
||||
|
||||
with open("py/llama_parse/pyproject.toml", "r") as pt:
|
||||
parse_toml_data = toml.load(pt)
|
||||
|
||||
toml_data["project"]["version"] = package_data["version"]
|
||||
parse_toml_data["project"]["version"] = package_data["version"]
|
||||
parse_toml_data = change_parse_dep(parse_toml_data, package_data["version"])
|
||||
|
||||
with open("py/pyproject.toml", "w") as w:
|
||||
toml.dump(toml_data, w)
|
||||
|
||||
with open("py/llama_parse/pyproject.toml", "w") as pw:
|
||||
toml.dump(parse_toml_data, pw)
|
||||
@@ -1,22 +0,0 @@
|
||||
# create uv virtual environment
|
||||
uv venv
|
||||
source .venv/bin/activate
|
||||
|
||||
# install toml
|
||||
uv pip install -r py/scripts/requirements.txt
|
||||
|
||||
# run version change
|
||||
uv run -- python3 py/scripts/new_version.py
|
||||
|
||||
# test version change
|
||||
status_code=$(uv run -- python3 py/scripts/test_new_version.py) # returns 0 if the versions are the same, 1 if they are not
|
||||
|
||||
if [ "$status_code" -eq 1 ]; then
|
||||
echo "Versions do not match, the version change failed..."
|
||||
exit 1
|
||||
elif [ "$status_code" -eq 0 ]; then
|
||||
# lock the version changes
|
||||
cd py/ && uv lock
|
||||
echo "Versions successfully changed"
|
||||
exit 0
|
||||
fi
|
||||
@@ -1,21 +0,0 @@
|
||||
import toml # type: ignore [import]
|
||||
import requests
|
||||
|
||||
|
||||
def get_latest_version(package_name: str) -> str:
|
||||
url = f"https://pypi.org/pypi/{package_name}/json"
|
||||
response = requests.get(url)
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return data["info"]["version"]
|
||||
else:
|
||||
raise Exception(f"Package '{package_name}' not found on PyPI.")
|
||||
|
||||
|
||||
with open("pyproject.toml", "r") as t:
|
||||
toml_data = toml.load(t)
|
||||
|
||||
if toml_data["project"]["version"] == get_latest_version("llama-cloud-services"):
|
||||
print("1")
|
||||
else:
|
||||
print("0")
|
||||
@@ -1,36 +0,0 @@
|
||||
# initialize pypi_token
|
||||
pypi_token="no_token"
|
||||
|
||||
while
|
||||
[[ $# -gt 0 ]] \
|
||||
;
|
||||
do
|
||||
case "$1" in
|
||||
-t | --token)
|
||||
pypi_token="$2"
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ $pypi_token == "no_token" ]]; then
|
||||
if [[ $LLAMA_PARSE_PYPI_TOKEN == "" ]]; then
|
||||
echo "No token provided and no token in the environment, exiting..."
|
||||
exit 1
|
||||
else
|
||||
pypi_token="$LLAMA_PARSE_PYPI_TOKEN"
|
||||
fi
|
||||
fi
|
||||
|
||||
# check pre-release
|
||||
uv venv && source .venv/bin/activate && uv pip install -r scripts/requirements.txt
|
||||
do_not_release=$(python3 scripts/pre_release_check.py)
|
||||
|
||||
if [ "$do_not_release" -eq 1 ]; then
|
||||
echo "Nothing to release"
|
||||
exit 0
|
||||
else
|
||||
# build and publish llama_cloud_services
|
||||
## build
|
||||
uv build
|
||||
## publish
|
||||
uv publish --token $pypi_token
|
||||
fi
|
||||
@@ -1,2 +0,0 @@
|
||||
toml
|
||||
requests
|
||||
@@ -1,32 +0,0 @@
|
||||
import toml # type: ignore [import]
|
||||
import json
|
||||
|
||||
|
||||
def test_versions_equal(
|
||||
package_data: dict, toml_data: dict, parse_toml_data: dict
|
||||
) -> None:
|
||||
llama_cloud_version = toml_data["project"]["version"]
|
||||
package_version = package_data["version"]
|
||||
parse_version = parse_toml_data["project"]["version"]
|
||||
parse_dep_version = parse_toml_data["project"]["dependencies"][0].split(">=")[1]
|
||||
try:
|
||||
assert (
|
||||
llama_cloud_version == package_version
|
||||
and llama_cloud_version == parse_version
|
||||
and llama_cloud_version == parse_dep_version
|
||||
)
|
||||
print("0")
|
||||
except AssertionError:
|
||||
print("1")
|
||||
|
||||
|
||||
with open("py/package.json", "r") as p:
|
||||
package_data = json.load(p)
|
||||
|
||||
with open("py/pyproject.toml", "r") as t:
|
||||
toml_data = toml.load(t)
|
||||
|
||||
with open("py/llama_parse/pyproject.toml", "r") as pt:
|
||||
parse_toml_data = toml.load(pt)
|
||||
|
||||
test_versions_equal(package_data, toml_data, parse_toml_data)
|
||||
@@ -37,7 +37,7 @@ LLAMA_CLOUD_BASE_URL = os.getenv("LLAMA_CLOUD_BASE_URL")
|
||||
LLAMA_DEPLOY_DEPLOYMENT_NAME = os.getenv("LLAMA_DEPLOY_DEPLOYMENT_NAME")
|
||||
|
||||
|
||||
class ExampleData(BaseModel):
|
||||
class TestData(BaseModel):
|
||||
"""Simple test data model for agent data testing"""
|
||||
|
||||
name: str
|
||||
@@ -66,13 +66,13 @@ async def test_agent_data_crud_operations():
|
||||
# Create agent data client with unique collection name
|
||||
agent_data_client = AsyncAgentDataClient(
|
||||
client=client,
|
||||
type=ExampleData,
|
||||
type=TestData,
|
||||
collection=f"test-collection-{test_id[:8]}",
|
||||
agent_url_id=LLAMA_DEPLOY_DEPLOYMENT_NAME,
|
||||
)
|
||||
|
||||
# Create test data
|
||||
test_data = ExampleData(name="test-item", test_id=test_id, value=42)
|
||||
test_data = TestData(name="test-item", test_id=test_id, value=42)
|
||||
|
||||
created_item = None
|
||||
try:
|
||||
@@ -107,7 +107,7 @@ async def test_agent_data_crud_operations():
|
||||
assert aggregate_results.items[0].count == 1
|
||||
|
||||
# Test UPDATE
|
||||
updated_data = ExampleData(name="updated-item", test_id=test_id, value=84)
|
||||
updated_data = TestData(name="updated-item", test_id=test_id, value=84)
|
||||
updated_item = await agent_data_client.update_item(
|
||||
created_item.id, updated_data
|
||||
)
|
||||
|
||||
+6
-110
@@ -230,20 +230,20 @@ def test_parse_extracted_field_metadata():
|
||||
raw_metadata = {
|
||||
"name": {
|
||||
"confidence": 0.95,
|
||||
"citation": [{"page": 1, "matching_text": "John Smith"}],
|
||||
"citations": [{"page_number": 1, "matching_text": "John Smith"}],
|
||||
},
|
||||
"age": {
|
||||
"confidence": 0.87,
|
||||
"citation": [
|
||||
"citations": [
|
||||
{
|
||||
"page": 2.0, # Float page number
|
||||
"page_number": 2.0, # Float page number
|
||||
"matching_text": "25 years old",
|
||||
}
|
||||
],
|
||||
},
|
||||
"email": {
|
||||
"confidence": 0.92,
|
||||
"citation": [], # Empty citations
|
||||
"citations": [], # Empty citations
|
||||
},
|
||||
}
|
||||
|
||||
@@ -268,110 +268,6 @@ def test_parse_extracted_field_metadata():
|
||||
assert result["email"].confidence == 0.92
|
||||
|
||||
|
||||
def test_parse_extracted_field_metadata_complex():
|
||||
"""Test parse_extracted_field_metadata with new citation format and reasoning field."""
|
||||
raw_metadata = {
|
||||
"title": {
|
||||
"reasoning": "Combined key parametrics and construction from the datasheet for a structured title.",
|
||||
"citation": [
|
||||
{
|
||||
"page": 1,
|
||||
"matching_text": "PHE844/F844, Film, Metallized Polypropylene, Safety, 0.47 uF",
|
||||
}
|
||||
],
|
||||
"extraction_confidence": 0.9470628580889779,
|
||||
"confidence": 0.9470628580889779,
|
||||
},
|
||||
"manufacturer": {
|
||||
"reasoning": "VERBATIM EXTRACTION",
|
||||
"citation": [{"page": 1, "matching_text": "YAGEO KEMET"}],
|
||||
"extraction_confidence": 0.9997446550976602,
|
||||
"confidence": 0.9997446550976602,
|
||||
},
|
||||
"features": [
|
||||
{
|
||||
"reasoning": "VERBATIM EXTRACTION",
|
||||
"citation": [
|
||||
{"page": 1, "matching_text": "Features</td><td>EMI Safety"}
|
||||
],
|
||||
"extraction_confidence": 0.9999308195540074,
|
||||
"confidence": 0.9999308195540074,
|
||||
},
|
||||
{
|
||||
"reasoning": "VERBATIM EXTRACTION",
|
||||
"citation": [
|
||||
{"page": 1, "matching_text": "THB Performance</td><td>Yes"}
|
||||
],
|
||||
"extraction_confidence": 0.8642493886452225,
|
||||
"confidence": 0.8642493886452225,
|
||||
},
|
||||
],
|
||||
"dimensions": {
|
||||
"length": {
|
||||
"citation": [{"page": 1, "matching_text": "L</td><td>41mm MAX"}],
|
||||
"extraction_confidence": 0.8986941382802304,
|
||||
"confidence": 0.8986941382802304,
|
||||
},
|
||||
"width": {
|
||||
"citation": [{"page": 1, "matching_text": "T</td><td>13mm MAX"}],
|
||||
"extraction_confidence": 0.9999377974447091,
|
||||
"confidence": 0.9999377974447091,
|
||||
},
|
||||
"reasoning": "VERBATIM EXTRACTION",
|
||||
},
|
||||
}
|
||||
|
||||
result = parse_extracted_field_metadata(raw_metadata)
|
||||
assert result == {
|
||||
"title": ExtractedFieldMetadata(
|
||||
reasoning="Combined key parametrics and construction from the datasheet for a structured title.",
|
||||
confidence=0.9470628580889779,
|
||||
extraction_confidence=0.9470628580889779,
|
||||
page_number=1,
|
||||
matching_text="PHE844/F844, Film, Metallized Polypropylene, Safety, 0.47 uF",
|
||||
),
|
||||
"manufacturer": ExtractedFieldMetadata(
|
||||
reasoning="VERBATIM EXTRACTION",
|
||||
confidence=0.9997446550976602,
|
||||
extraction_confidence=0.9997446550976602,
|
||||
page_number=1,
|
||||
matching_text="YAGEO KEMET",
|
||||
),
|
||||
"features": [
|
||||
ExtractedFieldMetadata(
|
||||
reasoning="VERBATIM EXTRACTION",
|
||||
confidence=0.9999308195540074,
|
||||
extraction_confidence=0.9999308195540074,
|
||||
page_number=1,
|
||||
matching_text="Features</td><td>EMI Safety",
|
||||
),
|
||||
ExtractedFieldMetadata(
|
||||
reasoning="VERBATIM EXTRACTION",
|
||||
confidence=0.8642493886452225,
|
||||
extraction_confidence=0.8642493886452225,
|
||||
page_number=1,
|
||||
matching_text="THB Performance</td><td>Yes",
|
||||
),
|
||||
],
|
||||
"dimensions": {
|
||||
"length": ExtractedFieldMetadata(
|
||||
reasoning="VERBATIM EXTRACTION",
|
||||
confidence=0.8986941382802304,
|
||||
extraction_confidence=0.8986941382802304,
|
||||
page_number=1,
|
||||
matching_text="L</td><td>41mm MAX",
|
||||
),
|
||||
"width": ExtractedFieldMetadata(
|
||||
reasoning="VERBATIM EXTRACTION",
|
||||
confidence=0.9999377974447091,
|
||||
extraction_confidence=0.9999377974447091,
|
||||
page_number=1,
|
||||
matching_text="T</td><td>13mm MAX",
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def create_file(
|
||||
id: str = "file-456",
|
||||
name: str = "resume.pdf",
|
||||
@@ -394,12 +290,12 @@ def create_extract_run(
|
||||
extraction_metadata: Dict[str, Any] = {
|
||||
"name": {
|
||||
"confidence": 0.95,
|
||||
"citation": [{"page": 1, "matching_text": "John Doe"}],
|
||||
"citations": [{"page_number": 1, "matching_text": "John Doe"}],
|
||||
},
|
||||
"age": {"confidence": 0.87},
|
||||
"email": {
|
||||
"confidence": 0.92,
|
||||
"citation": [{"page": 1, "matching_text": "john@example.com"}],
|
||||
"citations": [{"page_number": 1, "matching_text": "john@example.com"}],
|
||||
},
|
||||
},
|
||||
data_schema: Dict[str, Any] = {},
|
||||
@@ -6,12 +6,6 @@ from llama_cloud_services.extract import LlamaExtract
|
||||
_TEST_AGENTS_TO_CLEANUP: List[str] = []
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
"""Register custom markers for extract tests."""
|
||||
config.addinivalue_line("markers", "agent_name: custom agent name for test")
|
||||
config.addinivalue_line("markers", "agent_schema: custom agent schema for test")
|
||||
|
||||
|
||||
def pytest_sessionfinish(session, exitstatus):
|
||||
"""Hook that runs after all tests complete - cleanup agents here"""
|
||||
print(
|
||||
|
||||
@@ -23,9 +23,8 @@ LLAMA_CLOUD_API_KEY = os.getenv("LLAMA_CLOUD_API_KEY")
|
||||
LLAMA_CLOUD_BASE_URL = os.getenv("LLAMA_CLOUD_BASE_URL")
|
||||
LLAMA_CLOUD_PROJECT_ID = os.getenv("LLAMA_CLOUD_PROJECT_ID")
|
||||
|
||||
BenchmarkTestCase = namedtuple(
|
||||
"BenchmarkTestCase",
|
||||
["name", "schema_path", "config", "input_file", "expected_output"],
|
||||
TestCase = namedtuple(
|
||||
"TestCase", ["name", "schema_path", "config", "input_file", "expected_output"]
|
||||
)
|
||||
|
||||
|
||||
@@ -33,7 +32,7 @@ def get_test_cases():
|
||||
"""Get all test cases from TEST_DIR.
|
||||
|
||||
Returns:
|
||||
List[BenchmarkTestCase]: List of test cases
|
||||
List[TestCase]: List of test cases
|
||||
"""
|
||||
test_cases = []
|
||||
|
||||
@@ -74,7 +73,7 @@ def get_test_cases():
|
||||
test_name = f"{data_type}/{os.path.basename(input_file)}"
|
||||
for setting in settings:
|
||||
test_cases.append(
|
||||
BenchmarkTestCase(
|
||||
TestCase(
|
||||
name=test_name,
|
||||
schema_path=schema_path,
|
||||
input_file=input_file,
|
||||
@@ -101,7 +100,7 @@ def extractor():
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def extraction_agent(test_case: BenchmarkTestCase, extractor: LlamaExtract):
|
||||
def extraction_agent(test_case: TestCase, extractor: LlamaExtract):
|
||||
"""Fixture to create and cleanup extraction agent for each test."""
|
||||
# Create unique name with random UUID (important for CI to avoid conflicts)
|
||||
unique_id = uuid.uuid4().hex[:8]
|
||||
@@ -125,13 +124,13 @@ def extraction_agent(test_case: BenchmarkTestCase, extractor: LlamaExtract):
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
"CI" in os.environ or not LLAMA_CLOUD_API_KEY,
|
||||
reason="LLAMA_CLOUD_API_KEY not set or CI environment not suitable for benchmarking",
|
||||
"CI" in os.environ,
|
||||
reason="CI environment is not suitable for benchmarking",
|
||||
)
|
||||
@pytest.mark.parametrize("test_case", get_test_cases(), ids=lambda x: x.name)
|
||||
@pytest.mark.asyncio(loop_scope="session")
|
||||
async def test_extraction(
|
||||
test_case: BenchmarkTestCase, extraction_agent: ExtractionAgent
|
||||
test_case: TestCase, extraction_agent: ExtractionAgent
|
||||
) -> None:
|
||||
start = perf_counter()
|
||||
result = await extraction_agent._run_extraction_test(
|
||||
|
||||
@@ -4,7 +4,6 @@ from pathlib import Path
|
||||
from pydantic import BaseModel
|
||||
|
||||
from llama_cloud_services.extract import LlamaExtract, ExtractionAgent, SourceText
|
||||
from llama_cloud.types import ExtractConfig, ExtractMode, ExtractRun
|
||||
from tests.extract.util import load_test_dotenv
|
||||
from .conftest import register_agent_for_cleanup
|
||||
|
||||
@@ -22,7 +21,7 @@ pytestmark = pytest.mark.skipif(
|
||||
|
||||
|
||||
# Test data
|
||||
class ExampleSchema(BaseModel):
|
||||
class TestSchema(BaseModel):
|
||||
title: str
|
||||
summary: str
|
||||
|
||||
@@ -108,7 +107,7 @@ class TestLlamaExtract:
|
||||
assert isinstance(test_agent, ExtractionAgent)
|
||||
|
||||
@pytest.mark.agent_name("test-pydantic-schema-agent")
|
||||
@pytest.mark.agent_schema((ExampleSchema,))
|
||||
@pytest.mark.agent_schema((TestSchema,))
|
||||
def test_create_agent_with_pydantic_schema(self, test_agent):
|
||||
assert isinstance(test_agent, ExtractionAgent)
|
||||
|
||||
@@ -223,189 +222,7 @@ class TestExtractionAgent:
|
||||
|
||||
def test_delete_extraction_run(self, test_agent: ExtractionAgent):
|
||||
assert test_agent.list_extraction_runs().total == 0
|
||||
run: ExtractRun = test_agent.extract(TEST_PDF)
|
||||
run = test_agent.extract(TEST_PDF)
|
||||
test_agent.delete_extraction_run(run.id)
|
||||
runs = test_agent.list_extraction_runs()
|
||||
assert runs.total == 0
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
"CI" in os.environ, reason="Test locally; functionality is mostly duplicated."
|
||||
)
|
||||
class TestStatelessExtraction:
|
||||
"""Tests for stateless extraction methods that don't require creating an agent."""
|
||||
|
||||
@pytest.fixture
|
||||
def test_config(self):
|
||||
return ExtractConfig(extraction_mode=ExtractMode.FAST)
|
||||
|
||||
@pytest.fixture
|
||||
def test_schema_dict(self):
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string"},
|
||||
"summary": {"type": "string"},
|
||||
},
|
||||
}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aextract_single_file(
|
||||
self, llama_extract, test_schema_dict, test_config
|
||||
):
|
||||
"""Test async stateless extraction with a single file."""
|
||||
result = await llama_extract.aextract(test_schema_dict, test_config, TEST_PDF)
|
||||
assert result.status == "SUCCESS"
|
||||
assert result.data is not None
|
||||
assert isinstance(result.data, dict)
|
||||
assert "title" in result.data
|
||||
assert "summary" in result.data
|
||||
|
||||
def test_extract_single_file(self, llama_extract, test_schema_dict, test_config):
|
||||
"""Test synchronous stateless extraction with a single file."""
|
||||
result = llama_extract.extract(test_schema_dict, test_config, TEST_PDF)
|
||||
assert result.status == "SUCCESS"
|
||||
assert result.data is not None
|
||||
assert isinstance(result.data, dict)
|
||||
assert "title" in result.data
|
||||
assert "summary" in result.data
|
||||
|
||||
def test_extract_from_bytes_with_source_text(
|
||||
self, llama_extract, test_schema_dict, test_config
|
||||
):
|
||||
"""Test stateless extraction from bytes using SourceText with filename."""
|
||||
with open(TEST_PDF, "rb") as f:
|
||||
file_bytes = f.read()
|
||||
source_text = SourceText(file=file_bytes, filename=TEST_PDF.name)
|
||||
result = llama_extract.extract(test_schema_dict, test_config, source_text)
|
||||
assert result.status == "SUCCESS"
|
||||
assert result.data is not None
|
||||
assert isinstance(result.data, dict)
|
||||
assert "title" in result.data
|
||||
assert "summary" in result.data
|
||||
|
||||
def test_extract_from_source_text_with_file(
|
||||
self, llama_extract, test_schema_dict, test_config
|
||||
):
|
||||
"""Test stateless extraction from SourceText with file."""
|
||||
source_text = SourceText(file=TEST_PDF, filename=TEST_PDF.name)
|
||||
result = llama_extract.extract(test_schema_dict, test_config, source_text)
|
||||
assert result.status == "SUCCESS"
|
||||
assert result.data is not None
|
||||
assert isinstance(result.data, dict)
|
||||
assert "title" in result.data
|
||||
assert "summary" in result.data
|
||||
|
||||
def test_extract_from_buffered_io(
|
||||
self, llama_extract, test_schema_dict, test_config
|
||||
):
|
||||
"""Test stateless extraction from BufferedIO file handle."""
|
||||
with open(TEST_PDF, "rb") as file_handle:
|
||||
result = llama_extract.extract(test_schema_dict, test_config, file_handle)
|
||||
assert result.status == "SUCCESS"
|
||||
assert result.data is not None
|
||||
assert isinstance(result.data, dict)
|
||||
assert "title" in result.data
|
||||
assert "summary" in result.data
|
||||
|
||||
def test_extract_from_source_text_with_text_content(
|
||||
self, llama_extract, test_schema_dict, test_config
|
||||
):
|
||||
"""Test stateless extraction from SourceText with text content."""
|
||||
TEST_TEXT = """
|
||||
# Llamas
|
||||
Llamas are social animals and live with others as a herd. Their wool is soft and
|
||||
contains only a small amount of lanolin. Llamas can learn simple tasks after a
|
||||
few repetitions. When using a pack, they can carry about 25 to 30% of their body
|
||||
weight for 8 to 13 km (5–8 miles). The name llama was adopted by European settlers
|
||||
from native Peruvians.
|
||||
"""
|
||||
source_text = SourceText(text_content=TEST_TEXT)
|
||||
result = llama_extract.extract(test_schema_dict, test_config, source_text)
|
||||
assert result.status == "SUCCESS"
|
||||
assert result.data is not None
|
||||
assert isinstance(result.data, dict)
|
||||
assert "title" in result.data
|
||||
assert "summary" in result.data
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_queue_extraction_single_file(
|
||||
self, llama_extract, test_schema_dict, test_config
|
||||
):
|
||||
"""Test queuing extraction job without waiting for completion."""
|
||||
job = await llama_extract.queue_extraction(
|
||||
test_schema_dict, test_config, TEST_PDF
|
||||
)
|
||||
assert hasattr(job, "id")
|
||||
assert hasattr(job, "status")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_multiple_files(
|
||||
self, llama_extract, test_schema_dict, test_config
|
||||
):
|
||||
"""Test stateless extraction with multiple files."""
|
||||
files = [TEST_PDF, TEST_PDF] # Using same file twice for testing
|
||||
results = await llama_extract.aextract(test_schema_dict, test_config, files)
|
||||
|
||||
assert isinstance(results, list)
|
||||
assert len(results) == 2
|
||||
|
||||
for result in results:
|
||||
assert result.status == "SUCCESS"
|
||||
assert result.data is not None
|
||||
assert isinstance(result.data, dict)
|
||||
assert "title" in result.data
|
||||
assert "summary" in result.data
|
||||
|
||||
def test_extract_with_pydantic_schema(self, llama_extract, test_config):
|
||||
"""Test stateless extraction with Pydantic schema."""
|
||||
result = llama_extract.extract(ExampleSchema, test_config, TEST_PDF)
|
||||
assert result.status == "SUCCESS"
|
||||
assert result.data is not None
|
||||
assert isinstance(result.data, dict)
|
||||
assert "title" in result.data
|
||||
assert "summary" in result.data
|
||||
|
||||
def test_extract_from_raw_bytes_raises_error(
|
||||
self, llama_extract, test_schema_dict, test_config
|
||||
):
|
||||
"""Test that raw bytes without filename raises an error."""
|
||||
with open(TEST_PDF, "rb") as f:
|
||||
file_bytes = f.read()
|
||||
|
||||
with pytest.raises(
|
||||
ValueError, match="Cannot determine file type from raw bytes"
|
||||
):
|
||||
llama_extract.extract(test_schema_dict, test_config, file_bytes)
|
||||
|
||||
def test_mime_type_detection(self, llama_extract):
|
||||
"""Test that MIME types are correctly detected for various file types."""
|
||||
# Test PDF
|
||||
assert llama_extract._get_mime_type(filename="test.pdf") == "application/pdf"
|
||||
|
||||
# Test DOCX
|
||||
assert (
|
||||
llama_extract._get_mime_type(filename="test.docx")
|
||||
== "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
|
||||
)
|
||||
|
||||
# Test text files
|
||||
assert llama_extract._get_mime_type(filename="test.txt") == "text/plain"
|
||||
assert llama_extract._get_mime_type(filename="test.csv") == "text/csv"
|
||||
assert llama_extract._get_mime_type(filename="test.json") == "application/json"
|
||||
|
||||
# Test image files
|
||||
assert llama_extract._get_mime_type(filename="test.png") == "image/png"
|
||||
assert llama_extract._get_mime_type(filename="test.jpg") == "image/jpeg"
|
||||
|
||||
# Test file path
|
||||
from pathlib import Path
|
||||
|
||||
assert (
|
||||
llama_extract._get_mime_type(file_path=Path("test.pdf"))
|
||||
== "application/pdf"
|
||||
)
|
||||
|
||||
# Test unsupported file type
|
||||
with pytest.raises(ValueError, match="Unsupported file type: 'xyz'"):
|
||||
llama_extract._get_mime_type(filename="test.xyz")
|
||||
|
||||
@@ -19,9 +19,8 @@ LLAMA_CLOUD_API_KEY = os.getenv("LLAMA_CLOUD_API_KEY")
|
||||
LLAMA_CLOUD_BASE_URL = os.getenv("LLAMA_CLOUD_BASE_URL")
|
||||
LLAMA_CLOUD_PROJECT_ID = os.getenv("LLAMA_CLOUD_PROJECT_ID")
|
||||
|
||||
ExtractionTestCase = namedtuple(
|
||||
"ExtractionTestCase",
|
||||
["name", "schema_path", "config", "input_file", "expected_output"],
|
||||
TestCase = namedtuple(
|
||||
"TestCase", ["name", "schema_path", "config", "input_file", "expected_output"]
|
||||
)
|
||||
|
||||
|
||||
@@ -29,7 +28,7 @@ def get_test_cases():
|
||||
"""Get all test cases from TEST_DIR.
|
||||
|
||||
Returns:
|
||||
List[ExtractionTestCase]: List of test cases
|
||||
List[TestCase]: List of test cases
|
||||
"""
|
||||
test_cases = []
|
||||
|
||||
@@ -70,7 +69,7 @@ def get_test_cases():
|
||||
test_name = f"{data_type}/{os.path.basename(input_file)}"
|
||||
for setting in settings:
|
||||
test_cases.append(
|
||||
ExtractionTestCase(
|
||||
TestCase(
|
||||
name=test_name,
|
||||
schema_path=schema_path,
|
||||
input_file=input_file,
|
||||
@@ -97,7 +96,7 @@ def extractor():
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def extraction_agent(test_case: ExtractionTestCase, extractor: LlamaExtract):
|
||||
def extraction_agent(test_case: TestCase, extractor: LlamaExtract):
|
||||
"""Fixture to create and cleanup extraction agent for each test."""
|
||||
# Create unique name with random UUID (important for CI to avoid conflicts)
|
||||
unique_id = uuid.uuid4().hex[:8]
|
||||
@@ -129,9 +128,7 @@ def extraction_agent(test_case: ExtractionTestCase, extractor: LlamaExtract):
|
||||
reason="LLAMA_CLOUD_API_KEY not set",
|
||||
)
|
||||
@pytest.mark.parametrize("test_case", get_test_cases(), ids=lambda x: x.name)
|
||||
def test_extraction(
|
||||
test_case: ExtractionTestCase, extraction_agent: ExtractionAgent
|
||||
) -> None:
|
||||
def test_extraction(test_case: TestCase, extraction_agent: ExtractionAgent) -> None:
|
||||
result = extraction_agent.extract(test_case.input_file).data # type: ignore
|
||||
with open(test_case.expected_output, "r") as f:
|
||||
expected = json.load(f)
|
||||
|
||||
@@ -16,6 +16,7 @@ from llama_cloud import (
|
||||
from llama_cloud.client import LlamaCloud
|
||||
from llama_index.core.bridge.pydantic import BaseModel
|
||||
from llama_index.core.constants import DEFAULT_BASE_URL
|
||||
from llama_index.core.indices.managed.base import BaseManagedIndex
|
||||
from llama_index.core.schema import Document, ImageNode
|
||||
from llama_cloud_services.index import (
|
||||
LlamaCloudIndex,
|
||||
@@ -27,8 +28,6 @@ api_key = os.environ.get("LLAMA_CLOUD_API_KEY", None)
|
||||
organization_id = os.environ.get("LLAMA_CLOUD_ORGANIZATION_ID", None)
|
||||
project_name = os.environ.get("LLAMA_CLOUD_PROJECT_NAME", "framework_integration_test")
|
||||
|
||||
print("api-key", api_key, "base-url", base_url)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def remote_file() -> Tuple[str, str]:
|
||||
@@ -92,6 +91,16 @@ def _setup_index_with_file(
|
||||
return pipeline
|
||||
|
||||
|
||||
def test_class():
|
||||
names_of_base_classes = [b.__name__ for b in LlamaCloudIndex.__mro__]
|
||||
assert BaseManagedIndex.__name__ in names_of_base_classes
|
||||
|
||||
|
||||
def test_conflicting_index_identifiers():
|
||||
with pytest.raises(ValueError):
|
||||
LlamaCloudIndex(name="test", pipeline_id="test", index_id="test")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not base_url or not api_key, reason="No platform base url or api key set"
|
||||
)
|
||||
@@ -353,9 +362,6 @@ async def test_page_figure_retrieval(index_name: str, local_figures_file: str):
|
||||
not base_url or not api_key, reason="No platform base url or api key set"
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skip(
|
||||
reason="Consistently failing with 'Server disconnected without sending a response'"
|
||||
)
|
||||
async def test_composite_retriever(index_name: str):
|
||||
"""Test the LlamaCloudCompositeRetriever with multiple indices."""
|
||||
# Create first index with documents
|
||||
|
||||
@@ -1,9 +1,5 @@
|
||||
from unittest.mock import Mock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from pydantic import BaseModel
|
||||
from llama_cloud_services.utils import check_extra_params, check_for_updates
|
||||
from llama_cloud_services.utils import check_extra_params
|
||||
|
||||
|
||||
class MyModel(BaseModel):
|
||||
@@ -68,30 +64,3 @@ def test_check_extra_params_completely_invalid():
|
||||
for suggestion in suggestions:
|
||||
assert "check the documentation or update the package" in suggestion
|
||||
assert "Did you mean" not in suggestion
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_for_updates(capsys: pytest.CaptureFixture):
|
||||
"""Test update checker."""
|
||||
|
||||
mock_response = Mock()
|
||||
mock_client = Mock(spec=httpx.AsyncClient)
|
||||
mock_client.get.return_value = mock_response
|
||||
|
||||
mock_response.json.return_value = {"info": {"version": "0.0.0"}}
|
||||
assert not await check_for_updates(mock_client)
|
||||
out, err = capsys.readouterr()
|
||||
assert not out and not err
|
||||
|
||||
assert not await check_for_updates(mock_client, quiet=False)
|
||||
out, _ = capsys.readouterr()
|
||||
assert "up to date" in out
|
||||
|
||||
mock_response.json.return_value = {"info": {"version": "999.0.0"}}
|
||||
assert await check_for_updates(mock_client)
|
||||
out, err = capsys.readouterr()
|
||||
assert not out and not err
|
||||
|
||||
assert await check_for_updates(mock_client, quiet=False)
|
||||
out, _ = capsys.readouterr()
|
||||
assert "out of date" in out
|
||||
@@ -1,16 +0,0 @@
|
||||
import pytest
|
||||
|
||||
from llama_index.core.indices.managed.base import BaseManagedIndex
|
||||
from llama_cloud_services.index import (
|
||||
LlamaCloudIndex,
|
||||
)
|
||||
|
||||
|
||||
def test_class():
|
||||
names_of_base_classes = [b.__name__ for b in LlamaCloudIndex.__mro__]
|
||||
assert BaseManagedIndex.__name__ in names_of_base_classes
|
||||
|
||||
|
||||
def test_conflicting_index_identifiers():
|
||||
with pytest.raises(ValueError):
|
||||
LlamaCloudIndex(name="test", pipeline_id="test", index_id="test")
|
||||
Generated
+6
-8
@@ -734,7 +734,7 @@ name = "exceptiongroup"
|
||||
version = "1.3.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.11'" },
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.13'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" }
|
||||
wheels = [
|
||||
@@ -1573,21 +1573,21 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "llama-cloud"
|
||||
version = "0.1.37"
|
||||
version = "0.1.35"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "certifi" },
|
||||
{ name = "httpx" },
|
||||
{ name = "pydantic" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/9c/dd/c4f2516523778a0d4b284fa4b66a0136afdd59694f105102838cf793e675/llama_cloud-0.1.37.tar.gz", hash = "sha256:b6d62e7386d1aa85905b7e3f7c19a40694be54c1596668bceaa456cd84ede666", size = 108707, upload-time = "2025-08-04T22:00:37.18Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/9b/72/816e6e900448e1b4a8137d90e65876b296c5264a23db6ae888bd3e6660ba/llama_cloud-0.1.35.tar.gz", hash = "sha256:200349d5d57424d7461f304cdb1355a58eea3e6ca1e6b0d75c66b2e937216983", size = 106403, upload-time = "2025-07-28T17:22:06.41Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/59/ca/a7f874b041d2566f000ecc88bf1b76819a8bdbbaea85036ca6905ae3f0f7/llama_cloud-0.1.37-py3-none-any.whl", hash = "sha256:3109ec74575f53311ec4957ca8aea2d6e30556a43d24c6316ceab91c4fed6ab7", size = 314244, upload-time = "2025-08-04T22:00:35.696Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1d/d2/8d18a021ab757cea231428404f21fe3186bf1ebaac3f57a73c379483fd3f/llama_cloud-0.1.35-py3-none-any.whl", hash = "sha256:b7abab4423118e6f638d2f326749e7a07c6426543bea6da99b623c715b22af71", size = 303280, upload-time = "2025-07-28T17:22:04.946Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "llama-cloud-services"
|
||||
version = "0.6.54"
|
||||
version = "0.6.53"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
|
||||
@@ -1595,7 +1595,6 @@ dependencies = [
|
||||
{ name = "eval-type-backport", marker = "python_full_version < '3.10'" },
|
||||
{ name = "llama-cloud" },
|
||||
{ name = "llama-index-core" },
|
||||
{ name = "packaging" },
|
||||
{ name = "platformdirs" },
|
||||
{ name = "pydantic" },
|
||||
{ name = "python-dotenv" },
|
||||
@@ -1620,9 +1619,8 @@ dev = [
|
||||
requires-dist = [
|
||||
{ name = "click", specifier = ">=8.1.7,<9" },
|
||||
{ name = "eval-type-backport", marker = "python_full_version < '3.10'", specifier = ">=0.2.0,<0.3" },
|
||||
{ name = "llama-cloud", specifier = "==0.1.37" },
|
||||
{ name = "llama-cloud", specifier = "==0.1.35" },
|
||||
{ name = "llama-index-core", specifier = ">=0.12.0" },
|
||||
{ name = "packaging", specifier = ">=25.0" },
|
||||
{ name = "platformdirs", specifier = ">=4.3.7,<5" },
|
||||
{ name = "pydantic", specifier = ">=2.8,!=2.10" },
|
||||
{ name = "python-dotenv", specifier = ">=1.0.1,<2" },
|
||||
|
||||
Regular → Executable
+23
-56
@@ -13,26 +13,17 @@ from pathlib import Path
|
||||
def get_current_versions() -> tuple[str, str, str]:
|
||||
"""Get current versions from both pyproject.toml files."""
|
||||
# Read main pyproject.toml
|
||||
main_content = Path("py/pyproject.toml").read_text()
|
||||
main_content = Path("pyproject.toml").read_text()
|
||||
main_doc = tomlkit.parse(main_content)
|
||||
main_version = main_doc["project"]["version"]
|
||||
main_version = main_doc["tool"]["poetry"]["version"]
|
||||
|
||||
# Read llama_parse/pyproject.toml
|
||||
llama_parse_content = Path("py/llama_parse/pyproject.toml").read_text()
|
||||
llama_parse_content = Path("llama_parse/pyproject.toml").read_text()
|
||||
llama_parse_doc = tomlkit.parse(llama_parse_content)
|
||||
llama_parse_version = llama_parse_doc["project"]["version"]
|
||||
# Find llama-cloud-services dependency in the dependencies list
|
||||
dependency_version = None
|
||||
for dep in llama_parse_doc["project"]["dependencies"]:
|
||||
if isinstance(dep, str) and dep.startswith("llama-cloud-services"):
|
||||
dependency_version = (
|
||||
dep.split("==")[1]
|
||||
if "==" in dep
|
||||
else dep.split(">=")[1]
|
||||
if ">=" in dep
|
||||
else None
|
||||
)
|
||||
break
|
||||
llama_parse_version = llama_parse_doc["tool"]["poetry"]["version"]
|
||||
dependency_version = llama_parse_doc["tool"]["poetry"]["dependencies"][
|
||||
"llama-cloud-services"
|
||||
]
|
||||
|
||||
return str(main_version), str(llama_parse_version), str(dependency_version)
|
||||
|
||||
@@ -62,22 +53,19 @@ def validate_versions(
|
||||
def set_version(version: str) -> None:
|
||||
"""Set version across all pyproject.toml files using tomlkit to preserve formatting."""
|
||||
# Update main pyproject.toml
|
||||
main_content = Path("py/pyproject.toml").read_text()
|
||||
main_content = Path("pyproject.toml").read_text()
|
||||
main_doc = tomlkit.parse(main_content)
|
||||
main_doc["project"]["version"] = version
|
||||
Path("py/pyproject.toml").write_text(tomlkit.dumps(main_doc))
|
||||
main_doc["tool"]["poetry"]["version"] = version
|
||||
Path("pyproject.toml").write_text(tomlkit.dumps(main_doc))
|
||||
|
||||
# Update llama_parse/pyproject.toml
|
||||
llama_parse_content = Path("py/llama_parse/pyproject.toml").read_text()
|
||||
llama_parse_content = Path("llama_parse/pyproject.toml").read_text()
|
||||
llama_parse_doc = tomlkit.parse(llama_parse_content)
|
||||
llama_parse_doc["project"]["version"] = version
|
||||
for dep_index, dep in enumerate(llama_parse_doc["project"]["dependencies"]):
|
||||
if isinstance(dep, str) and dep.startswith("llama-cloud-services"):
|
||||
llama_parse_doc["project"]["dependencies"][
|
||||
dep_index
|
||||
] = f"llama-cloud-services>={version}"
|
||||
break
|
||||
Path("py/llama_parse/pyproject.toml").write_text(tomlkit.dumps(llama_parse_doc))
|
||||
llama_parse_doc["tool"]["poetry"]["version"] = version
|
||||
llama_parse_doc["tool"]["poetry"]["dependencies"][
|
||||
"llama-cloud-services"
|
||||
] = f">={version}"
|
||||
Path("llama_parse/pyproject.toml").write_text(tomlkit.dumps(llama_parse_doc))
|
||||
|
||||
click.echo(f"Updated all versions to {version}")
|
||||
|
||||
@@ -90,7 +78,7 @@ def get_current_branch() -> str:
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def create_if_not_exists(version: str) -> None:
|
||||
def create_and_push_tag(version: str) -> None:
|
||||
"""Create a git tag and push it."""
|
||||
current_branch = get_current_branch()
|
||||
if current_branch != "main":
|
||||
@@ -100,26 +88,12 @@ def create_if_not_exists(version: str) -> None:
|
||||
sys.exit(1)
|
||||
|
||||
tag_name = f"v{version}"
|
||||
if not tag_exists(version):
|
||||
# Create tag
|
||||
subprocess.run(["git", "tag", tag_name], check=True)
|
||||
click.echo(f"Created tag {tag_name}")
|
||||
else:
|
||||
click.echo(f"Tag {tag_name} already exists")
|
||||
|
||||
# Create tag
|
||||
subprocess.run(["git", "tag", tag_name], check=True)
|
||||
click.echo(f"Created tag {tag_name}")
|
||||
|
||||
def tag_exists(version: str) -> bool:
|
||||
"""Check if a git tag exists."""
|
||||
tag_name = f"v{version}"
|
||||
result = subprocess.run(
|
||||
["git", "tag", "-l", tag_name], capture_output=True, text=True, check=True
|
||||
)
|
||||
return tag_name in result.stdout.strip()
|
||||
|
||||
|
||||
def push_tag(version: str) -> None:
|
||||
"""Push a git tag."""
|
||||
tag_name = f"v{version}"
|
||||
# Push tag
|
||||
subprocess.run(["git", "push", "origin", tag_name], check=True)
|
||||
click.echo(f"Pushed tag {tag_name}")
|
||||
|
||||
@@ -160,20 +134,13 @@ def set(version: str) -> None:
|
||||
@click.option(
|
||||
"--version", help="Version to tag (uses current version if not specified)"
|
||||
)
|
||||
@click.option(
|
||||
"--push",
|
||||
is_flag=True,
|
||||
help="Push the tag to the remote repository",
|
||||
)
|
||||
def tag(version: str | None = None, push: bool = False) -> None:
|
||||
def tag(version: str | None = None) -> None:
|
||||
"""Create and push a git tag for the current version."""
|
||||
if not version:
|
||||
main_version, _, _ = get_current_versions()
|
||||
version = main_version
|
||||
|
||||
create_if_not_exists(version)
|
||||
if push:
|
||||
push_tag(version)
|
||||
create_and_push_tag(version)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
{
|
||||
"name": "@llamaindex/llama-cloud-services-e2e-tests",
|
||||
"private": true,
|
||||
"version": "0.0.1",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "pnpm run test:node16 && pnpm run test:nodenext && pnpm run test:bundler",
|
||||
"build": "pnpm run build:node16 && pnpm run build:nodenext && pnpm run build:bundler",
|
||||
"build:node16": "tsc -p src/tsconfig.node16.json --outDir dist/node16",
|
||||
"test:node16": "pnpm run build:node16 && node --test dist/node16/index.e2e.js",
|
||||
"build:nodenext": "tsc -p src/tsconfig.nodenext.json --outDir dist/nodenext",
|
||||
"test:nodenext": "pnpm run build:nodenext && node --test dist/nodenext/index.e2e.js",
|
||||
"build:bundler": "tsc -p src/tsconfig.bundler.json --outDir dist/bundler",
|
||||
"test:bundler": "pnpm run build:bundler && node --test dist/bundler/index.e2e.js",
|
||||
"build:node": "tsc -p src/tsconfig.node.json --outDir dist/node",
|
||||
"test:node": "pnpm run build:node && node --test dist/node/index.e2e.js"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.0.13",
|
||||
"llama-cloud-services": "workspace:*",
|
||||
"typescript": "^5.9.2"
|
||||
}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
import { ok } from "node:assert";
|
||||
import { test } from "node:test";
|
||||
|
||||
import { LlamaCloudIndex } from "llama-cloud-services";
|
||||
import { LlamaParseReader } from "llama-cloud-services";
|
||||
|
||||
test("LlamaIndex module resolution test", async (t) => {
|
||||
await t.test("works with ES module", () => {
|
||||
const index = new LlamaCloudIndex({
|
||||
name: "test-index",
|
||||
projectName: "Default",
|
||||
});
|
||||
const reader = new LlamaParseReader({
|
||||
resultType: "markdown",
|
||||
verbose: false,
|
||||
});
|
||||
ok(index !== undefined);
|
||||
ok(reader !== undefined);
|
||||
});
|
||||
|
||||
await t.test("works with dynamic imports", async () => {
|
||||
const mod = await import("llama-cloud-services"); // simulates commonjs
|
||||
ok(mod !== undefined);
|
||||
const index = new mod.LlamaCloudIndex({
|
||||
name: "test-index",
|
||||
projectName: "Default",
|
||||
});
|
||||
ok(index !== undefined);
|
||||
});
|
||||
|
||||
await t.test("all imports work", () => {
|
||||
const allImports = [
|
||||
LlamaCloudIndex,
|
||||
];
|
||||
|
||||
ok(allImports.filter(Boolean).length === allImports.length);
|
||||
});
|
||||
});
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "esnext",
|
||||
"moduleResolution": "bundler",
|
||||
"target": "esnext",
|
||||
"outDir": "dist",
|
||||
"rootDir": ".",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": ["**/*.ts"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "esnext",
|
||||
"moduleResolution": "node",
|
||||
"target": "esnext",
|
||||
"outDir": "dist",
|
||||
"rootDir": ".",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": ["**/*.ts"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "node16",
|
||||
"moduleResolution": "node16",
|
||||
"target": "esnext",
|
||||
"outDir": "dist",
|
||||
"rootDir": ".",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": ["**/*.ts"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "nodenext",
|
||||
"moduleResolution": "nodenext",
|
||||
"target": "esnext",
|
||||
"outDir": "dist",
|
||||
"rootDir": ".",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": ["**/*.ts"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"verbatimModuleSyntax": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"strictNullChecks": true,
|
||||
"exactOptionalPropertyTypes": true,
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist/type",
|
||||
"tsBuildInfoFile": "./dist/.tsbuildinfo",
|
||||
"incremental": true,
|
||||
"lib": [
|
||||
"ES2022",
|
||||
"DOM",
|
||||
"DOM.Iterable",
|
||||
"DOM.AsyncIterable"
|
||||
],
|
||||
"types": [
|
||||
"node"
|
||||
]
|
||||
},
|
||||
"include": [
|
||||
"./src"
|
||||
],
|
||||
"exclude": [
|
||||
"node_modules"
|
||||
]
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "llama-cloud-services",
|
||||
"version": "0.2.0",
|
||||
"version": "0.1.0",
|
||||
"type": "module",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
@@ -9,12 +9,10 @@
|
||||
"dev": "bunchee --watch",
|
||||
"lint": "eslint src/ --ignore-pattern client/*.ts --no-warn-ignored",
|
||||
"format": "prettier --write ./src/",
|
||||
"test": "vitest run --testTimeout=60000",
|
||||
"test": "vitest",
|
||||
"test:watch": "vitest --watch",
|
||||
"test:ui": "vitest --ui",
|
||||
"test:coverage": "vitest --coverage",
|
||||
"version": "cd .. && pnpm changeset version",
|
||||
"release": "pnpm run build && cd .. && pnpm changeset publish"
|
||||
"test:coverage": "vitest --coverage"
|
||||
},
|
||||
"files": [
|
||||
"openapi.json",
|
||||
@@ -91,12 +89,12 @@
|
||||
"@eslint/js": "^9.32.0",
|
||||
"@hey-api/client-fetch": "^0.10.1",
|
||||
"@hey-api/openapi-ts": "^0.67.5",
|
||||
"@llamaindex/core": "^0.6.19",
|
||||
"@llama-flow/core": "^0.4.1",
|
||||
"@llamaindex/core": "^0.6.18",
|
||||
"@llamaindex/env": "^0.1.30",
|
||||
"@llamaindex/workflow-core": "^0.4.1",
|
||||
"@types/node": "^20.19.9",
|
||||
"@typescript-eslint/eslint-plugin": "^8.38.0",
|
||||
"@typescript-eslint/parser": "^8.38.0",
|
||||
"@types/node": "^20.0.0",
|
||||
"@vitest/coverage-v8": "^2.0.0",
|
||||
"@vitest/ui": "^2.0.0",
|
||||
"bunchee": "^6.5.4",
|
||||
@@ -109,12 +107,12 @@
|
||||
"vitest": "^2.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@llamaindex/core": "^0.6.19",
|
||||
"@llama-flow/core": "^0.4.1",
|
||||
"@llamaindex/core": "^0.6.18",
|
||||
"@llamaindex/env": "^0.1.30",
|
||||
"@llamaindex/workflow-core": "^0.4.1"
|
||||
"llamaindex": "^0.11.23"
|
||||
},
|
||||
"dependencies": {
|
||||
"ajv": "^8.17.1",
|
||||
"p-retry": "^6.2.1",
|
||||
"zod": "^3.25.76"
|
||||
},
|
||||
|
||||
Generated
+4968
File diff suppressed because it is too large
Load Diff
@@ -1,10 +1,8 @@
|
||||
import type { BaseNodePostprocessor } from "@llamaindex/core/postprocessor";
|
||||
import {
|
||||
RetrieverQueryEngine,
|
||||
type BaseQueryEngine,
|
||||
} from "@llamaindex/core/query-engine";
|
||||
import type { BaseQueryEngine } from "@llamaindex/core/query-engine";
|
||||
import type { BaseSynthesizer } from "@llamaindex/core/response-synthesizers";
|
||||
import type { Document } from "@llamaindex/core/schema";
|
||||
import { RetrieverQueryEngine } from "llamaindex/engines";
|
||||
import type { CloudRetrieveParams } from "./LlamaCloudRetriever.js";
|
||||
import { LlamaCloudRetriever } from "./LlamaCloudRetriever.js";
|
||||
import type { CloudConstructorParams } from "./type.js";
|
||||
@@ -27,14 +25,9 @@ import {
|
||||
} from "./api";
|
||||
import type { BaseRetriever } from "@llamaindex/core/retriever";
|
||||
import { getEnv } from "@llamaindex/env";
|
||||
import { createQueryEngineTool, type QueryToolParams } from "./query-tool.js";
|
||||
import type { BaseTool } from "@llamaindex/core/llms";
|
||||
|
||||
type QueryEngineParams = {
|
||||
responseSynthesizer?: BaseSynthesizer;
|
||||
preFilters?: unknown;
|
||||
nodePostprocessors?: BaseNodePostprocessor[];
|
||||
} & CloudRetrieveParams;
|
||||
import type { QueryToolParams } from "llamaindex/indices";
|
||||
import { Settings } from "llamaindex";
|
||||
import { QueryEngineTool } from "llamaindex/tools";
|
||||
|
||||
export class LlamaCloudIndex {
|
||||
params: CloudConstructorParams;
|
||||
@@ -45,7 +38,7 @@ export class LlamaCloudIndex {
|
||||
}
|
||||
|
||||
private async waitForPipelineIngestion(
|
||||
verbose = false,
|
||||
verbose = Settings.debug,
|
||||
raiseOnError = false,
|
||||
): Promise<void> {
|
||||
const pipelineId = await this.getPipelineId();
|
||||
@@ -90,7 +83,7 @@ export class LlamaCloudIndex {
|
||||
|
||||
private async waitForDocumentIngestion(
|
||||
docIds: string[],
|
||||
verbose = false,
|
||||
verbose = Settings.debug,
|
||||
raiseOnError = false,
|
||||
): Promise<void> {
|
||||
const pipelineId = await this.getPipelineId();
|
||||
@@ -263,7 +256,13 @@ export class LlamaCloudIndex {
|
||||
return new LlamaCloudRetriever({ ...this.params, ...params });
|
||||
}
|
||||
|
||||
asQueryEngine(params?: QueryEngineParams): BaseQueryEngine {
|
||||
asQueryEngine(
|
||||
params?: {
|
||||
responseSynthesizer?: BaseSynthesizer;
|
||||
preFilters?: unknown;
|
||||
nodePostprocessors?: BaseNodePostprocessor[];
|
||||
} & CloudRetrieveParams,
|
||||
): BaseQueryEngine {
|
||||
const retriever = new LlamaCloudRetriever({
|
||||
...this.params,
|
||||
...params,
|
||||
@@ -275,15 +274,19 @@ export class LlamaCloudIndex {
|
||||
);
|
||||
}
|
||||
|
||||
asQueryTool(params: QueryEngineParams & QueryToolParams): BaseTool {
|
||||
return createQueryEngineTool({
|
||||
asQueryTool(params: QueryToolParams): QueryEngineTool {
|
||||
if (params.options) {
|
||||
params.retriever = this.asRetriever(params.options);
|
||||
}
|
||||
|
||||
return new QueryEngineTool({
|
||||
queryEngine: this.asQueryEngine(params),
|
||||
metadata: params?.metadata,
|
||||
includeSourceNodes: params?.includeSourceNodes ?? false,
|
||||
});
|
||||
}
|
||||
|
||||
queryTool(params: QueryEngineParams & QueryToolParams) {
|
||||
queryTool(params: QueryToolParams): QueryEngineTool {
|
||||
return this.asQueryTool(params);
|
||||
}
|
||||
|
||||
|
||||
@@ -261,7 +261,8 @@ export class AgentClient<T = unknown> {
|
||||
}
|
||||
}
|
||||
|
||||
export interface AgentDataClientOptions {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
export interface AgentDataClientOptions<T = unknown> {
|
||||
/** API key for the client */
|
||||
apiKey?: string;
|
||||
/** Base URL for the client */
|
||||
|
||||
@@ -28,31 +28,6 @@ export type ComparisonOperator =
|
||||
*/
|
||||
export type FilterOperation = RawFilterOperation;
|
||||
|
||||
/**
|
||||
* Metadata for an extracted field, including confidence and citation information
|
||||
*/
|
||||
export interface ExtractedFieldMetadata {
|
||||
/** The reasoning for the confidence score */
|
||||
reasoning?: string;
|
||||
/** The confidence score for the field, combined with parsing confidence if applicable */
|
||||
confidence?: number;
|
||||
/** The confidence score for the field based on the extracted text only */
|
||||
extraction_confidence?: number;
|
||||
/** The page number that the field occurred on */
|
||||
page_number?: number;
|
||||
/** The original text this field's value was derived from */
|
||||
matching_text?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Dictionary mapping field names to their metadata
|
||||
* Values can be ExtractedFieldMetadata objects, nested dictionaries, or arrays
|
||||
*/
|
||||
export type ExtractedFieldMetadataDict = Record<
|
||||
string,
|
||||
ExtractedFieldMetadata | Record<string, unknown> | unknown[]
|
||||
>;
|
||||
|
||||
/**
|
||||
* Base extracted data interface
|
||||
*/
|
||||
@@ -60,13 +35,11 @@ export interface ExtractedData<T = unknown> {
|
||||
/** The original data that was extracted from the document. For tracking changes. Should not be updated. */
|
||||
original_data: T;
|
||||
/** The latest state of the data. Will differ if data has been updated. */
|
||||
data: T;
|
||||
data?: T;
|
||||
/** The status of the extracted data. Prefer to use the StatusType values, but any string is allowed. */
|
||||
status: StatusType | string;
|
||||
/** The overall confidence score for the extracted data. */
|
||||
overall_confidence?: number;
|
||||
/** Page links, and perhaps eventually bounding boxes, for individual fields in the extracted data. */
|
||||
field_metadata?: ExtractedFieldMetadataDict;
|
||||
/** Confidence scores, if any, for each primitive field in the original_data data. */
|
||||
confidence?: Record<string, unknown>;
|
||||
/** The ID of the file that was used to extract the data. */
|
||||
file_id?: string;
|
||||
/** The name of the file that was used to extract the data. */
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { workflowEvent } from "@llamaindex/workflow-core";
|
||||
import { zodEvent } from "@llamaindex/workflow-core/util/zod";
|
||||
import { workflowEvent } from "@llama-flow/core";
|
||||
import { zodEvent } from "@llama-flow/core/util/zod";
|
||||
import { z } from "zod";
|
||||
import { parseFormSchema } from "./schema";
|
||||
|
||||
|
||||
@@ -1,11 +1,8 @@
|
||||
import { createClient, createConfig } from "@hey-api/client-fetch";
|
||||
import {
|
||||
createWorkflow,
|
||||
type InferWorkflowEventData,
|
||||
} from "@llamaindex/workflow-core";
|
||||
import { createStatefulMiddleware } from "@llamaindex/workflow-core/middleware/state";
|
||||
import { withTraceEvents } from "@llamaindex/workflow-core/middleware/trace-events";
|
||||
import { pRetryHandler } from "@llamaindex/workflow-core/util/p-retry";
|
||||
import { createWorkflow, type InferWorkflowEventData } from "@llama-flow/core";
|
||||
import { createStatefulMiddleware } from "@llama-flow/core/middleware/state";
|
||||
import { withTraceEvents } from "@llama-flow/core/middleware/trace-events";
|
||||
import { pRetryHandler } from "@llama-flow/core/util/p-retry";
|
||||
import { fs, getEnv, path } from "@llamaindex/env";
|
||||
import {
|
||||
type BodyUploadFileApiV1ParsingUploadPost,
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
import type { JSONValue } from "@llamaindex/core/global";
|
||||
import type { ToolMetadata } from "@llamaindex/core/llms";
|
||||
import type { BaseQueryEngine } from "@llamaindex/core/query-engine";
|
||||
import { tool } from "@llamaindex/core/tools";
|
||||
import { z } from "zod";
|
||||
|
||||
const DEFAULT_NAME = "llama_cloud_index_tool";
|
||||
const DEFAULT_DESCRIPTION =
|
||||
"Useful for retrieving relevant information from document stored in a LlamaCloud Index";
|
||||
|
||||
export type QueryToolParams = {
|
||||
metadata?: Omit<ToolMetadata, "parameters"> | undefined;
|
||||
includeSourceNodes?: boolean;
|
||||
};
|
||||
|
||||
export function createQueryEngineTool(
|
||||
options: { queryEngine: BaseQueryEngine } & QueryToolParams,
|
||||
) {
|
||||
const { queryEngine, metadata, includeSourceNodes } = options;
|
||||
return tool({
|
||||
name: metadata?.name ?? DEFAULT_NAME,
|
||||
description: metadata?.description ?? DEFAULT_DESCRIPTION,
|
||||
parameters: z.object({
|
||||
query: z.string({
|
||||
description: "The query to search for",
|
||||
}),
|
||||
}),
|
||||
execute: async ({ query }) => {
|
||||
const response = await queryEngine.query({ query });
|
||||
if (!includeSourceNodes) {
|
||||
return { content: response.message.content } as unknown as JSONValue;
|
||||
}
|
||||
return {
|
||||
content: response.message.content,
|
||||
sourceNodes: response.sourceNodes,
|
||||
} as unknown as JSONValue;
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -350,25 +350,6 @@ describe("Integration Tests", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it.skipIf(skipIfNoApiKey)("should create query engine tool", async () => {
|
||||
try {
|
||||
const queryEngineTool = index.asQueryTool({
|
||||
metadata: {
|
||||
name: "test-tool",
|
||||
description: "Test tool description",
|
||||
},
|
||||
});
|
||||
expect(queryEngineTool).toBeDefined();
|
||||
expect(typeof queryEngineTool.call).toBe("function");
|
||||
expect(queryEngineTool.metadata.name).toBe("test-tool");
|
||||
expect(queryEngineTool.metadata.description).toBe(
|
||||
"Test tool description",
|
||||
);
|
||||
} catch (error) {
|
||||
console.warn("Query engine tool creation test failed:", error.message);
|
||||
}
|
||||
});
|
||||
|
||||
it.skipIf(skipIfNoApiKey)(
|
||||
"should get pipeline and project IDs",
|
||||
async () => {
|
||||
|
||||
@@ -14,9 +14,10 @@
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist/type",
|
||||
"tsBuildInfoFile": "./dist/.tsbuildinfo",
|
||||
"emitDeclarationOnly": true,
|
||||
"incremental": true,
|
||||
"lib": ["ES2022", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
|
||||
"types": ["node"]
|
||||
"types": []
|
||||
},
|
||||
"include": ["./src"],
|
||||
"exclude": ["node_modules"]
|
||||
|
||||
Reference in New Issue
Block a user