Compare commits

...

3 Commits

Author SHA1 Message Date
Clelia (Astra) Bertelli 9d27eafb1c ci: add a changeset release action; docs: contributing update 2025-07-31 15:57:37 +02:00
Clelia (Astra) Bertelli c59628b8c6 ci: correct pnpm version 2025-07-31 15:12:42 +02:00
Clelia (Astra) Bertelli 7d2cd26222 feat: add changesets 2025-07-31 15:07:36 +02:00
106 changed files with 2289 additions and 2575 deletions
+8
View File
@@ -0,0 +1,8 @@
# Changesets
Hello and welcome! This folder has been automatically generated by `@changesets/cli`, a build tool that works
with multi-package repos, or single-package repos to help you version and publish your code. You can
find the full documentation for it [in our repository](https://github.com/changesets/changesets)
We have a quick list of common questions to get you started engaging with this project in
[our documentation](https://github.com/changesets/changesets/blob/main/docs/common-questions.md)
+11
View File
@@ -0,0 +1,11 @@
{
"$schema": "https://unpkg.com/@changesets/config@3.1.1/schema.json",
"changelog": "@changesets/cli/changelog",
"commit": false,
"fixed": [],
"linked": [],
"access": "restricted",
"baseBranch": "main",
"updateInternalDependencies": "patch",
"ignore": []
}
+3 -3
View File
@@ -35,16 +35,16 @@ jobs:
run: python --version
- name: Build
working-directory: py
working-directory: py/llama-cloud-services-py
run: uv build
- name: Test installing built package
shell: bash
working-directory: py
working-directory: py/llama-cloud-services-py
run: |
uv venv
uv pip install dist/*.whl
- name: Test import
working-directory: py
working-directory: py/llama-cloud-services-py
run: uv run -- python -c "import llama_cloud_services"
-28
View File
@@ -1,28 +0,0 @@
name: Build Package - TypeScript
on: [pull_request]
jobs:
pre_release:
name: Pre Release
runs-on: ubuntu-latest
steps:
- name: Checkout Repo
uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
version: 10
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version-file: "ts/llama_cloud_services/.nvmrc"
- name: Install dependencies
working-directory: ts/llama_cloud_services/
run: pnpm install --no-frozen-lockfile
- name: Build
working-directory: ts/llama_cloud_services/
run: pnpm run build
+67
View File
@@ -0,0 +1,67 @@
name: Release
on:
push:
branches:
- main
concurrency: ${{ github.workflow }}-${{ github.ref }}
jobs:
release:
name: Release
runs-on: ubuntu-latest
steps:
- name: Checkout Repo
uses: actions/checkout@v4
- uses: pnpm/action-setup@v3
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install uv
uses: astral-sh/setup-uv@v3
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version-file: ".nvmrc"
cache: "pnpm"
- name: Install dependencies
run: pnpm install
- name: Add auth token to .npmrc file
run: |
cat << EOF >> ".npmrc"
//registry.npmjs.org/:_authToken=$NPM_TOKEN
EOF
env:
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
- name: Get changeset status
id: get-changeset-status
run: |
pnpm changeset status --output .changeset/status.json
new_version=$(jq -r '.releases[0].newVersion' < .changeset/status.json)
rm -v .changeset/status.json
echo "new-version=${new_version}" >> "$GITHUB_OUTPUT"
- name: Create Release Pull Request or Publish to npm
id: changesets
uses: changesets/action@v1
with:
commit: Release ${{ steps.get-changeset-status.outputs.new-version }}
title: Release ${{ steps.get-changeset-status.outputs.new-version }}
# bump versions
version: pnpm new-version
# build package and call changeset publish
publish: pnpm release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
LLAMA_PARSE_PYPI_TOKEN: ${{ secrets.LLAMA_PARSE_PYPI_TOKEN }}
UV_PUBLISH_TOKEN: ${{ secrets.LLAMA_PARSE_PYPI_TOKEN }}
@@ -1,12 +1,10 @@
name: Lint - TypeScript
name: Lint, Format and Build - TypeScript
on:
push:
branches:
- main
pull_request:
branches:
- main
env:
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
@@ -20,7 +18,7 @@ jobs:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
version: 10
version: "10.12.4"
- name: Setup Node.js
uses: actions/setup-node@v4
with:
@@ -28,9 +26,5 @@ jobs:
- name: Install dependencies
working-directory: ts/llama_cloud_services/
run: pnpm install --no-frozen-lockfile
- name: Run lint
working-directory: ts/llama_cloud_services/
run: pnpm run lint
- name: Run Prettier
working-directory: ts/llama_cloud_services/
run: pnpm run format
- name: Run lint, formatting and building
run: pnpm pre-commit
+1 -1
View File
@@ -31,5 +31,5 @@ jobs:
- name: Run linter
shell: bash
working-directory: py
working-directory: py/llama-cloud-services-py
run: uv run -- pre-commit run -a
+12 -25
View File
@@ -24,33 +24,20 @@ jobs:
with:
version: ${{ env.UV_VERSION }}
- name: Set up Python
run: uv python install
- name: Install pnpm
- uses: pnpm/action-setup@v4
with:
version: 10
- name: Display Python version
run: python --version
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version-file: "ts/llama_cloud_services/.nvmrc"
- name: Build
working-directory: py
run: uv build
- name: Test installing built package
shell: bash
working-directory: py
run: |
uv venv
uv pip install dist/*.whl
- name: Publish package
shell: bash
working-directory: py
run: uv publish --token ${{ secrets.LLAMA_PARSE_PYPI_TOKEN }}
- name: Build and publish llama-parse
working-directory: py/llama_parse/
run: |
uv build
uv publish --token ${{ secrets.LLAMA_PARSE_PYPI_TOKEN }}
- name: Publish llama-cloud-services and llama-parse
run: pnpm release-py
env:
LLAMA_PARSE_PYPI_TOKEN: ${{ secrets.LLAMA_PARSE_PYPI_TOKEN }}
- name: Create GitHub Release
id: create_release
+1 -2
View File
@@ -36,11 +36,10 @@ jobs:
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
- name: Release
working-directory: ts/llama_cloud_services
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
run: pnpm publish --access public --no-git-checks
run: pnpm release-ts
- name: Create release
uses: ncipollo/release-action@v1
+2 -2
View File
@@ -31,9 +31,9 @@ jobs:
run: uv python install ${{ matrix.python-version }} && uv python pin ${{ matrix.python-version }}
- name: Run Tests
working-directory: py
working-directory: py/llama-cloud-services-py
run: uv run -- pytest tests/**/test_*.py
- name: Remove virtual environment
working-directory: py
working-directory: py/llama-cloud-services-py
run: rm -rf .venv/
+2
View File
@@ -9,3 +9,5 @@ __pycache__/
node_modules/
.turbo/
dist/
pnpm-lock.yaml
.venv/
+1 -1
View File
@@ -33,7 +33,7 @@ repos:
rev: v1.0.1
hooks:
- id: mypy
exclude: ^py/tests/
exclude: ^py/llama-cloud-services-py/tests/
additional_dependencies:
[
"types-requests",
+152 -18
View File
@@ -1,33 +1,167 @@
# Python
# Contribute to `llama-cloud-services`
## Installation
## Common Patterns
This project uses uv. Create a virtual environment, and run `uv sync`
### Issues
## Versioning (Maintainers only)
One of the forms of contribution can be issues.
Before merging your changes, make sure to bump the versions.
Issues should be used when there are bugs or feature request you would like to bring to the attention of the maintainers.
Make a version bump to `pyproject.toml`. If the underlying dependency on the llamacloud platform OpenAPI
sdk needs bumping, make sure to bring that in as well. If updating dependencies, run `uv lock`.
When opening an issue:
The legacy `llama_parse` package re-exports some of `llama_cloud_services` in the old namespace. The
versions need to be kept consistent to sidecar it with `llama_cloud_services`. Bump it's version in `llama_parse/pyproject.toml`, and also bump it's dependency version of `llama-cloud-services` to match.
- preferably, use the provided templates
- check for other issues (closed and open) to avoid duplicates
- try to be detailed and specific, reporting all the pieces the maintainer would need to have in order to reproduce your issue.
**Note**: Don't worry about updating the `llama_parse/poetry.lock` file when bumping versions. The GitHub action will automatically run `poetry lock` for the llama_parse package during the build process (though it doesn't commit the updated lockfile back to the repo).
### Pull requests
You can also do this with `./scripts/version-bump.py set 0.x.x` if you have `uv` installed.
In order to open a valid pull request:
Once the change is merged, push a tag `git tag -a v0.x.x -m 0.x.x` and `git push origin 0.x.x`.
- Fork the repository
- Checkout a secondary branch (common prefixes for secondary branches include: `fix`, `feat`, `chore`, `docs`). We tend to prefer the naming convention that uses `/`, such as: `fix/your-awesome-bug-fix`.
- Add and commit the changes to the secondary branch, following language-specific logic (see below)
- When the changes are pushed to your branch, open a pull request
This tagging step can be done with `./scripts/version-bump tag`.
## Python
# Typescript
### Set Up
## Installation
The two python packages, which can be found under `py/llama-cloud-services-py/`, are:
...
- `llama-cloud-services`
- `llama-parse`
## Versioning
> [!NOTE]
>
> `llama-parse` mostly re-exports from `llama-cloud-services`, so you should not modify that directly.
...
These packages are managed through [uv](https://docs.astral.sh/uv/), so make sure to have uv [installed](https://docs.astral.sh/uv/getting-started/installation/).
### Tests
It is important to make sure all tests pass after your changes, and cover new features with suitable unit tests.
Tests are found in `py/llama-cloud-services-py/tests/` and you can execute them with:
```bash
pytest tests/**/test_*.py
```
### Pre-Commit Versioning
Once you made your changes and tested them, **prior to committing** you should run (from the root folder) this command to automatically bump the version of python packages:
```bash
pnpm pre-commit-version
```
this will prompt you to choose what package's version you want to bump and what kind of bump you want to perform. Choose `@llama_cloud_services/llama-cloud-services-py` for python and choose the version bump according to the type of changing you made.
### Pre-commit checks
Before you commit, your files should pass the linting and formatting requirements. In order to do that, you should have `pre-commit` installed and set-up in your repository:
```bash
pip install pre-commit
pre-commit install
```
Once you have that set up, the files will be automatically linted and formatted according to the requirements.
## TypeScript
### Set Up
The TypeScript package, which can be found under `ts/llama_cloud_services/`, is managed through [`pnpm`](https://pnpm.io), so make sure to have it [installed](https://pnpm.io/installation).
In order to be able to run and test the package, make sure to install all the dependencies:
```bash
pnpm install
```
### Activate Test Mode
In order to activate test mode (to dynamically test your changes while you are performing them) you can use:
```bash
pnpm turbo run dev
```
### Test
It is important to make sure all tests pass after your changes, and cover new features with suitable unit tests.
Tests are found in `ts/llama_cloud_services/tests/` and you can execute them with:
```bash
pnpm test
```
### Pre-Commit Versioning
Once you made your changes and tested them, **prior to committing** you should run (from the root folder) two commands to automatically bump the version of python packages:
```bash
pnpm pre-commit-version
```
This will prompt you to choose what package's version you want to bump and what kind of bump you want to perform. Choose `llama-cloud-services` for TypeScript and choose the version bump according to the type of changing you made.
### Pre-commit checks
Before you commit, your files should pass the linting and formatting requirements. In order to do that, run (from the root folder):
```bash
pnpm pre-commit
```
The files will be then automatically linted and formatted according to the requirements.
## TypeScript _and_ Python
If you change **both PY and TS**, for versioning run:
```bash
pnpm pre-commit-version # choose both packages
pnpm new-version # bumps the version for both packages
```
## Release (maintainers only)
Every push to main might trigger a release.
Whether a release is pushed out or not depends on the presence of versioning files in `.changesets`: if you want a release to be packaged, then, you need to always run `pnpm pre-commit-version` prior to merging a pull request into main.
You can, nevertheless, manually set up language-specific releases, using the logic reported below.
### Python
To release `llama-cloud-services` and `llama-parse` in Python, run:
```bash
git checkout main
git pull
git tag <your-version> # e.g. v0.7.0
git push <your-version>
```
> [!NOTE]
>
> The tag must start with `v`
This will trigger the release workflow automatically.
### TypeScript
To release `llama-cloud-services` in TypeScript, run:
```bash
git checkout main
git pull
git tag llama-cloud-services@<your-version> # e.g. llama-cloud-services@0.3.0
git push origin llama-cloud-services@<your-version>
```
This will trigger the release workflow automatically.
+36
View File
@@ -0,0 +1,36 @@
{
"name": "llama_cloud_services",
"version": "1.0.0",
"description": "Monorepo for llama-cloud-services, mainly needed for versioning and release purposes",
"main": "index.js",
"scripts": {
"pre-commit": "pnpm --filter llama-cloud-services pre-commit",
"pre-commit-version": "pnpm changeset",
"new-version-py": "pnpm --filter @llama_cloud_services/llama-cloud-services-py new-version",
"new-version-ts": "pnpm --filter llama-cloud-services build && pnpm changeset version",
"new-version": "pnpm new-version-ts && pnpm new-version-py",
"release-py": "pnpm --filter @llama_cloud_services/llama-cloud-services-py release",
"release-ts": "pnpm --filter llama-cloud-services release",
"release": "pnpm release-ts && pnpm release-py"
},
"keywords": [
"ai",
"llms",
"cloud",
"intelligent-document-processing",
"knowledge-management",
"parsing",
"structured-data-extraction",
"rag"
],
"author": "LlamaIndex",
"license": "MIT",
"packageManager": "pnpm@10.12.4",
"devDependencies": {
"@changesets/cli": "^2.29.5"
},
"workspaces": [
"ts/*",
"py/*"
]
}
+3
View File
@@ -0,0 +1,3 @@
packages:
- "ts/*"
- "py/*"
@@ -0,0 +1,8 @@
# Changesets
Hello and welcome! This folder has been automatically generated by `@changesets/cli`, a build tool that works
with multi-package repos, or single-package repos to help you version and publish your code. You can
find the full documentation for it [in our repository](https://github.com/changesets/changesets)
We have a quick list of common questions to get you started engaging with this project in
[our documentation](https://github.com/changesets/changesets/blob/main/docs/common-questions.md)
@@ -0,0 +1,11 @@
{
"$schema": "https://unpkg.com/@changesets/config@3.1.1/schema.json",
"changelog": "@changesets/cli/changelog",
"commit": false,
"fixed": [],
"linked": [],
"access": "restricted",
"baseBranch": "main",
"updateInternalDependencies": "patch",
"ignore": []
}
@@ -8,10 +8,10 @@ This repository contains the code for hand-written SDKs and clients for interact
This includes:
- [LlamaParse](../parse.md) - A GenAI-native document parser that can parse complex document data for any downstream LLM use case (Agents, RAG, data processing, etc.).
- [LlamaReport (beta/invite-only)](../report.md) - A prebuilt agentic report builder that can be used to build reports from a variety of data sources.
- [LlamaExtract](../extract.md) - A prebuilt agentic data extractor that can be used to transform data into a structured JSON representation.
- [LlamaCloud Index](../index.md) - A widely customizable and fully automated document ingestion pipeline that also serves retrieval purposes.
- [LlamaParse](../../parse.md) - A GenAI-native document parser that can parse complex document data for any downstream LLM use case (Agents, RAG, data processing, etc.).
- [LlamaReport (beta/invite-only)](../../report.md) - A prebuilt agentic report builder that can be used to build reports from a variety of data sources.
- [LlamaExtract](../../extract.md) - A prebuilt agentic data extractor that can be used to transform data into a structured JSON representation.
- [LlamaCloud Index](../../index.md) - A widely customizable and fully automated document ingestion pipeline that also serves retrieval purposes.
## Getting Started
@@ -3,22 +3,21 @@ requires = ["hatchling"]
build-backend = "hatchling.build"
[dependency-groups]
dev = [
"pytest>=8.0.0,<9",
"pytest-asyncio",
"ipykernel>=6.29.0,<7"
]
dev = ["pytest>=8.0.0,<9", "pytest-asyncio", "ipykernel>=6.29.0,<7"]
[project]
name = "llama-parse"
version = "0.6.53"
description = "Parse files into RAG-Optimized formats."
authors = [{name = "Logan Markewich", email = "logan@llamaindex.ai"}]
requires-python = ">=3.9,<4.0"
readme = "README.md"
license = "MIT"
dependencies = ["llama-cloud-services>=0.6.53"]
[[project.authors]]
name = "Logan Markewich"
email = "logan@llamaindex.ai"
[project.scripts]
llama-parse = "llama_parse.cli.main:parse"
+18
View File
@@ -0,0 +1,18 @@
{
"name": "@llama_cloud_services/llama-cloud-services-py",
"version": "0.6.54",
"private": true,
"description": "Python package for llama-cloud-services. This file is mainly for versioning and release purposes.",
"type": "module",
"scripts": {
"new-version": "bash scripts/new_version.sh",
"release": "bash scripts/release.sh"
},
"keywords": [],
"author": "LlamaIndex",
"license": "MIT",
"packageManager": "pnpm@10.12.4",
"devDependencies": {
"@changesets/cli": "^2.29.5"
}
}
@@ -3,36 +3,20 @@ requires = ["hatchling"]
build-backend = "hatchling.build"
[dependency-groups]
dev = [
"pytest>=8.0.0,<9",
"pytest-asyncio",
"ipykernel>=6.29.0,<7",
"pre-commit==3.2.0",
"autoevals>=0.0.114,<0.0.115",
"deepdiff>=8.1.1,<9",
"ipython>=8.12.3,<9",
"jupyter>=1.1.1,<2",
"mypy>=1.14.1,<2"
]
dev = ["pytest>=8.0.0,<9", "pytest-asyncio", "ipykernel>=6.29.0,<7", "pre-commit==3.2.0", "autoevals>=0.0.114,<0.0.115", "deepdiff>=8.1.1,<9", "ipython>=8.12.3,<9", "jupyter>=1.1.1,<2", "mypy>=1.14.1,<2"]
[project]
name = "llama-cloud-services"
version = "0.6.53"
description = "Tailored SDK clients for LlamaCloud services."
authors = [{name = "Logan Markewich", email = "logan@runllama.ai"}]
requires-python = ">=3.9,<4.0"
readme = "README.md"
license = "MIT"
dependencies = [
"llama-index-core>=0.12.0",
"llama-cloud==0.1.35",
"pydantic>=2.8,!=2.10",
"click>=8.1.7,<9",
"python-dotenv>=1.0.1,<2",
"eval-type-backport>=0.2.0,<0.3 ; python_version < '3.10'",
"platformdirs>=4.3.7,<5",
"tenacity>=8.5.0, <10.0"
]
dependencies = ["llama-index-core>=0.12.0", "llama-cloud==0.1.35", "pydantic>=2.8,!=2.10", "click>=8.1.7,<9", "python-dotenv>=1.0.1,<2", "eval-type-backport>=0.2.0,<0.3 ; python_version < '3.10'", "platformdirs>=4.3.7,<5", "tenacity>=8.5.0, <10.0"]
[[project.authors]]
name = "Logan Markewich"
email = "logan@runllama.ai"
[project.scripts]
llama-parse = "llama_cloud_services.parse.cli.main:parse"
@@ -0,0 +1,34 @@
import toml # type: ignore [import]
import json
def change_parse_dep(data: dict, version: str) -> dict:
dep, ver = (
data["project"]["dependencies"][0].split(">=")[0],
data["project"]["dependencies"][0].split(">=")[1],
)
ver = version
dependency = dep + ">=" + ver
data["project"]["dependencies"].insert(0, dependency)
data["project"]["dependencies"].pop(1)
return data
with open("package.json", "r") as p:
package_data = json.load(p)
with open("pyproject.toml", "r") as t:
toml_data = toml.load(t)
with open("llama_parse/pyproject.toml", "r") as pt:
parse_toml_data = toml.load(pt)
toml_data["project"]["version"] = package_data["version"]
parse_toml_data["project"]["version"] = package_data["version"]
parse_toml_data = change_parse_dep(parse_toml_data, package_data["version"])
with open("pyproject.toml", "w") as w:
toml.dump(toml_data, w)
with open("llama_parse/pyproject.toml", "w") as pw:
toml.dump(parse_toml_data, pw)
@@ -0,0 +1,22 @@
# create uv virtual environment
uv venv
source .venv/bin/activate
# install toml
uv pip install -r scripts/requirements.txt
# run version change
uv run -- python3 scripts/new_version.py
# test version change
status_code=$(uv run -- python3 scripts/test_new_version.py) # returns 0 if the versions are the same, 1 if they are not
if [ "$status_code" -eq 1 ]; then
echo "Versions do not match, the version change failed..."
exit 1
elif [ "$status_code" -eq 0 ]; then
# lock the version changes
uv lock && cd llama_parse/ && uv lock
echo "Versions successfully changed"
exit 0
fi
@@ -0,0 +1,34 @@
# initialize pypi_token
pypi_token="no_token"
while
[[ $# -gt 0 ]] \
;
do
case "$1" in
-t | --token)
pypi_token="$2"
esac
done
if [[ $pypi_token == "no_token" ]]; then
if [[ $LLAMA_PARSE_PYPI_TOKEN == "" ]]; then
echo "No token provided and no token in the environment, exiting..."
exit 1
else
pypi_token="$LLAMA_PARSE_PYPI_TOKEN"
fi
fi
# build and publish llama_cloud_services
## build
uv build
## publish
uv publish --token $pypi_token
# build and publish llama_parse
cd llama_parse/
## build
uv build
## publish
uv publish --token $pypi_token
@@ -0,0 +1 @@
toml==0.10.2
@@ -0,0 +1,32 @@
import toml # type: ignore [import]
import json
def test_versions_equal(
package_data: dict, toml_data: dict, parse_toml_data: dict
) -> None:
llama_cloud_version = toml_data["project"]["version"]
package_version = package_data["version"]
parse_version = parse_toml_data["project"]["version"]
parse_dep_version = parse_toml_data["project"]["dependencies"][0].split(">=")[1]
try:
assert (
llama_cloud_version == package_version
and llama_cloud_version == parse_version
and llama_cloud_version == parse_dep_version
)
print("0")
except AssertionError:
print("1")
with open("package.json", "r") as p:
package_data = json.load(p)
with open("pyproject.toml", "r") as t:
toml_data = toml.load(t)
with open("llama_parse/pyproject.toml", "r") as pt:
parse_toml_data = toml.load(pt)
test_versions_equal(package_data, toml_data, parse_toml_data)

Some files were not shown because too many files have changed in this diff Show More