Compare commits
29 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3d05fe5d77 | |||
| c16ca673af | |||
| 6619034bce | |||
| c56fb5d8f7 | |||
| b407a5edb5 | |||
| e6a27d17fb | |||
| 34077fd479 | |||
| 7a68ad5a7f | |||
| 74a1b6c2f2 | |||
| 9a90ae5264 | |||
| 310c1bc105 | |||
| cd20b29299 | |||
| 0cb7aeb81c | |||
| 98db5eeeae | |||
| c21cb34ff6 | |||
| e28c7b9d92 | |||
| ee4e565604 | |||
| 6dbb089f4c | |||
| c4b694db8d | |||
| 97f428ad06 | |||
| ef92ee5408 | |||
| d094668d03 | |||
| 5bb5fc1625 | |||
| 1d57e0071d | |||
| 2a344c4f5c | |||
| ce02559b8d | |||
| e42746e372 | |||
| 3149dfd03a | |||
| e499fdbdab |
@@ -6,8 +6,11 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "py/**"
|
||||
pull_request:
|
||||
|
||||
paths:
|
||||
- "py/**"
|
||||
env:
|
||||
UV_VERSION: "0.7.20"
|
||||
|
||||
|
||||
@@ -1,5 +1,13 @@
|
||||
name: Build Package - TypeScript
|
||||
on: [pull_request]
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "ts/**"
|
||||
pull_request:
|
||||
paths:
|
||||
- "ts/**"
|
||||
|
||||
jobs:
|
||||
pre_release:
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
name: Claude Code
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
pull_request_review_comment:
|
||||
types: [created]
|
||||
issues:
|
||||
types: [opened, assigned]
|
||||
pull_request_review:
|
||||
types: [submitted]
|
||||
|
||||
jobs:
|
||||
claude:
|
||||
if: |
|
||||
(github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
|
||||
(github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
issues: read
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Check repository access
|
||||
id: check-access
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
# Get the user who triggered the event
|
||||
case "${{ github.event_name }}" in
|
||||
"issue_comment")
|
||||
USER="${{ github.event.comment.user.login }}"
|
||||
;;
|
||||
"pull_request_review_comment")
|
||||
USER="${{ github.event.comment.user.login }}"
|
||||
;;
|
||||
"pull_request_review")
|
||||
USER="${{ github.event.review.user.login }}"
|
||||
;;
|
||||
"issues")
|
||||
USER="${{ github.event.issue.user.login }}"
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "Checking repository access for user: $USER"
|
||||
|
||||
# Check if user has write access to the repository
|
||||
REPO="${{ github.repository }}"
|
||||
if gh api repos/$REPO/collaborators/$USER/permission --jq '.permission' | grep -E "(admin|write)" > /dev/null 2>&1; then
|
||||
echo "User $USER has write access to the repository"
|
||||
echo "authorized=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "User $USER does not have write access to the repository"
|
||||
echo "authorized=false" >> $GITHUB_OUTPUT
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Checkout repository
|
||||
if: steps.check-access.outputs.authorized == 'true'
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Run Claude Code
|
||||
if: steps.check-access.outputs.authorized == 'true'
|
||||
id: claude
|
||||
uses: anthropics/claude-code-action@beta
|
||||
with:
|
||||
anthropic_api_key: ${{ secrets.ANTHROPIC_GITHUB_API_KEY }}
|
||||
|
||||
# Optional: Specify model (defaults to Claude Sonnet 4, uncomment for Claude Opus 4)
|
||||
# model: "claude-opus-4-20250514"
|
||||
|
||||
# Optional: Customize the trigger phrase (default: @claude)
|
||||
# trigger_phrase: "/claude"
|
||||
|
||||
# Optional: Trigger when specific user is assigned to an issue
|
||||
# assignee_trigger: "claude-bot"
|
||||
|
||||
# Optional: Allow Claude to run specific commands
|
||||
# Allow bash commands to be run, for things like running tests, linting, etc.
|
||||
allowed_tools: "Bash(rg:*),Bash(find:*),Bash(grep:*),Bash(pnpm:*),Bash(npm:*),Bash(uv:*),Bash(pip:*),Bash(pipx:*),Bash(make:*),Bash(cd:*),WebFetch"
|
||||
|
||||
# Optional: Add custom instructions for Claude to customize its behavior for your project
|
||||
# custom_instructions: |
|
||||
# Follow our coding standards
|
||||
# Ensure all new code has tests
|
||||
# Use TypeScript for new files
|
||||
|
||||
# Optional: Custom environment variables for Claude
|
||||
# claude_env: |
|
||||
# NODE_ENV: test
|
||||
@@ -4,9 +4,11 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "ts/**"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "ts/**"
|
||||
|
||||
env:
|
||||
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
|
||||
@@ -26,7 +28,6 @@ jobs:
|
||||
with:
|
||||
node-version-file: "ts/llama_cloud_services/.nvmrc"
|
||||
- name: Install dependencies
|
||||
working-directory: ts/llama_cloud_services/
|
||||
run: pnpm install --no-frozen-lockfile
|
||||
- name: Run lint
|
||||
working-directory: ts/llama_cloud_services/
|
||||
|
||||
@@ -22,14 +22,29 @@ jobs:
|
||||
node-version-file: "ts/llama_cloud_services/.nvmrc"
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: ts/llama_cloud_services
|
||||
run: pnpm install --no-frozen-lockfile
|
||||
|
||||
- name: Run Build
|
||||
working-directory: ts/llama_cloud_services/
|
||||
run: pnpm build
|
||||
|
||||
- name: Build tarball
|
||||
run: |
|
||||
pnpm pack
|
||||
working-directory: ts/llama_cloud_services
|
||||
|
||||
- name: Setup npm authentication
|
||||
run: echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/.npmrc
|
||||
env:
|
||||
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
|
||||
- name: Release
|
||||
working-directory: ts/llama_cloud_services
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
run: pnpm publish --access public --no-git-checks
|
||||
|
||||
- name: Create release
|
||||
uses: ncipollo/release-action@v1
|
||||
with:
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
name: Test end-to-end - Python
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "py/**"
|
||||
|
||||
env:
|
||||
UV_VERSION: "0.7.20"
|
||||
LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
|
||||
|
||||
jobs:
|
||||
test_e2e:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
# You can use PyPy versions in python-version.
|
||||
# For example, pypy-2.7 and pypy-3.8
|
||||
matrix:
|
||||
python-version: ["3.12"]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
version: ${{ env.UV_VERSION }}
|
||||
|
||||
- name: Set up Python
|
||||
run: uv python install ${{ matrix.python-version }} && uv python pin ${{ matrix.python-version }}
|
||||
|
||||
- name: Run Tests
|
||||
working-directory: py
|
||||
run: uv run pytest unit_tests/ tests/ -v
|
||||
|
||||
- name: Remove virtual environment
|
||||
working-directory: py
|
||||
run: rm -rf .venv/
|
||||
@@ -4,11 +4,14 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "py/**"
|
||||
pull_request:
|
||||
paths:
|
||||
- "py/**"
|
||||
|
||||
env:
|
||||
UV_VERSION: "0.7.20"
|
||||
LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
@@ -32,7 +35,7 @@ jobs:
|
||||
|
||||
- name: Run Tests
|
||||
working-directory: py
|
||||
run: uv run -- pytest tests/**/test_*.py
|
||||
run: uv run pytest unit_tests/ -v
|
||||
|
||||
- name: Remove virtual environment
|
||||
working-directory: py
|
||||
|
||||
@@ -4,9 +4,11 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "ts/**"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "ts/**"
|
||||
|
||||
env:
|
||||
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
|
||||
@@ -15,7 +17,8 @@ env:
|
||||
LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
test:
|
||||
name: Test - TypeScript
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -27,8 +30,13 @@ jobs:
|
||||
with:
|
||||
node-version-file: "ts/llama_cloud_services/.nvmrc"
|
||||
- name: Install dependencies
|
||||
working-directory: ts/llama_cloud_services/
|
||||
run: pnpm install --no-frozen-lockfile
|
||||
- name: Run tests
|
||||
- name: Run Build
|
||||
working-directory: ts/llama_cloud_services/
|
||||
run: pnpm test --run
|
||||
run: pnpm build
|
||||
- name: Run Tests
|
||||
working-directory: ts/llama_cloud_services/
|
||||
run: pnpm test
|
||||
- name: Run e2e tests
|
||||
working-directory: ts/e2e-tests/
|
||||
run: pnpm test
|
||||
|
||||
@@ -15,6 +15,7 @@ repos:
|
||||
- id: end-of-file-fixer
|
||||
- id: mixed-line-ending
|
||||
- id: trailing-whitespace
|
||||
exclude: ^ts/llama_cloud_services/src/client/
|
||||
- repo: https://github.com/charliermarsh/ruff-pre-commit
|
||||
rev: v0.1.5
|
||||
|
||||
@@ -33,7 +34,7 @@ repos:
|
||||
rev: v1.0.1
|
||||
hooks:
|
||||
- id: mypy
|
||||
exclude: ^py/tests/
|
||||
exclude: ^py/tests|^py/unit_tests
|
||||
additional_dependencies:
|
||||
[
|
||||
"types-requests",
|
||||
@@ -63,13 +64,13 @@ repos:
|
||||
rev: v3.0.3
|
||||
hooks:
|
||||
- id: prettier
|
||||
exclude: uv.lock
|
||||
exclude: ^(uv.lock|ts/llama_cloud_services/pnpm-lock.yaml|ts/e2e-tests)
|
||||
- repo: https://github.com/codespell-project/codespell
|
||||
rev: v2.2.6
|
||||
hooks:
|
||||
- id: codespell
|
||||
additional_dependencies: [tomli]
|
||||
exclude: ^(uv.lock|docs|ts)
|
||||
exclude: ^(uv.lock|docs|ts|examples|pnpm-lock.yaml)
|
||||
args:
|
||||
[
|
||||
"--ignore-words-list",
|
||||
@@ -86,4 +87,4 @@ repos:
|
||||
- id: toml-sort-fix
|
||||
exclude: ".*uv.lock"
|
||||
|
||||
exclude: .github/ISSUE_TEMPLATE
|
||||
exclude: ^(.github/ISSUE_TEMPLATE|ts/llama_cloud_services/src/client|pnpm-lock.yaml)
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
# LlamaCloud Services Examples - Python
|
||||
|
||||
In this folder you will find several TypeScript end-to-end applications that contain examples regarding:
|
||||
|
||||
- [LlamaParse](./parse/)
|
||||
- [LlamaCloud Index](./index/)
|
||||
|
||||
Follow the instructions in each example folder to get started!
|
||||
@@ -0,0 +1,122 @@
|
||||
# LlamaExtract Demo
|
||||
|
||||
A TypeScript demo application showcasing the power of **LlamaExract** - a structured data extraction agentic service from [LlamaCloud](https://cloud.llamaindex.ai). This demo allows you to extract structured information from scientific papers and get them into a nice markdown format.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Features](#features)
|
||||
- [Prerequisites](#prerequisites)
|
||||
- [Installation](#installation)
|
||||
- [Usage](#usage)
|
||||
- [Start the Demo](#start-the-demo)
|
||||
- [Development Mode](#development-mode)
|
||||
- [Build the Project](#build-the-project)
|
||||
- [Code Quality](#code-quality)
|
||||
- [Quick Commands Reference](#quick-commands-reference)
|
||||
- [How It Works](#how-it-works)
|
||||
- [API Dependencies](#api-dependencies)
|
||||
- [Troubleshooting](#troubleshooting)
|
||||
- [Common Issues](#common-issues)
|
||||
- [License](#license)
|
||||
- [Contributing](#contributing)
|
||||
|
||||
## Features
|
||||
|
||||
- 📄 **Structured Data Extraction**: Extract data from your files effortlessly, and structure them the way you want!
|
||||
- 🤖 **Markdown Rendering**: Generate markdown directly from your extracted data
|
||||
- 🎨 **Beautiful CLI**: Styled console interface with colors and ASCII art
|
||||
- ⚡ **Fast Development**: Hot reload support with watch mode
|
||||
- 🛠️ **TypeScript**: Full TypeScript support with strict type checking
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Node.js (version 18 or higher)
|
||||
- pnpm package manager
|
||||
- LlamaCloud API key
|
||||
|
||||
## Installation
|
||||
|
||||
1. Clone the repository:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/run-llama/llama_cloud_services
|
||||
cd lama_cloud_services/examples-ts/extract/
|
||||
```
|
||||
|
||||
2. Install dependencies:
|
||||
|
||||
```bash
|
||||
npm install
|
||||
```
|
||||
|
||||
3. Set up your environment variables:
|
||||
|
||||
```bash
|
||||
# Add your API key to your environment
|
||||
export LLAMA_CLOUD_API_KEY="your-llamacloud-api-key"
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### Start the Demo
|
||||
|
||||
```bash
|
||||
npm run start
|
||||
```
|
||||
|
||||
The application will display a welcome screen and prompt you to enter the path to a document you'd like to process.
|
||||
|
||||
### Development Mode
|
||||
|
||||
For development with hot reload:
|
||||
|
||||
```bash
|
||||
npm run dev
|
||||
```
|
||||
|
||||
### Build the Project
|
||||
|
||||
```bash
|
||||
npm run build
|
||||
```
|
||||
|
||||
### Code Quality
|
||||
|
||||
Format code:
|
||||
|
||||
```bash
|
||||
npm run format
|
||||
```
|
||||
|
||||
Lint code:
|
||||
|
||||
```bash
|
||||
npm run lint
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
1. **Document Input**: Enter the path to your document when prompted
|
||||
2. **Parsing**: LlamaExtract, based on the schema you can find [here](./src/schema.ts), processes the document and extracts structured data
|
||||
3. **Markdown Rendering**: The extracted content is rendered into beautiful markdown
|
||||
4. **Results**: View the results directly in your terminal
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
1. **Module Resolution Errors**: Ensure you're using Node.js 18+ and have all dependencies installed
|
||||
2. **API Key Issues**: Verify your LlamaCloud API key is correctly set
|
||||
3. **File Path Errors**: Use absolute paths or ensure relative paths are correct from the project root
|
||||
|
||||
## License
|
||||
|
||||
MIT License - see the [LICENSE](../../LICENSE) file for details.
|
||||
|
||||
## Contributing
|
||||
|
||||
1. Fork the repository
|
||||
2. Create a feature branch
|
||||
3. Make your changes
|
||||
4. Run `npm run format` and `npm run lint`
|
||||
5. Submit a pull request
|
||||
@@ -0,0 +1,14 @@
|
||||
import js from "@eslint/js";
|
||||
import globals from "globals";
|
||||
import tseslint from "typescript-eslint";
|
||||
import { defineConfig } from "eslint/config";
|
||||
|
||||
export default defineConfig([
|
||||
{
|
||||
files: ["**/*.{js,mjs,cjs,ts,mts,cts}"],
|
||||
plugins: { js },
|
||||
extends: ["js/recommended"],
|
||||
languageOptions: { globals: globals.browser },
|
||||
},
|
||||
tseslint.configs.recommended,
|
||||
]);
|
||||
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"name": "llama-extract-demo",
|
||||
"version": "0.1.0",
|
||||
"description": "Demo for LlamaExtract in TypeScript",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "echo \"There are no tests\"",
|
||||
"start": "npm exec tsx src/index.ts",
|
||||
"lint": "eslint ./src/",
|
||||
"format": "prettier --write ./src/",
|
||||
"build": "tsc",
|
||||
"dev": "npm exec tsx --watch src/index.ts"
|
||||
},
|
||||
"author": "LlamaIndex",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"cli-markdown": "^3.5.1",
|
||||
"consola": "^3.4.2",
|
||||
"figlet": "^1.8.2",
|
||||
"llama-cloud-services": "file:../../ts/llama_cloud_services",
|
||||
"marked": "^15.0.12",
|
||||
"marked-terminal": "^7.3.0",
|
||||
"picocolors": "^1.1.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.32.0",
|
||||
"@types/figlet": "^1.7.0",
|
||||
"@types/marked-terminal": "^6.1.1",
|
||||
"@types/node": "^24.2.0",
|
||||
"eslint": "^9.32.0",
|
||||
"globals": "^16.3.0",
|
||||
"jiti": "^2.5.1",
|
||||
"prettier": "^3.6.2",
|
||||
"typescript": "^5.9.2",
|
||||
"typescript-eslint": "^8.39.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { LlamaExtract, ExtractConfig } from "llama-cloud-services";
|
||||
import cliMarkdown from "cli-markdown";
|
||||
import { logger } from "./logger";
|
||||
import pc from "picocolors";
|
||||
import { consoleInput, renderLogo } from "./utils";
|
||||
import { dataSchema } from "./schema";
|
||||
import { renderMarkdown, ResearchData } from "./markdown";
|
||||
|
||||
export async function main(): Promise<number> {
|
||||
const extractClient = new LlamaExtract(
|
||||
process.env.LLAMA_CLOUD_API_KEY!,
|
||||
"https://api.cloud.llamaindex.ai",
|
||||
);
|
||||
await renderLogo();
|
||||
logger.log(
|
||||
`Welcome to ${pc.bold(
|
||||
pc.magentaBright("LlamaExtract Demo✨"),
|
||||
)}, our demo for ${pc.bold(pc.green("LlamaExtract"))}, a ${pc.bold(
|
||||
pc.cyan("LlamaCloud☁️"),
|
||||
)} (https://cloud.llamaindex.ai) product!.\nIn this demo we are going to try extracting relevant information ${pc.bold(
|
||||
pc.yellowBright("from scientific papers"),
|
||||
)}. Type the path to the paper you would like to process below👇\nIf you wish to exit, just type ${pc.bold(
|
||||
pc.gray("quit"),
|
||||
)}.\n`,
|
||||
);
|
||||
while (true) {
|
||||
const userInput = await consoleInput();
|
||||
if (userInput.toLowerCase() == "quit") {
|
||||
break;
|
||||
}
|
||||
try {
|
||||
const generatedData = await extractClient.extract(
|
||||
dataSchema,
|
||||
{} as ExtractConfig,
|
||||
userInput,
|
||||
);
|
||||
const research = renderMarkdown(generatedData?.data as ResearchData); // Added await here
|
||||
logger.log(`${pc.bold(pc.cyan("Extracted information:✨"))}:\n`);
|
||||
logger.log(cliMarkdown(research));
|
||||
} catch (error) {
|
||||
logger.error(`Error processing file: ${error}`);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
@@ -0,0 +1,8 @@
|
||||
import { createConsola } from "consola";
|
||||
import type { ConsolaInstance } from "consola";
|
||||
|
||||
export const logger: ConsolaInstance = createConsola({
|
||||
formatOptions: {
|
||||
date: false,
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,172 @@
|
||||
type Author = {
|
||||
name: string;
|
||||
affiliation?: string;
|
||||
email?: string;
|
||||
};
|
||||
|
||||
type Methodology = {
|
||||
approach?: string;
|
||||
participants?: string;
|
||||
methods?: string[];
|
||||
};
|
||||
|
||||
type Result = {
|
||||
finding?: string;
|
||||
significance?: string;
|
||||
supportingData?: string;
|
||||
};
|
||||
|
||||
type Reference = {
|
||||
title: string;
|
||||
authors: string;
|
||||
year?: string;
|
||||
relevance?: string;
|
||||
};
|
||||
|
||||
type Discussion = {
|
||||
implications?: string[];
|
||||
limitations?: string[];
|
||||
futureWork?: string[];
|
||||
};
|
||||
|
||||
type Publication = {
|
||||
journal?: string;
|
||||
year: string;
|
||||
doi?: string;
|
||||
url?: string;
|
||||
};
|
||||
|
||||
export type ResearchData = {
|
||||
title: string;
|
||||
authors: Author[];
|
||||
abstract: string;
|
||||
keywords?: string[];
|
||||
mainFindings: string[];
|
||||
methodology?: Methodology;
|
||||
results?: Result[];
|
||||
discussion?: Discussion;
|
||||
references?: Reference[];
|
||||
publication?: Publication;
|
||||
};
|
||||
|
||||
export function renderMarkdown(data: ResearchData): string {
|
||||
const {
|
||||
title,
|
||||
authors,
|
||||
abstract,
|
||||
keywords,
|
||||
mainFindings,
|
||||
methodology,
|
||||
results,
|
||||
discussion,
|
||||
references,
|
||||
publication,
|
||||
} = data;
|
||||
|
||||
const md: string[] = [];
|
||||
|
||||
md.push(`# ${title}\n`);
|
||||
|
||||
// Authors
|
||||
md.push(`## Authors`);
|
||||
md.push(
|
||||
authors
|
||||
.map(
|
||||
(author) =>
|
||||
`- **${author.name}**${
|
||||
author.affiliation ? `, *${author.affiliation}*` : ""
|
||||
}${author.email ? ` (${author.email})` : ""}`,
|
||||
)
|
||||
.join("\n"),
|
||||
);
|
||||
|
||||
// Abstract
|
||||
md.push(`\n## Abstract\n${abstract}`);
|
||||
|
||||
// Keywords
|
||||
if (keywords && keywords.length > 0) {
|
||||
md.push(`\n## Keywords\n${keywords.map((k) => `- ${k}`).join("\n")}`);
|
||||
}
|
||||
|
||||
// Main Findings
|
||||
md.push(
|
||||
`\n## Main Findings\n${mainFindings.map((f) => `- ${f}`).join("\n")}`,
|
||||
);
|
||||
|
||||
// Methodology
|
||||
if (methodology) {
|
||||
md.push(`\n## Methodology`);
|
||||
if (methodology.approach) md.push(`**Approach:** ${methodology.approach}`);
|
||||
if (methodology.participants)
|
||||
md.push(`**Participants:** ${methodology.participants}`);
|
||||
if (methodology.methods?.length) {
|
||||
md.push(
|
||||
`**Methods:**\n${methodology.methods.map((m) => `- ${m}`).join("\n")}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Results
|
||||
if (results?.length) {
|
||||
md.push(`\n## Results`);
|
||||
results.forEach((result, i) => {
|
||||
md.push(`\n### Result ${i + 1}`);
|
||||
if (result.finding) md.push(`- **Finding:** ${result.finding}`);
|
||||
if (result.significance)
|
||||
md.push(`- **Significance:** ${result.significance}`);
|
||||
if (result.supportingData)
|
||||
md.push(`- **Supporting Data:** ${result.supportingData}`);
|
||||
});
|
||||
}
|
||||
|
||||
// Discussion
|
||||
if (discussion) {
|
||||
md.push(`\n## Discussion`);
|
||||
if (discussion.implications?.length) {
|
||||
md.push(
|
||||
`### Implications\n${discussion.implications
|
||||
.map((d) => `- ${d}`)
|
||||
.join("\n")}`,
|
||||
);
|
||||
}
|
||||
if (discussion.limitations?.length) {
|
||||
md.push(
|
||||
`### Limitations\n${discussion.limitations
|
||||
.map((d) => `- ${d}`)
|
||||
.join("\n")}`,
|
||||
);
|
||||
}
|
||||
if (discussion.futureWork?.length) {
|
||||
md.push(
|
||||
`### Future Work\n${discussion.futureWork
|
||||
.map((d) => `- ${d}`)
|
||||
.join("\n")}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// References
|
||||
if (references?.length) {
|
||||
md.push(`\n## References`);
|
||||
references.forEach((ref, i) => {
|
||||
md.push(
|
||||
`\n**[${i + 1}]** ${ref.title} — *${ref.authors}*${
|
||||
ref.year ? ` (${ref.year})` : ""
|
||||
}`,
|
||||
);
|
||||
if (ref.relevance) md.push(`> ${ref.relevance}`);
|
||||
});
|
||||
}
|
||||
|
||||
// Publication Info
|
||||
if (publication) {
|
||||
md.push(`\n## Publication`);
|
||||
if (publication.journal) md.push(`- **Journal:** ${publication.journal}`);
|
||||
if (publication.year) md.push(`- **Year:** ${publication.year}`);
|
||||
if (publication.doi) md.push(`- **DOI:** ${publication.doi}`);
|
||||
if (publication.url)
|
||||
md.push(`- **URL:** [${publication.url}](${publication.url})`);
|
||||
}
|
||||
|
||||
return md.join("\n");
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
export const dataSchema = {
|
||||
type: "object",
|
||||
required: ["title", "authors", "abstract", "mainFindings"],
|
||||
properties: {
|
||||
title: {
|
||||
type: "string",
|
||||
description: "The full title of the research paper",
|
||||
},
|
||||
authors: {
|
||||
type: "array",
|
||||
description: "List of all authors of the paper",
|
||||
items: {
|
||||
type: "object",
|
||||
properties: {
|
||||
name: {
|
||||
type: "string",
|
||||
description: "Full name of the author",
|
||||
},
|
||||
affiliation: {
|
||||
type: "string",
|
||||
description:
|
||||
"Institution or organization the author is affiliated with",
|
||||
},
|
||||
email: {
|
||||
type: "string",
|
||||
description: "Contact email of the author if provided",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
abstract: {
|
||||
type: "string",
|
||||
description: "Complete abstract or summary of the paper",
|
||||
},
|
||||
keywords: {
|
||||
type: "array",
|
||||
description:
|
||||
"Key terms and phrases that describe the paper's main topics",
|
||||
items: {
|
||||
type: "string",
|
||||
},
|
||||
},
|
||||
mainFindings: {
|
||||
type: "array",
|
||||
description: "Key findings, conclusions, or contributions of the paper",
|
||||
items: {
|
||||
type: "string",
|
||||
},
|
||||
},
|
||||
methodology: {
|
||||
type: "object",
|
||||
description: "Research methods and approaches used",
|
||||
properties: {
|
||||
approach: {
|
||||
type: "string",
|
||||
description: "Overall research approach or study design",
|
||||
},
|
||||
participants: {
|
||||
type: "string",
|
||||
description: "Description of study participants or data sources",
|
||||
},
|
||||
methods: {
|
||||
type: "array",
|
||||
description: "Specific methods, techniques, or tools used",
|
||||
items: {
|
||||
type: "string",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
results: {
|
||||
type: "array",
|
||||
description: "Main results and outcomes of the research",
|
||||
items: {
|
||||
type: "object",
|
||||
properties: {
|
||||
finding: {
|
||||
type: "string",
|
||||
description: "Description of the specific result or finding",
|
||||
},
|
||||
significance: {
|
||||
type: "string",
|
||||
description:
|
||||
"Statistical significance or importance of the finding",
|
||||
},
|
||||
supportingData: {
|
||||
type: "string",
|
||||
description: "Relevant statistics, measurements, or data points",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
discussion: {
|
||||
type: "object",
|
||||
properties: {
|
||||
implications: {
|
||||
type: "array",
|
||||
description: "Theoretical or practical implications of the findings",
|
||||
items: {
|
||||
type: "string",
|
||||
},
|
||||
},
|
||||
limitations: {
|
||||
type: "array",
|
||||
description: "Study limitations or constraints",
|
||||
items: {
|
||||
type: "string",
|
||||
},
|
||||
},
|
||||
futureWork: {
|
||||
type: "array",
|
||||
description: "Suggested future research directions",
|
||||
items: {
|
||||
type: "string",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
references: {
|
||||
type: "array",
|
||||
description:
|
||||
"Key papers cited that are crucial to understanding this work",
|
||||
items: {
|
||||
type: "object",
|
||||
properties: {
|
||||
title: {
|
||||
type: "string",
|
||||
description: "Title of the cited paper",
|
||||
},
|
||||
authors: {
|
||||
type: "string",
|
||||
description: "Authors of the cited paper",
|
||||
},
|
||||
year: {
|
||||
type: "string",
|
||||
description: "Publication year",
|
||||
},
|
||||
relevance: {
|
||||
type: "string",
|
||||
description: "Why this reference is important to the current paper",
|
||||
},
|
||||
},
|
||||
required: ["title", "authors"],
|
||||
},
|
||||
},
|
||||
publication: {
|
||||
type: "object",
|
||||
properties: {
|
||||
journal: {
|
||||
type: "string",
|
||||
description: "Name of the journal or conference",
|
||||
},
|
||||
year: {
|
||||
type: "string",
|
||||
description: "Year of publication",
|
||||
},
|
||||
doi: {
|
||||
type: "string",
|
||||
description: "Digital Object Identifier (DOI) of the paper",
|
||||
},
|
||||
url: {
|
||||
type: "string",
|
||||
description: "URL where the paper can be accessed",
|
||||
},
|
||||
},
|
||||
required: ["year"],
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,4 @@
|
||||
declare module "cli-markdown" {
|
||||
function cliMarkdown(input: string): string;
|
||||
export default cliMarkdown;
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import * as readline from "readline/promises";
|
||||
import figlet from "figlet";
|
||||
import pc from "picocolors";
|
||||
|
||||
export async function renderLogo(): Promise<void> {
|
||||
const logoText = figlet.textSync("Extract Demo", {
|
||||
font: "ANSI Shadow",
|
||||
horizontalLayout: "default",
|
||||
verticalLayout: "default",
|
||||
width: 100,
|
||||
whitespaceBreak: true,
|
||||
});
|
||||
|
||||
// Add some styling with picocolors
|
||||
const styledLogo = pc.bold(pc.redBright(logoText));
|
||||
|
||||
// Add some padding/margin
|
||||
console.log("\n");
|
||||
console.log(styledLogo);
|
||||
console.log(pc.gray("─".repeat(60)));
|
||||
console.log("\n");
|
||||
}
|
||||
|
||||
export async function consoleInput(): Promise<string> {
|
||||
const rl = readline.createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
});
|
||||
|
||||
const answer = await rl.question("Path to your file: ");
|
||||
rl.close();
|
||||
return answer;
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
# LlamaCloud Index Demo
|
||||
|
||||
A TypeScript demo application showcasing the power of **LlamaCloud Index** - a fully automated document ingestion and retrieval serviced offered within [LlamaCloud](https://cloud.llamaindex.ai). This demo allows you to ask questions, retrieve relevant contextual information and generate AI-powered responses using OpenAI's GPT models.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Features](#features)
|
||||
- [Prerequisites](#prerequisites)
|
||||
- [Installation](#installation)
|
||||
- [Usage](#usage)
|
||||
- [Start the Demo](#start-the-demo)
|
||||
- [Development Mode](#development-mode)
|
||||
- [Build the Project](#build-the-project)
|
||||
- [Code Quality](#code-quality)
|
||||
- [Quick Commands Reference](#quick-commands-reference)
|
||||
- [How It Works](#how-it-works)
|
||||
- [API Dependencies](#api-dependencies)
|
||||
- [Troubleshooting](#troubleshooting)
|
||||
- [Common Issues](#common-issues)
|
||||
- [License](#license)
|
||||
- [Contributing](#contributing)
|
||||
|
||||
## Features
|
||||
|
||||
- 🤖 **RAG**: Simple-yet-effective Retrieval Augmented Generation pipeline built on top of LlamaCloud Index and OpenAI
|
||||
- 🎨 **Beautiful CLI**: Styled console interface with colors and ASCII art
|
||||
- ⚡ **Fast Development**: Hot reload support with watch mode
|
||||
- 🛠️ **TypeScript**: Full TypeScript support with strict type checking
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Node.js (version 18 or higher)
|
||||
- pnpm package manager
|
||||
- OpenAI API key
|
||||
- LlamaCloud API key
|
||||
- An existing LlamaCloud Index pipeline
|
||||
|
||||
## Installation
|
||||
|
||||
1. Clone the repository:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/run-llama/llama_cloud_services
|
||||
cd lama_cloud_services/examples-ts/index/
|
||||
```
|
||||
|
||||
2. Install dependencies:
|
||||
|
||||
```bash
|
||||
pnpm install
|
||||
```
|
||||
|
||||
3. Set up your environment variables:
|
||||
|
||||
```bash
|
||||
export OPENAI_API_KEY="your-openai-api-key"
|
||||
export LLAMA_CLOUD_API_KEY="your-llamacloud-api-key"
|
||||
export PIPELINE_NAME="your-pipeline-name"
|
||||
```
|
||||
|
||||
4. Or write them into a `.env` file:
|
||||
|
||||
```env
|
||||
OPENAI_API_KEY="your-openai-api-key"
|
||||
LLAMA_CLOUD_API_KEY="your-llamacloud-api-key"
|
||||
PIPELINE_NAME="your-pipeline-name"
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### Start the Demo
|
||||
|
||||
```bash
|
||||
pnpm run start
|
||||
```
|
||||
|
||||
The application will display a welcome screen and prompt you to start chatting!
|
||||
|
||||
### Development Mode
|
||||
|
||||
For development with hot reload:
|
||||
|
||||
```bash
|
||||
pnpm run dev
|
||||
```
|
||||
|
||||
### Build the Project
|
||||
|
||||
```bash
|
||||
pnpm run build
|
||||
```
|
||||
|
||||
### Code Quality
|
||||
|
||||
Format code:
|
||||
|
||||
```bash
|
||||
pnpm run format
|
||||
```
|
||||
|
||||
Lint code:
|
||||
|
||||
```bash
|
||||
pnpm run lint
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
1. **Message Input**: Enter a message
|
||||
2. **Retrieval**: Several nodes are retrieved from the LlamaCloud index you specified
|
||||
3. **AI Response Generation**: The retrieved information is passed on to the AI model, along with its relevance score, and a reply to your original message is generated starting from that.
|
||||
4. **Results**: View the AI-generated summary in your terminal
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
1. **Module Resolution Errors**: Ensure you're using Node.js 18+ and have all dependencies installed
|
||||
2. **API Key Issues**: Verify your OpenAI and LlamaCloud API keys are correctly set
|
||||
|
||||
## License
|
||||
|
||||
MIT License - see the [LICENSE](../../LICENSE) file for details.
|
||||
|
||||
## Contributing
|
||||
|
||||
1. Fork the repository
|
||||
2. Create a feature branch
|
||||
3. Make your changes
|
||||
4. Run `pnpm run format` and `pnpm run lint`
|
||||
5. Submit a pull request
|
||||
@@ -0,0 +1,15 @@
|
||||
import js from "@eslint/js";
|
||||
import globals from "globals";
|
||||
import tseslint from "typescript-eslint";
|
||||
import { defineConfig } from "eslint/config";
|
||||
|
||||
export default defineConfig([
|
||||
{
|
||||
files: ["**/*.{js,mjs,cjs,ts,mts,cts}"],
|
||||
plugins: { js },
|
||||
extends: ["js/recommended"],
|
||||
languageOptions: { globals: globals.browser },
|
||||
},
|
||||
{ files: ["**/*.js"], languageOptions: { sourceType: "script" } },
|
||||
tseslint.configs.recommended,
|
||||
]);
|
||||
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"name": "llama-chat",
|
||||
"version": "0.1.0",
|
||||
"description": "Demo for LlamaCloud Index in TypeScript",
|
||||
"type": "module",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "echo \"There are no tests\"",
|
||||
"start": "pnpm exec tsx src/index.ts",
|
||||
"lint": "eslint ./src/",
|
||||
"format": "prettier --write ./src/",
|
||||
"build": "tsc",
|
||||
"dev": "pnpm exec tsx --watch src/index.ts"
|
||||
},
|
||||
"keywords": [
|
||||
"ai",
|
||||
"rag",
|
||||
"retrieval",
|
||||
"pipeline",
|
||||
"llms",
|
||||
"chatbot"
|
||||
],
|
||||
"author": "LlamaIndex",
|
||||
"license": "MIT",
|
||||
"packageManager": "pnpm@10.12.4",
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.32.0",
|
||||
"@types/figlet": "^1.7.0",
|
||||
"@types/node": "^24.1.0",
|
||||
"@typescript-eslint/eslint-plugin": "^8.38.0",
|
||||
"@typescript-eslint/parser": "^8.38.0",
|
||||
"eslint": "^9.32.0",
|
||||
"globals": "^16.3.0",
|
||||
"jiti": "^2.5.1",
|
||||
"prettier": "^3.6.2",
|
||||
"typescript": "^5.8.3",
|
||||
"typescript-eslint": "^8.38.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/openai": "^1.3.23",
|
||||
"ai": "^4.3.19",
|
||||
"consola": "^3.4.2",
|
||||
"dotenv": "^17.2.1",
|
||||
"figlet": "^1.8.2",
|
||||
"llama-cloud-services": "link:../../ts/llama_cloud_services",
|
||||
"picocolors": "^1.1.1"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { LlamaCloudIndex } from "llama-cloud-services";
|
||||
import { logger } from "./logger";
|
||||
import pc from "picocolors";
|
||||
import {
|
||||
consoleInput,
|
||||
retrievalAugmentedGeneration,
|
||||
renderLogo,
|
||||
} from "./utils";
|
||||
import dotenv from "dotenv";
|
||||
|
||||
dotenv.config();
|
||||
|
||||
export async function main(): Promise<number> {
|
||||
const index = new LlamaCloudIndex({
|
||||
name: process.env.PIPELINE_NAME as string,
|
||||
projectName: "Default",
|
||||
apiKey: process.env.LLAMA_CLOUD_API_KEY, // can provide API-key in the constructor or in the env
|
||||
});
|
||||
const retriever = index.asRetriever({
|
||||
similarityTopK: 5,
|
||||
});
|
||||
await renderLogo();
|
||||
logger.log(
|
||||
`Welcome to ${pc.bold(
|
||||
pc.magentaBright("✨LlamaChat✨"),
|
||||
)}, our demo for ${pc.bold(pc.green("Index🦙"))}, a ${pc.bold(
|
||||
pc.cyan("LlamaCloud☁️"),
|
||||
)} (https://cloud.llamaindex.ai) product!.\nType a question below, and you will get an answer!👇\nIf you wish to exit, just type ${pc.bold(
|
||||
pc.gray("quit"),
|
||||
)}.\n`,
|
||||
);
|
||||
while (true) {
|
||||
const userInput = await consoleInput();
|
||||
if (userInput.toLowerCase() == "quit") {
|
||||
break;
|
||||
}
|
||||
try {
|
||||
const nodes = await retriever.retrieve(userInput);
|
||||
const summary = await retrievalAugmentedGeneration(nodes, userInput);
|
||||
logger.log(`${pc.bold(pc.magentaBright("LlamaChat✨:"))}\n${summary}`);
|
||||
} catch (error) {
|
||||
logger.error(`Error processing your request: ${error}`);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
@@ -0,0 +1,8 @@
|
||||
import { createConsola } from "consola";
|
||||
import type { ConsolaInstance } from "consola";
|
||||
|
||||
export const logger: ConsolaInstance = createConsola({
|
||||
formatOptions: {
|
||||
date: false,
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,56 @@
|
||||
import { generateText } from "ai";
|
||||
import { openai } from "@ai-sdk/openai";
|
||||
import { NodeWithScore, MetadataMode } from "llamaindex";
|
||||
import * as readline from "readline/promises";
|
||||
import figlet from "figlet";
|
||||
import pc from "picocolors";
|
||||
|
||||
export async function renderLogo(): Promise<void> {
|
||||
const logoText = figlet.textSync("LlamaChat", {
|
||||
font: "ANSI Shadow",
|
||||
horizontalLayout: "default",
|
||||
verticalLayout: "default",
|
||||
width: 100,
|
||||
whitespaceBreak: true,
|
||||
});
|
||||
|
||||
// Add some styling with picocolors
|
||||
const styledLogo = pc.bold(pc.yellowBright(logoText));
|
||||
|
||||
// Add some padding/margin
|
||||
console.log("\n");
|
||||
console.log(styledLogo);
|
||||
console.log(pc.gray("─".repeat(60)));
|
||||
console.log("\n");
|
||||
}
|
||||
|
||||
export async function consoleInput(): Promise<string> {
|
||||
const rl = readline.createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
});
|
||||
|
||||
const answer = await rl.question(pc.cyanBright("You✨:"));
|
||||
rl.close();
|
||||
return answer;
|
||||
}
|
||||
|
||||
export async function retrievalAugmentedGeneration(
|
||||
nodes: NodeWithScore[],
|
||||
prompt: string,
|
||||
): Promise<string> {
|
||||
let mainText: string = "";
|
||||
|
||||
for (const node of nodes) {
|
||||
mainText += `\t{information: '${node.node.getContent(
|
||||
MetadataMode.ALL,
|
||||
)}', relevanceScore: '${node.score ?? "no score"}'}\n`;
|
||||
}
|
||||
|
||||
const { text } = await generateText({
|
||||
model: openai("gpt-4.1"),
|
||||
prompt: `[\n${mainText}\n]\n\nBased on the information you are given and on the relevance score of that (where -1 means no score available), answer to this user prompt: '${prompt}'`,
|
||||
});
|
||||
|
||||
return text;
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ES2022",
|
||||
"lib": ["ES2022"],
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"types": ["node"],
|
||||
"moduleResolution": "bundler",
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"resolveJsonModule": true
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
# LlamaParse Demo
|
||||
|
||||
A TypeScript demo application showcasing the power of **LlamaParse** - an intelligent document parsing service from [LlamaCloud](https://cloud.llamaindex.ai). This demo allows you to parse various document formats and generate AI-powered summaries using OpenAI's GPT models.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Features](#features)
|
||||
- [Prerequisites](#prerequisites)
|
||||
- [Installation](#installation)
|
||||
- [Usage](#usage)
|
||||
- [Start the Demo](#start-the-demo)
|
||||
- [Development Mode](#development-mode)
|
||||
- [Build the Project](#build-the-project)
|
||||
- [Code Quality](#code-quality)
|
||||
- [Quick Commands Reference](#quick-commands-reference)
|
||||
- [How It Works](#how-it-works)
|
||||
- [API Dependencies](#api-dependencies)
|
||||
- [Troubleshooting](#troubleshooting)
|
||||
- [Common Issues](#common-issues)
|
||||
- [License](#license)
|
||||
- [Contributing](#contributing)
|
||||
|
||||
## Features
|
||||
|
||||
- 📄 **Document Parsing**: Parse PDFs, Word docs, and other formats using LlamaParse
|
||||
- 🤖 **AI Summaries**: Generate intelligent summaries using OpenAI GPT-4
|
||||
- 🎨 **Beautiful CLI**: Styled console interface with colors and ASCII art
|
||||
- ⚡ **Fast Development**: Hot reload support with watch mode
|
||||
- 🛠️ **TypeScript**: Full TypeScript support with strict type checking
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Node.js (version 18 or higher)
|
||||
- pnpm package manager
|
||||
- OpenAI API key
|
||||
- LlamaCloud API key
|
||||
|
||||
## Installation
|
||||
|
||||
1. Clone the repository:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/run-llama/llama_cloud_services
|
||||
cd lama_cloud_services/examples-ts/parse/
|
||||
```
|
||||
|
||||
2. Install dependencies:
|
||||
|
||||
```bash
|
||||
pnpm install
|
||||
```
|
||||
|
||||
3. Set up your environment variables:
|
||||
|
||||
```bash
|
||||
# Add your API keys to your environment
|
||||
export OPENAI_API_KEY="your-openai-api-key"
|
||||
export LLAMA_CLOUD_API_KEY="your-llamacloud-api-key"
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### Start the Demo
|
||||
|
||||
```bash
|
||||
pnpm run start
|
||||
```
|
||||
|
||||
The application will display a welcome screen and prompt you to enter the path to a document you'd like to process.
|
||||
|
||||
### Development Mode
|
||||
|
||||
For development with hot reload:
|
||||
|
||||
```bash
|
||||
pnpm run dev
|
||||
```
|
||||
|
||||
### Build the Project
|
||||
|
||||
```bash
|
||||
pnpm run build
|
||||
```
|
||||
|
||||
### Code Quality
|
||||
|
||||
Format code:
|
||||
|
||||
```bash
|
||||
pnpm run format
|
||||
```
|
||||
|
||||
Lint code:
|
||||
|
||||
```bash
|
||||
pnpm run lint
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
1. **Document Input**: Enter the path to your document when prompted
|
||||
2. **Parsing**: LlamaParse processes the document and extracts structured content
|
||||
3. **AI Summary**: The extracted content is sent to OpenAI GPT-4 for summarization
|
||||
4. **Results**: View the AI-generated summary in your terminal
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
1. **Module Resolution Errors**: Ensure you're using Node.js 18+ and have all dependencies installed
|
||||
2. **API Key Issues**: Verify your OpenAI and LlamaCloud API keys are correctly set
|
||||
3. **File Path Errors**: Use absolute paths or ensure relative paths are correct from the project root
|
||||
|
||||
## License
|
||||
|
||||
MIT License - see the [LICENSE](../../LICENSE) file for details.
|
||||
|
||||
## Contributing
|
||||
|
||||
1. Fork the repository
|
||||
2. Create a feature branch
|
||||
3. Make your changes
|
||||
4. Run `pnpm run format` and `pnpm run lint`
|
||||
5. Submit a pull request
|
||||
@@ -0,0 +1,15 @@
|
||||
import js from "@eslint/js";
|
||||
import globals from "globals";
|
||||
import tseslint from "typescript-eslint";
|
||||
import { defineConfig } from "eslint/config";
|
||||
|
||||
export default defineConfig([
|
||||
{
|
||||
files: ["**/*.{js,mjs,cjs,ts,mts,cts}"],
|
||||
plugins: { js },
|
||||
extends: ["js/recommended"],
|
||||
languageOptions: { globals: globals.browser },
|
||||
},
|
||||
{ files: ["**/*.js"], languageOptions: { sourceType: "script" } },
|
||||
tseslint.configs.recommended,
|
||||
]);
|
||||
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"name": "llamaparse-demo",
|
||||
"version": "0.1.0",
|
||||
"description": "Demo for LlamaParse in TypeScript",
|
||||
"type": "module",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "echo \"There are no tests\"",
|
||||
"start": "pnpm exec tsx src/index.ts",
|
||||
"lint": "eslint ./src/",
|
||||
"format": "prettier --write ./src/",
|
||||
"build": "tsc",
|
||||
"dev": "pnpm exec tsx --watch src/index.ts"
|
||||
},
|
||||
"keywords": [
|
||||
"ai",
|
||||
"ocr",
|
||||
"parsing",
|
||||
"intelligent-document-processing",
|
||||
"pdf",
|
||||
"llms"
|
||||
],
|
||||
"author": "LlamaIndex",
|
||||
"license": "MIT",
|
||||
"packageManager": "pnpm@10.12.4",
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.32.0",
|
||||
"@types/figlet": "^1.7.0",
|
||||
"@types/node": "^24.1.0",
|
||||
"@typescript-eslint/eslint-plugin": "^8.38.0",
|
||||
"@typescript-eslint/parser": "^8.38.0",
|
||||
"eslint": "^9.32.0",
|
||||
"globals": "^16.3.0",
|
||||
"jiti": "^2.5.1",
|
||||
"prettier": "^3.6.2",
|
||||
"typescript": "^5.8.3",
|
||||
"typescript-eslint": "^8.38.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/openai": "^1.3.23",
|
||||
"ai": "^4.3.19",
|
||||
"consola": "^3.4.2",
|
||||
"figlet": "^1.8.2",
|
||||
"llama-cloud-services": "link:../../ts/llama_cloud_services",
|
||||
"picocolors": "^1.1.1"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { LlamaParseReader } from "llama-cloud-services";
|
||||
import { logger } from "./logger";
|
||||
import pc from "picocolors";
|
||||
import { consoleInput, generateSummary, renderLogo } from "./utils";
|
||||
|
||||
export async function main(): Promise<number> {
|
||||
const reader = new LlamaParseReader({ resultType: "markdown" });
|
||||
await renderLogo();
|
||||
logger.log(
|
||||
`Welcome to ${pc.bold(
|
||||
pc.magentaBright("✨LlamaParse Demo✨"),
|
||||
)}, our demo for ${pc.bold(pc.green("LlamaParse🦙"))}, a ${pc.bold(
|
||||
pc.cyan("LlamaCloud☁️"),
|
||||
)} (https://cloud.llamaindex.ai) product!.\nType the path to the document you would like to process below👇\nIf you wish to exit, just type ${pc.bold(
|
||||
pc.gray("quit"),
|
||||
)}.\n`,
|
||||
);
|
||||
while (true) {
|
||||
const userInput = await consoleInput();
|
||||
if (userInput.toLowerCase() == "quit") {
|
||||
break;
|
||||
}
|
||||
try {
|
||||
const documents = await reader.loadData(userInput);
|
||||
const summary = await generateSummary(documents); // Added await here
|
||||
logger.log(`${pc.bold(pc.cyan("AI-generated summary✨"))}:\n${summary}`);
|
||||
} catch (error) {
|
||||
logger.error(`Error processing file: ${error}`);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
@@ -0,0 +1,8 @@
|
||||
import { createConsola } from "consola";
|
||||
import type { ConsolaInstance } from "consola";
|
||||
|
||||
export const logger: ConsolaInstance = createConsola({
|
||||
formatOptions: {
|
||||
date: false,
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
import { generateText } from "ai";
|
||||
import { openai } from "@ai-sdk/openai";
|
||||
import { Document } from "llamaindex";
|
||||
import * as readline from "readline/promises";
|
||||
import figlet from "figlet";
|
||||
import pc from "picocolors";
|
||||
|
||||
export async function renderLogo(): Promise<void> {
|
||||
const logoText = figlet.textSync("LlamaParse Demo", {
|
||||
font: "ANSI Shadow",
|
||||
horizontalLayout: "default",
|
||||
verticalLayout: "default",
|
||||
width: 100,
|
||||
whitespaceBreak: true,
|
||||
});
|
||||
|
||||
// Add some styling with picocolors
|
||||
const styledLogo = pc.bold(pc.magentaBright(logoText));
|
||||
|
||||
// Add some padding/margin
|
||||
console.log("\n");
|
||||
console.log(styledLogo);
|
||||
console.log(pc.gray("─".repeat(60)));
|
||||
console.log("\n");
|
||||
}
|
||||
|
||||
export async function consoleInput(): Promise<string> {
|
||||
const rl = readline.createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
});
|
||||
|
||||
const answer = await rl.question("Path to your file: ");
|
||||
rl.close();
|
||||
return answer;
|
||||
}
|
||||
|
||||
export async function generateSummary(documents: Document[]): Promise<string> {
|
||||
let mainText: string = "";
|
||||
|
||||
for (const document of documents) {
|
||||
mainText += `${document.text}\n\n---\n\n`;
|
||||
}
|
||||
|
||||
const { text } = await generateText({
|
||||
model: openai("gpt-4.1"),
|
||||
prompt: `</chat>\n\t<text>${mainText}</text>\n\t<instructions>Could you please generate a summary of the given text?</instructions>\n</chat>`,
|
||||
});
|
||||
|
||||
return text;
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ES2022",
|
||||
"lib": ["ES2022"],
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"types": ["node"],
|
||||
"moduleResolution": "bundler",
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"resolveJsonModule": true
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
# LlamaCloud Services Examples - Python
|
||||
|
||||
In this folder you will find several python notebooks that contain examples regarding:
|
||||
|
||||
- [LlamaParse](./parse/)
|
||||
- [LlamaExtract](./extract/)
|
||||
- [LlamaReport](./report/)
|
||||
|
||||
Follow the instructions in each notebook to get started!
|
||||
|
Before Width: | Height: | Size: 3.3 MiB After Width: | Height: | Size: 3.3 MiB |
|
Before Width: | Height: | Size: 67 KiB After Width: | Height: | Size: 67 KiB |
|
Before Width: | Height: | Size: 202 KiB After Width: | Height: | Size: 202 KiB |
|
Before Width: | Height: | Size: 440 KiB After Width: | Height: | Size: 440 KiB |
|
Before Width: | Height: | Size: 156 KiB After Width: | Height: | Size: 156 KiB |
|
Before Width: | Height: | Size: 85 KiB After Width: | Height: | Size: 85 KiB |
|
Before Width: | Height: | Size: 893 KiB After Width: | Height: | Size: 893 KiB |
|
Before Width: | Height: | Size: 6.9 MiB After Width: | Height: | Size: 6.9 MiB |
|
Before Width: | Height: | Size: 195 KiB After Width: | Height: | Size: 195 KiB |
|
Before Width: | Height: | Size: 363 KiB After Width: | Height: | Size: 363 KiB |
|
Before Width: | Height: | Size: 343 KiB After Width: | Height: | Size: 343 KiB |
|
Before Width: | Height: | Size: 185 KiB After Width: | Height: | Size: 185 KiB |
|
Before Width: | Height: | Size: 254 KiB After Width: | Height: | Size: 254 KiB |
|
Before Width: | Height: | Size: 650 KiB After Width: | Height: | Size: 650 KiB |
|
Before Width: | Height: | Size: 72 KiB After Width: | Height: | Size: 72 KiB |
|
Before Width: | Height: | Size: 173 KiB After Width: | Height: | Size: 173 KiB |
|
Before Width: | Height: | Size: 72 KiB After Width: | Height: | Size: 72 KiB |
|
Before Width: | Height: | Size: 88 KiB After Width: | Height: | Size: 88 KiB |
|
Before Width: | Height: | Size: 200 KiB After Width: | Height: | Size: 200 KiB |
|
Before Width: | Height: | Size: 115 KiB After Width: | Height: | Size: 115 KiB |
|
Before Width: | Height: | Size: 334 KiB After Width: | Height: | Size: 334 KiB |
|
Before Width: | Height: | Size: 202 KiB After Width: | Height: | Size: 202 KiB |