mirror of
https://github.com/run-llama/llama_cloud_services.git
synced 2026-07-21 03:55:22 -04:00
Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 90d674cf72 | |||
| 51a7534733 | |||
| 4f5d2bde13 | |||
| 3d05fe5d77 | |||
| c16ca673af | |||
| 6619034bce | |||
| c56fb5d8f7 | |||
| b407a5edb5 | |||
| e6a27d17fb |
@@ -0,0 +1,122 @@
|
||||
# LlamaExtract Demo
|
||||
|
||||
A TypeScript demo application showcasing the power of **LlamaExract** - a structured data extraction agentic service from [LlamaCloud](https://cloud.llamaindex.ai). This demo allows you to extract structured information from scientific papers and get them into a nice markdown format.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Features](#features)
|
||||
- [Prerequisites](#prerequisites)
|
||||
- [Installation](#installation)
|
||||
- [Usage](#usage)
|
||||
- [Start the Demo](#start-the-demo)
|
||||
- [Development Mode](#development-mode)
|
||||
- [Build the Project](#build-the-project)
|
||||
- [Code Quality](#code-quality)
|
||||
- [Quick Commands Reference](#quick-commands-reference)
|
||||
- [How It Works](#how-it-works)
|
||||
- [API Dependencies](#api-dependencies)
|
||||
- [Troubleshooting](#troubleshooting)
|
||||
- [Common Issues](#common-issues)
|
||||
- [License](#license)
|
||||
- [Contributing](#contributing)
|
||||
|
||||
## Features
|
||||
|
||||
- 📄 **Structured Data Extraction**: Extract data from your files effortlessly, and structure them the way you want!
|
||||
- 🤖 **Markdown Rendering**: Generate markdown directly from your extracted data
|
||||
- 🎨 **Beautiful CLI**: Styled console interface with colors and ASCII art
|
||||
- ⚡ **Fast Development**: Hot reload support with watch mode
|
||||
- 🛠️ **TypeScript**: Full TypeScript support with strict type checking
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Node.js (version 18 or higher)
|
||||
- pnpm package manager
|
||||
- LlamaCloud API key
|
||||
|
||||
## Installation
|
||||
|
||||
1. Clone the repository:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/run-llama/llama_cloud_services
|
||||
cd lama_cloud_services/examples-ts/extract/
|
||||
```
|
||||
|
||||
2. Install dependencies:
|
||||
|
||||
```bash
|
||||
npm install
|
||||
```
|
||||
|
||||
3. Set up your environment variables:
|
||||
|
||||
```bash
|
||||
# Add your API key to your environment
|
||||
export LLAMA_CLOUD_API_KEY="your-llamacloud-api-key"
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### Start the Demo
|
||||
|
||||
```bash
|
||||
npm run start
|
||||
```
|
||||
|
||||
The application will display a welcome screen and prompt you to enter the path to a document you'd like to process.
|
||||
|
||||
### Development Mode
|
||||
|
||||
For development with hot reload:
|
||||
|
||||
```bash
|
||||
npm run dev
|
||||
```
|
||||
|
||||
### Build the Project
|
||||
|
||||
```bash
|
||||
npm run build
|
||||
```
|
||||
|
||||
### Code Quality
|
||||
|
||||
Format code:
|
||||
|
||||
```bash
|
||||
npm run format
|
||||
```
|
||||
|
||||
Lint code:
|
||||
|
||||
```bash
|
||||
npm run lint
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
1. **Document Input**: Enter the path to your document when prompted
|
||||
2. **Parsing**: LlamaExtract, based on the schema you can find [here](./src/schema.ts), processes the document and extracts structured data
|
||||
3. **Markdown Rendering**: The extracted content is rendered into beautiful markdown
|
||||
4. **Results**: View the results directly in your terminal
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
1. **Module Resolution Errors**: Ensure you're using Node.js 18+ and have all dependencies installed
|
||||
2. **API Key Issues**: Verify your LlamaCloud API key is correctly set
|
||||
3. **File Path Errors**: Use absolute paths or ensure relative paths are correct from the project root
|
||||
|
||||
## License
|
||||
|
||||
MIT License - see the [LICENSE](../../LICENSE) file for details.
|
||||
|
||||
## Contributing
|
||||
|
||||
1. Fork the repository
|
||||
2. Create a feature branch
|
||||
3. Make your changes
|
||||
4. Run `npm run format` and `npm run lint`
|
||||
5. Submit a pull request
|
||||
Binary file not shown.
@@ -0,0 +1,14 @@
|
||||
import js from "@eslint/js";
|
||||
import globals from "globals";
|
||||
import tseslint from "typescript-eslint";
|
||||
import { defineConfig } from "eslint/config";
|
||||
|
||||
export default defineConfig([
|
||||
{
|
||||
files: ["**/*.{js,mjs,cjs,ts,mts,cts}"],
|
||||
plugins: { js },
|
||||
extends: ["js/recommended"],
|
||||
languageOptions: { globals: globals.browser },
|
||||
},
|
||||
tseslint.configs.recommended,
|
||||
]);
|
||||
Generated
+3276
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"name": "llama-extract-demo",
|
||||
"version": "0.1.0",
|
||||
"description": "Demo for LlamaExtract in TypeScript",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "echo \"There are no tests\"",
|
||||
"start": "npm exec tsx src/index.ts",
|
||||
"lint": "eslint ./src/",
|
||||
"format": "prettier --write ./src/",
|
||||
"build": "tsc",
|
||||
"dev": "npm exec tsx --watch src/index.ts"
|
||||
},
|
||||
"author": "LlamaIndex",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"cli-markdown": "^3.5.1",
|
||||
"consola": "^3.4.2",
|
||||
"figlet": "^1.8.2",
|
||||
"llama-cloud-services": "file:../../ts/llama_cloud_services",
|
||||
"marked": "^15.0.12",
|
||||
"marked-terminal": "^7.3.0",
|
||||
"picocolors": "^1.1.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.32.0",
|
||||
"@types/figlet": "^1.7.0",
|
||||
"@types/marked-terminal": "^6.1.1",
|
||||
"@types/node": "^24.2.0",
|
||||
"eslint": "^9.32.0",
|
||||
"globals": "^16.3.0",
|
||||
"jiti": "^2.5.1",
|
||||
"prettier": "^3.6.2",
|
||||
"typescript": "^5.9.2",
|
||||
"typescript-eslint": "^8.39.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import { LlamaExtract, ExtractConfig } from "llama-cloud-services";
|
||||
import cliMarkdown from "cli-markdown";
|
||||
import { logger } from "./logger";
|
||||
import pc from "picocolors";
|
||||
import { consoleInput, renderLogo } from "./utils";
|
||||
import { dataSchema } from "./schema";
|
||||
import { renderMarkdown, ResearchData } from "./markdown";
|
||||
|
||||
export async function main(): Promise<number> {
|
||||
const extractClient = new LlamaExtract(
|
||||
process.env.LLAMA_CLOUD_API_KEY!,
|
||||
"https://api.cloud.llamaindex.ai",
|
||||
);
|
||||
await renderLogo();
|
||||
logger.log(
|
||||
`Welcome to ${pc.bold(
|
||||
pc.magentaBright("LlamaExtract Demo✨"),
|
||||
)}, our demo for ${pc.bold(pc.green("LlamaExtract"))}, a ${pc.bold(
|
||||
pc.cyan("LlamaCloud☁️"),
|
||||
)} (https://cloud.llamaindex.ai) product!.\nIn this demo we are going to try extracting relevant information ${pc.bold(
|
||||
pc.yellowBright("from scientific papers"),
|
||||
)}. Type the path to the paper you would like to process below👇\nIf you wish to exit, just type ${pc.bold(
|
||||
pc.gray("quit"),
|
||||
)}.\n`,
|
||||
);
|
||||
while (true) {
|
||||
const userInput = await consoleInput();
|
||||
if (userInput.toLowerCase() == "quit") {
|
||||
break;
|
||||
}
|
||||
try {
|
||||
const generatedData = await extractClient.extract(
|
||||
dataSchema,
|
||||
{} as ExtractConfig,
|
||||
userInput,
|
||||
);
|
||||
const research = renderMarkdown(generatedData?.data as ResearchData); // Added await here
|
||||
logger.log(`${pc.bold(pc.cyan("Extracted information:✨"))}:\n`);
|
||||
logger.log(cliMarkdown(research));
|
||||
} catch (error) {
|
||||
logger.error(`Error processing file: ${error}`);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
@@ -0,0 +1,8 @@
|
||||
import { createConsola } from "consola";
|
||||
import type { ConsolaInstance } from "consola";
|
||||
|
||||
export const logger: ConsolaInstance = createConsola({
|
||||
formatOptions: {
|
||||
date: false,
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,172 @@
|
||||
type Author = {
|
||||
name: string;
|
||||
affiliation?: string;
|
||||
email?: string;
|
||||
};
|
||||
|
||||
type Methodology = {
|
||||
approach?: string;
|
||||
participants?: string;
|
||||
methods?: string[];
|
||||
};
|
||||
|
||||
type Result = {
|
||||
finding?: string;
|
||||
significance?: string;
|
||||
supportingData?: string;
|
||||
};
|
||||
|
||||
type Reference = {
|
||||
title: string;
|
||||
authors: string;
|
||||
year?: string;
|
||||
relevance?: string;
|
||||
};
|
||||
|
||||
type Discussion = {
|
||||
implications?: string[];
|
||||
limitations?: string[];
|
||||
futureWork?: string[];
|
||||
};
|
||||
|
||||
type Publication = {
|
||||
journal?: string;
|
||||
year: string;
|
||||
doi?: string;
|
||||
url?: string;
|
||||
};
|
||||
|
||||
export type ResearchData = {
|
||||
title: string;
|
||||
authors: Author[];
|
||||
abstract: string;
|
||||
keywords?: string[];
|
||||
mainFindings: string[];
|
||||
methodology?: Methodology;
|
||||
results?: Result[];
|
||||
discussion?: Discussion;
|
||||
references?: Reference[];
|
||||
publication?: Publication;
|
||||
};
|
||||
|
||||
export function renderMarkdown(data: ResearchData): string {
|
||||
const {
|
||||
title,
|
||||
authors,
|
||||
abstract,
|
||||
keywords,
|
||||
mainFindings,
|
||||
methodology,
|
||||
results,
|
||||
discussion,
|
||||
references,
|
||||
publication,
|
||||
} = data;
|
||||
|
||||
const md: string[] = [];
|
||||
|
||||
md.push(`# ${title}\n`);
|
||||
|
||||
// Authors
|
||||
md.push(`## Authors`);
|
||||
md.push(
|
||||
authors
|
||||
.map(
|
||||
(author) =>
|
||||
`- **${author.name}**${
|
||||
author.affiliation ? `, *${author.affiliation}*` : ""
|
||||
}${author.email ? ` (${author.email})` : ""}`,
|
||||
)
|
||||
.join("\n"),
|
||||
);
|
||||
|
||||
// Abstract
|
||||
md.push(`\n## Abstract\n${abstract}`);
|
||||
|
||||
// Keywords
|
||||
if (keywords && keywords.length > 0) {
|
||||
md.push(`\n## Keywords\n${keywords.map((k) => `- ${k}`).join("\n")}`);
|
||||
}
|
||||
|
||||
// Main Findings
|
||||
md.push(
|
||||
`\n## Main Findings\n${mainFindings.map((f) => `- ${f}`).join("\n")}`,
|
||||
);
|
||||
|
||||
// Methodology
|
||||
if (methodology) {
|
||||
md.push(`\n## Methodology`);
|
||||
if (methodology.approach) md.push(`**Approach:** ${methodology.approach}`);
|
||||
if (methodology.participants)
|
||||
md.push(`**Participants:** ${methodology.participants}`);
|
||||
if (methodology.methods?.length) {
|
||||
md.push(
|
||||
`**Methods:**\n${methodology.methods.map((m) => `- ${m}`).join("\n")}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Results
|
||||
if (results?.length) {
|
||||
md.push(`\n## Results`);
|
||||
results.forEach((result, i) => {
|
||||
md.push(`\n### Result ${i + 1}`);
|
||||
if (result.finding) md.push(`- **Finding:** ${result.finding}`);
|
||||
if (result.significance)
|
||||
md.push(`- **Significance:** ${result.significance}`);
|
||||
if (result.supportingData)
|
||||
md.push(`- **Supporting Data:** ${result.supportingData}`);
|
||||
});
|
||||
}
|
||||
|
||||
// Discussion
|
||||
if (discussion) {
|
||||
md.push(`\n## Discussion`);
|
||||
if (discussion.implications?.length) {
|
||||
md.push(
|
||||
`### Implications\n${discussion.implications
|
||||
.map((d) => `- ${d}`)
|
||||
.join("\n")}`,
|
||||
);
|
||||
}
|
||||
if (discussion.limitations?.length) {
|
||||
md.push(
|
||||
`### Limitations\n${discussion.limitations
|
||||
.map((d) => `- ${d}`)
|
||||
.join("\n")}`,
|
||||
);
|
||||
}
|
||||
if (discussion.futureWork?.length) {
|
||||
md.push(
|
||||
`### Future Work\n${discussion.futureWork
|
||||
.map((d) => `- ${d}`)
|
||||
.join("\n")}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// References
|
||||
if (references?.length) {
|
||||
md.push(`\n## References`);
|
||||
references.forEach((ref, i) => {
|
||||
md.push(
|
||||
`\n**[${i + 1}]** ${ref.title} — *${ref.authors}*${
|
||||
ref.year ? ` (${ref.year})` : ""
|
||||
}`,
|
||||
);
|
||||
if (ref.relevance) md.push(`> ${ref.relevance}`);
|
||||
});
|
||||
}
|
||||
|
||||
// Publication Info
|
||||
if (publication) {
|
||||
md.push(`\n## Publication`);
|
||||
if (publication.journal) md.push(`- **Journal:** ${publication.journal}`);
|
||||
if (publication.year) md.push(`- **Year:** ${publication.year}`);
|
||||
if (publication.doi) md.push(`- **DOI:** ${publication.doi}`);
|
||||
if (publication.url)
|
||||
md.push(`- **URL:** [${publication.url}](${publication.url})`);
|
||||
}
|
||||
|
||||
return md.join("\n");
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
export const dataSchema = {
|
||||
type: "object",
|
||||
required: ["title", "authors", "abstract", "mainFindings"],
|
||||
properties: {
|
||||
title: {
|
||||
type: "string",
|
||||
description: "The full title of the research paper",
|
||||
},
|
||||
authors: {
|
||||
type: "array",
|
||||
description: "List of all authors of the paper",
|
||||
items: {
|
||||
type: "object",
|
||||
properties: {
|
||||
name: {
|
||||
type: "string",
|
||||
description: "Full name of the author",
|
||||
},
|
||||
affiliation: {
|
||||
type: "string",
|
||||
description:
|
||||
"Institution or organization the author is affiliated with",
|
||||
},
|
||||
email: {
|
||||
type: "string",
|
||||
description: "Contact email of the author if provided",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
abstract: {
|
||||
type: "string",
|
||||
description: "Complete abstract or summary of the paper",
|
||||
},
|
||||
keywords: {
|
||||
type: "array",
|
||||
description:
|
||||
"Key terms and phrases that describe the paper's main topics",
|
||||
items: {
|
||||
type: "string",
|
||||
},
|
||||
},
|
||||
mainFindings: {
|
||||
type: "array",
|
||||
description: "Key findings, conclusions, or contributions of the paper",
|
||||
items: {
|
||||
type: "string",
|
||||
},
|
||||
},
|
||||
methodology: {
|
||||
type: "object",
|
||||
description: "Research methods and approaches used",
|
||||
properties: {
|
||||
approach: {
|
||||
type: "string",
|
||||
description: "Overall research approach or study design",
|
||||
},
|
||||
participants: {
|
||||
type: "string",
|
||||
description: "Description of study participants or data sources",
|
||||
},
|
||||
methods: {
|
||||
type: "array",
|
||||
description: "Specific methods, techniques, or tools used",
|
||||
items: {
|
||||
type: "string",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
results: {
|
||||
type: "array",
|
||||
description: "Main results and outcomes of the research",
|
||||
items: {
|
||||
type: "object",
|
||||
properties: {
|
||||
finding: {
|
||||
type: "string",
|
||||
description: "Description of the specific result or finding",
|
||||
},
|
||||
significance: {
|
||||
type: "string",
|
||||
description:
|
||||
"Statistical significance or importance of the finding",
|
||||
},
|
||||
supportingData: {
|
||||
type: "string",
|
||||
description: "Relevant statistics, measurements, or data points",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
discussion: {
|
||||
type: "object",
|
||||
properties: {
|
||||
implications: {
|
||||
type: "array",
|
||||
description: "Theoretical or practical implications of the findings",
|
||||
items: {
|
||||
type: "string",
|
||||
},
|
||||
},
|
||||
limitations: {
|
||||
type: "array",
|
||||
description: "Study limitations or constraints",
|
||||
items: {
|
||||
type: "string",
|
||||
},
|
||||
},
|
||||
futureWork: {
|
||||
type: "array",
|
||||
description: "Suggested future research directions",
|
||||
items: {
|
||||
type: "string",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
references: {
|
||||
type: "array",
|
||||
description:
|
||||
"Key papers cited that are crucial to understanding this work",
|
||||
items: {
|
||||
type: "object",
|
||||
properties: {
|
||||
title: {
|
||||
type: "string",
|
||||
description: "Title of the cited paper",
|
||||
},
|
||||
authors: {
|
||||
type: "string",
|
||||
description: "Authors of the cited paper",
|
||||
},
|
||||
year: {
|
||||
type: "string",
|
||||
description: "Publication year",
|
||||
},
|
||||
relevance: {
|
||||
type: "string",
|
||||
description: "Why this reference is important to the current paper",
|
||||
},
|
||||
},
|
||||
required: ["title", "authors"],
|
||||
},
|
||||
},
|
||||
publication: {
|
||||
type: "object",
|
||||
properties: {
|
||||
journal: {
|
||||
type: "string",
|
||||
description: "Name of the journal or conference",
|
||||
},
|
||||
year: {
|
||||
type: "string",
|
||||
description: "Year of publication",
|
||||
},
|
||||
doi: {
|
||||
type: "string",
|
||||
description: "Digital Object Identifier (DOI) of the paper",
|
||||
},
|
||||
url: {
|
||||
type: "string",
|
||||
description: "URL where the paper can be accessed",
|
||||
},
|
||||
},
|
||||
required: ["year"],
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,4 @@
|
||||
declare module "cli-markdown" {
|
||||
function cliMarkdown(input: string): string;
|
||||
export default cliMarkdown;
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import * as readline from "readline/promises";
|
||||
import figlet from "figlet";
|
||||
import pc from "picocolors";
|
||||
|
||||
export async function renderLogo(): Promise<void> {
|
||||
const logoText = figlet.textSync("Extract Demo", {
|
||||
font: "ANSI Shadow",
|
||||
horizontalLayout: "default",
|
||||
verticalLayout: "default",
|
||||
width: 100,
|
||||
whitespaceBreak: true,
|
||||
});
|
||||
|
||||
// Add some styling with picocolors
|
||||
const styledLogo = pc.bold(pc.redBright(logoText));
|
||||
|
||||
// Add some padding/margin
|
||||
console.log("\n");
|
||||
console.log(styledLogo);
|
||||
console.log(pc.gray("─".repeat(60)));
|
||||
console.log("\n");
|
||||
}
|
||||
|
||||
export async function consoleInput(): Promise<string> {
|
||||
const rl = readline.createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
});
|
||||
|
||||
const answer = await rl.question("Path to your file: ");
|
||||
rl.close();
|
||||
return answer;
|
||||
}
|
||||
@@ -40,8 +40,8 @@ A TypeScript demo application showcasing the power of **LlamaCloud Index** - a f
|
||||
1. Clone the repository:
|
||||
|
||||
```bash
|
||||
git clone <repository-url>
|
||||
cd llamaparse-demo
|
||||
git clone https://github.com/run-llama/llama_cloud_services
|
||||
cd lama_cloud_services/examples-ts/index/
|
||||
```
|
||||
|
||||
2. Install dependencies:
|
||||
@@ -120,12 +120,12 @@ pnpm run lint
|
||||
|
||||
## License
|
||||
|
||||
MIT License - see the [LICENSE](../../../LICENSE) file for details.
|
||||
MIT License - see the [LICENSE](../../LICENSE) file for details.
|
||||
|
||||
## Contributing
|
||||
|
||||
1. Fork the repository
|
||||
2. Create a feature branch
|
||||
3. Make your changes
|
||||
4. Run `pnpm format` and `pnpm lint`
|
||||
4. Run `pnpm run format` and `pnpm run lint`
|
||||
5. Submit a pull request
|
||||
|
||||
@@ -42,7 +42,7 @@
|
||||
"consola": "^3.4.2",
|
||||
"dotenv": "^17.2.1",
|
||||
"figlet": "^1.8.2",
|
||||
"llama-cloud-services": "link:../../../ts/llama_cloud_services",
|
||||
"llama-cloud-services": "link:../../ts/llama_cloud_services",
|
||||
"picocolors": "^1.1.1"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,8 +40,8 @@ A TypeScript demo application showcasing the power of **LlamaParse** - an intell
|
||||
1. Clone the repository:
|
||||
|
||||
```bash
|
||||
git clone <repository-url>
|
||||
cd llamaparse-demo
|
||||
git clone https://github.com/run-llama/llama_cloud_services
|
||||
cd lama_cloud_services/examples-ts/parse/
|
||||
```
|
||||
|
||||
2. Install dependencies:
|
||||
@@ -113,12 +113,12 @@ pnpm run lint
|
||||
|
||||
## License
|
||||
|
||||
MIT License - see the [LICENSE](../../../LICENSE) file for details.
|
||||
MIT License - see the [LICENSE](../../LICENSE) file for details.
|
||||
|
||||
## Contributing
|
||||
|
||||
1. Fork the repository
|
||||
2. Create a feature branch
|
||||
3. Make your changes
|
||||
4. Run `pnpm format` and `pnpm lint`
|
||||
4. Run `pnpm run format` and `pnpm run lint`
|
||||
5. Submit a pull request
|
||||
|
||||
@@ -41,7 +41,7 @@
|
||||
"ai": "^4.3.19",
|
||||
"consola": "^3.4.2",
|
||||
"figlet": "^1.8.2",
|
||||
"llama-cloud-services": "link:../../../ts/llama_cloud_services",
|
||||
"llama-cloud-services": "link:../../ts/llama_cloud_services",
|
||||
"picocolors": "^1.1.1"
|
||||
}
|
||||
}
|
||||
|
||||
+94
-21
@@ -2,6 +2,29 @@
|
||||
|
||||
LlamaExtract provides a simple API for extracting structured data from unstructured documents like PDFs, text files and images.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Quick Start](#quick-start)
|
||||
- [Supported File Types](#supported-file-types)
|
||||
- [Different Input Types](#different-input-types)
|
||||
- [Async Extraction](#async-extraction)
|
||||
- [Core Concepts](#core-concepts)
|
||||
- [Defining Schemas](#defining-schemas)
|
||||
- [Using Pydantic (Recommended)](#using-pydantic-recommended)
|
||||
- [Using JSON Schema](#using-json-schema)
|
||||
- [Important restrictions on JSON/Pydantic Schema](#important-restrictions-on-jsonpydantic-schema)
|
||||
- [Extraction Configuration](#extraction-configuration)
|
||||
- [Configuration Options](#configuration-options)
|
||||
- [Extraction Agents (Advanced)](#extraction-agents-advanced)
|
||||
- [Creating Agents](#creating-agents)
|
||||
- [Agent Batch Processing](#agent-batch-processing)
|
||||
- [Updating Agent Schemas](#updating-agent-schemas)
|
||||
- [Managing Agents](#managing-agents)
|
||||
- [When to Use Agents vs Direct Extraction](#when-to-use-agents-vs-direct-extraction)
|
||||
- [Installation](#installation)
|
||||
- [Tips & Best Practices](#tips--best-practices)
|
||||
- [Additional Resources](#additional-resources)
|
||||
|
||||
## Quick Start
|
||||
|
||||
The simplest way to get started is to use the stateless API with the extraction configuration and the file/text to extract from:
|
||||
@@ -12,7 +35,7 @@ from llama_cloud import ExtractConfig, ExtractMode
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# Initialize client
|
||||
extractor = LlamaExtract()
|
||||
extractor = LlamaExtract(api_key="YOUR_API_KEY")
|
||||
|
||||
|
||||
# Define schema using Pydantic
|
||||
@@ -64,7 +87,9 @@ result = extractor.extract(Resume, config, SourceText(text_content=text))
|
||||
|
||||
### Async Extraction
|
||||
|
||||
For better performance with multiple files or when integrating with async applications:
|
||||
For better performance with multiple files or when integrating with async applications.
|
||||
Here `queue_extraction` will enqueue the extraction jobs and exit. Alternatively, you
|
||||
can use `aextract` to poll for the job and return the extraction results.
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
@@ -80,10 +105,18 @@ async def extract_resumes():
|
||||
Resume, config, ["resume1.pdf", "resume2.pdf"]
|
||||
)
|
||||
print(f"Queued {len(jobs)} extraction jobs")
|
||||
return jobs
|
||||
|
||||
|
||||
# Run async function
|
||||
asyncio.run(extract_resumes())
|
||||
jobs = asyncio.run(extract_resumes())
|
||||
# Check job status
|
||||
for job in jobs:
|
||||
status = agent.get_extraction_job(job.id).status
|
||||
print(f"Job {job.id}: {status}")
|
||||
|
||||
# Get results when complete
|
||||
results = [agent.get_extraction_run_for_job(job.id) for job in jobs]
|
||||
```
|
||||
|
||||
## Core Concepts
|
||||
@@ -159,24 +192,6 @@ config = ExtractConfig(extraction_mode=ExtractMode.FAST)
|
||||
result = extractor.extract(schema, config, "resume.pdf")
|
||||
```
|
||||
|
||||
## Extraction Configuration
|
||||
|
||||
Configure how extraction is performed using `ExtractConfig`:
|
||||
|
||||
```python
|
||||
from llama_cloud import ExtractConfig, ExtractMode
|
||||
|
||||
# Fast extraction (lower accuracy, faster processing)
|
||||
fast_config = ExtractConfig(extraction_mode=ExtractMode.FAST)
|
||||
|
||||
# Balanced extraction (good balance of speed and accuracy)
|
||||
balanced_config = ExtractConfig(extraction_mode=ExtractMode.BALANCED)
|
||||
|
||||
# Use different configs for different needs
|
||||
result = extractor.extract(schema, fast_config, "simple_document.pdf")
|
||||
result = extractor.extract(schema, balanced_config, "complex_document.pdf")
|
||||
```
|
||||
|
||||
### Important restrictions on JSON/Pydantic Schema
|
||||
|
||||
_LlamaExtract only supports a subset of the JSON Schema specification._ While limited, it should
|
||||
@@ -194,6 +209,62 @@ be sufficient for a wide variety of use-cases.
|
||||
your extraction workflow to fit within these constraints, e.g. by extracting subset of fields
|
||||
and later merging them together.
|
||||
|
||||
## Extraction Configuration
|
||||
|
||||
Configure how extraction is performed using `ExtractConfig`. The schema is the most important part, but several configuration options can significantly impact the extraction process.
|
||||
|
||||
```python
|
||||
from llama_cloud import ExtractConfig, ExtractMode, ChunkMode, ExtractTarget
|
||||
|
||||
# Basic configuration
|
||||
config = ExtractConfig(
|
||||
extraction_mode=ExtractMode.BALANCED, # FAST, BALANCED, MULTIMODAL, PREMIUM
|
||||
extraction_target=ExtractTarget.PER_DOC, # PER_DOC, PER_PAGE
|
||||
system_prompt="Focus on the most recent data",
|
||||
page_range="1-5,10-15", # Extract from specific pages
|
||||
)
|
||||
|
||||
# Advanced configuration
|
||||
advanced_config = ExtractConfig(
|
||||
extraction_mode=ExtractMode.MULTIMODAL,
|
||||
chunk_mode=ChunkMode.PAGE, # PAGE, SECTION
|
||||
high_resolution_mode=True, # Better OCR accuracy
|
||||
invalidate_cache=False, # Bypass cached results
|
||||
cite_sources=True, # Enable source citations
|
||||
use_reasoning=True, # Enable reasoning (not in FAST mode)
|
||||
confidence_scores=True, # MULTIMODAL/PREMIUM only
|
||||
)
|
||||
```
|
||||
|
||||
### Key Configuration Options
|
||||
|
||||
**Extraction Mode**: Controls processing quality and speed
|
||||
|
||||
- `FAST`: Fastest processing, suitable for simple documents with no OCR
|
||||
- `BALANCED`: Good speed/accuracy tradeoff for text-rich documents
|
||||
- `MULTIMODAL`: For visually rich documents with text, tables, and images (recommended)
|
||||
- `PREMIUM`: Highest accuracy with OCR, complex table/header detection
|
||||
|
||||
**Extraction Target**: Defines extraction scope
|
||||
|
||||
- `PER_DOC`: Apply schema to entire document (default)
|
||||
- `PER_PAGE`: Apply schema to each page, returns array of results
|
||||
|
||||
**Advanced Options**:
|
||||
|
||||
- `system_prompt`: Additional system-level instructions
|
||||
- `page_range`: Specific pages to extract (e.g., "1,3,5-7,9")
|
||||
- `chunk_mode`: Document splitting strategy (`PAGE` or `SECTION`)
|
||||
- `high_resolution_mode`: Better OCR for small text (slower processing)
|
||||
|
||||
**Extensions** (return additional metadata):
|
||||
|
||||
- `cite_sources`: Source tracing for extracted fields
|
||||
- `use_reasoning`: Explanations for extraction decisions
|
||||
- `confidence_scores`: Quantitative confidence measures (MULTIMODAL/PREMIUM only)
|
||||
|
||||
For complete configuration options, advanced settings, and detailed examples, see the [LlamaExtract Configuration Documentation](https://docs.cloud.llamaindex.ai/llamaextract/features/options).
|
||||
|
||||
## Extraction Agents (Advanced)
|
||||
|
||||
For reusable extraction workflows, you can create extraction agents that encapsulate both schema and configuration:
|
||||
@@ -326,5 +397,7 @@ Another option (orthogonal to the above) is to break the document into smaller s
|
||||
|
||||
## Additional Resources
|
||||
|
||||
- [Extract Documentation](https://docs.cloud.llamaindex.ai/llamaextract/getting_started) - Details on Extract features, API and examples.
|
||||
- [Example Notebook](docs/examples-py/extract/resume_screening.ipynb) - Detailed walkthrough of resume parsing
|
||||
- [Example Application with TypeScript](./examples-ts/extract/) - End-to-end examples using LlamaExtract TypeScript client.
|
||||
- [Discord Community](https://discord.com/invite/eN6D2HQ4aX) - Get help and share feedback
|
||||
|
||||
Generated
+63
@@ -23,6 +23,9 @@ importers:
|
||||
ajv:
|
||||
specifier: ^8.17.1
|
||||
version: 8.17.1
|
||||
file-type:
|
||||
specifier: ^21.0.0
|
||||
version: 21.0.0
|
||||
p-retry:
|
||||
specifier: ^6.2.1
|
||||
version: 6.2.1
|
||||
@@ -92,6 +95,8 @@ importers:
|
||||
|
||||
ts/llama_cloud_services/beta/agent: {}
|
||||
|
||||
ts/llama_cloud_services/extract: {}
|
||||
|
||||
ts/llama_cloud_services/parse: {}
|
||||
|
||||
ts/llama_cloud_services/reader: {}
|
||||
@@ -681,6 +686,13 @@ packages:
|
||||
'@swc/types@0.1.23':
|
||||
resolution: {integrity: sha512-u1iIVZV9Q0jxY+yM2vw/hZGDNudsN85bBpTqzAQ9rzkxW9D+e3aEM4Han+ow518gSewkXgjmEK0BD79ZcNVgPw==}
|
||||
|
||||
'@tokenizer/inflate@0.2.7':
|
||||
resolution: {integrity: sha512-MADQgmZT1eKjp06jpI2yozxaU9uVs4GzzgSL+uEq7bVcJ9V1ZXQkeGNql1fsSI0gMy1vhvNTNbUqrx+pZfJVmg==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
'@tokenizer/token@0.3.0':
|
||||
resolution: {integrity: sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==}
|
||||
|
||||
'@types/estree@1.0.8':
|
||||
resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==}
|
||||
|
||||
@@ -1097,6 +1109,10 @@ packages:
|
||||
resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==}
|
||||
engines: {node: '>=16.0.0'}
|
||||
|
||||
file-type@21.0.0:
|
||||
resolution: {integrity: sha512-ek5xNX2YBYlXhiUXui3D/BXa3LdqPmoLJ7rqEx2bKJ7EAUEfmXgW0Das7Dc6Nr9MvqaOnIqiPV0mZk/r/UpNAg==}
|
||||
engines: {node: '>=20'}
|
||||
|
||||
fill-range@7.1.1:
|
||||
resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==}
|
||||
engines: {node: '>=8'}
|
||||
@@ -1182,6 +1198,9 @@ packages:
|
||||
html-escaper@2.0.2:
|
||||
resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==}
|
||||
|
||||
ieee754@1.2.1:
|
||||
resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
|
||||
|
||||
ignore@5.3.2:
|
||||
resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==}
|
||||
engines: {node: '>= 4'}
|
||||
@@ -1632,6 +1651,10 @@ packages:
|
||||
resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
strtok3@10.3.4:
|
||||
resolution: {integrity: sha512-KIy5nylvC5le1OdaaoCJ07L+8iQzJHGH6pWDuzS+d07Cu7n1MZ2x26P8ZKIWfbK02+XIL8Mp4RkWeqdUCrDMfg==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
supports-color@7.2.0:
|
||||
resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==}
|
||||
engines: {node: '>=8'}
|
||||
@@ -1674,6 +1697,10 @@ packages:
|
||||
resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==}
|
||||
engines: {node: '>=8.0'}
|
||||
|
||||
token-types@6.0.4:
|
||||
resolution: {integrity: sha512-MD9MjpVNhVyH4fyd5rKphjvt/1qj+PtQUz65aFqAZA6XniWAuSFRjLk3e2VALEFlh9OwBpXUN7rfeqSnT/Fmkw==}
|
||||
engines: {node: '>=14.16'}
|
||||
|
||||
totalist@3.0.1:
|
||||
resolution: {integrity: sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==}
|
||||
engines: {node: '>=6'}
|
||||
@@ -1745,6 +1772,10 @@ packages:
|
||||
engines: {node: '>=0.8.0'}
|
||||
hasBin: true
|
||||
|
||||
uint8array-extras@1.4.0:
|
||||
resolution: {integrity: sha512-ZPtzy0hu4cZjv3z5NW9gfKnNLjoz4y6uv4HlelAjDK7sY/xOkKZv9xK/WQpcsBB3jEybChz9DPC2U/+cusjJVQ==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
undici-types@6.21.0:
|
||||
resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==}
|
||||
|
||||
@@ -2312,6 +2343,16 @@ snapshots:
|
||||
dependencies:
|
||||
'@swc/counter': 0.1.3
|
||||
|
||||
'@tokenizer/inflate@0.2.7':
|
||||
dependencies:
|
||||
debug: 4.4.1
|
||||
fflate: 0.8.2
|
||||
token-types: 6.0.4
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
'@tokenizer/token@0.3.0': {}
|
||||
|
||||
'@types/estree@1.0.8': {}
|
||||
|
||||
'@types/json-schema@7.0.15': {}
|
||||
@@ -2815,6 +2856,15 @@ snapshots:
|
||||
dependencies:
|
||||
flat-cache: 4.0.1
|
||||
|
||||
file-type@21.0.0:
|
||||
dependencies:
|
||||
'@tokenizer/inflate': 0.2.7
|
||||
strtok3: 10.3.4
|
||||
token-types: 6.0.4
|
||||
uint8array-extras: 1.4.0
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
fill-range@7.1.1:
|
||||
dependencies:
|
||||
to-regex-range: 5.0.1
|
||||
@@ -2903,6 +2953,8 @@ snapshots:
|
||||
|
||||
html-escaper@2.0.2: {}
|
||||
|
||||
ieee754@1.2.1: {}
|
||||
|
||||
ignore@5.3.2: {}
|
||||
|
||||
ignore@7.0.5: {}
|
||||
@@ -3328,6 +3380,10 @@ snapshots:
|
||||
|
||||
strip-json-comments@3.1.1: {}
|
||||
|
||||
strtok3@10.3.4:
|
||||
dependencies:
|
||||
'@tokenizer/token': 0.3.0
|
||||
|
||||
supports-color@7.2.0:
|
||||
dependencies:
|
||||
has-flag: 4.0.0
|
||||
@@ -3368,6 +3424,11 @@ snapshots:
|
||||
dependencies:
|
||||
is-number: 7.0.0
|
||||
|
||||
token-types@6.0.4:
|
||||
dependencies:
|
||||
'@tokenizer/token': 0.3.0
|
||||
ieee754: 1.2.1
|
||||
|
||||
totalist@3.0.1: {}
|
||||
|
||||
ts-api-utils@2.1.0(typescript@5.9.2):
|
||||
@@ -3425,6 +3486,8 @@ snapshots:
|
||||
uglify-js@3.19.3:
|
||||
optional: true
|
||||
|
||||
uint8array-extras@1.4.0: {}
|
||||
|
||||
undici-types@6.21.0: {}
|
||||
|
||||
undici-types@7.10.0: {}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from .client import FileClient
|
||||
|
||||
__all__ = ["FileClient"]
|
||||
@@ -0,0 +1,82 @@
|
||||
from io import BytesIO
|
||||
from typing import BinaryIO
|
||||
import os
|
||||
from llama_cloud.client import AsyncLlamaCloud
|
||||
from llama_cloud.types import File, FileCreate
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class FileClient:
|
||||
"""
|
||||
Higher-level client for interacting with the LlamaCloud Files API.
|
||||
Uses presigned URLs for uploads by default.
|
||||
|
||||
Args:
|
||||
client: The LlamaCloud client to use.
|
||||
project_id: The project ID to use.
|
||||
organization_id: The organization ID to use.
|
||||
use_presigned_url: Whether to use presigned URLs for uploads (set to False when uploading to BYOC deployments).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
client: AsyncLlamaCloud,
|
||||
project_id: Optional[str] = None,
|
||||
organization_id: Optional[str] = None,
|
||||
use_presigned_url: bool = True,
|
||||
):
|
||||
self.client = client
|
||||
self.project_id = project_id
|
||||
self.organization_id = organization_id
|
||||
self.use_presigned_url = use_presigned_url
|
||||
|
||||
async def get_file(self, file_id: str) -> File:
|
||||
return await self.client.files.get_file(
|
||||
file_id, project_id=self.project_id, organization_id=self.organization_id
|
||||
)
|
||||
|
||||
async def upload_file(
|
||||
self, file_path: str, external_file_id: Optional[str] = None
|
||||
) -> File:
|
||||
external_file_id = external_file_id or file_path
|
||||
file_size = os.path.getsize(file_path)
|
||||
with open(file_path, "rb") as file:
|
||||
return await self.upload_buffer(file, external_file_id, file_size)
|
||||
|
||||
async def upload_bytes(self, bytes: bytes, external_file_id: str) -> File:
|
||||
return await self.upload_buffer(BytesIO(bytes), external_file_id, len(bytes))
|
||||
|
||||
async def upload_buffer(
|
||||
self,
|
||||
buffer: BinaryIO,
|
||||
external_file_id: str,
|
||||
file_size: int,
|
||||
) -> File:
|
||||
if self.use_presigned_url:
|
||||
if getattr(buffer, "name", None):
|
||||
name = os.path.basename(str(getattr(buffer, "name", external_file_id)))
|
||||
else:
|
||||
name = external_file_id
|
||||
presigned_url = await self.client.files.generate_presigned_url(
|
||||
project_id=self.project_id,
|
||||
organization_id=self.organization_id,
|
||||
request=FileCreate(
|
||||
name=name,
|
||||
external_file_id=external_file_id,
|
||||
file_size=file_size,
|
||||
),
|
||||
)
|
||||
httpx_client = self.client._client_wrapper.httpx_client
|
||||
httpx_client.post(presigned_url.url, data=buffer)
|
||||
return await self.client.files.get_file(
|
||||
presigned_url.file_id,
|
||||
project_id=self.project_id,
|
||||
organization_id=self.organization_id,
|
||||
)
|
||||
else:
|
||||
return await self.client.files.upload_file(
|
||||
upload_file=buffer,
|
||||
external_file_id=external_file_id,
|
||||
project_id=self.project_id,
|
||||
organization_id=self.organization_id,
|
||||
)
|
||||
@@ -52,6 +52,10 @@ class PageItem(BaseModel):
|
||||
bBox: Optional[BBox] = Field(
|
||||
default=None, description="The bounding box of the item."
|
||||
)
|
||||
html: Optional[str] = Field(
|
||||
default=None,
|
||||
description="The HTML-formatted content of the item. Only applicable for table items when output_tables_as_HTML=True.",
|
||||
)
|
||||
|
||||
|
||||
class ImageItem(BaseModel):
|
||||
@@ -108,7 +112,7 @@ class ChartItem(BaseModel):
|
||||
class Page(BaseModel):
|
||||
"""A page of the document."""
|
||||
|
||||
page: int = Field(description="The page number.")
|
||||
page: int = Field(default=0, description="The page number.")
|
||||
text: Optional[str] = Field(default=None, description="The text of the page.")
|
||||
md: Optional[str] = Field(default=None, description="The markdown of the page.")
|
||||
images: List[ImageItem] = Field(
|
||||
@@ -149,6 +153,12 @@ class Page(BaseModel):
|
||||
noTextContent: bool = Field(
|
||||
default=False, description="Whether the page has no text content."
|
||||
)
|
||||
isAudioTranscript: bool = Field(
|
||||
default=False, description="Whether the page is an audio transcript."
|
||||
)
|
||||
durationInSeconds: Optional[float] = Field(
|
||||
default=None, description="The duration of the audio transcript in seconds."
|
||||
)
|
||||
|
||||
|
||||
class JobResult(BaseModel):
|
||||
|
||||
@@ -11,13 +11,13 @@ dev = [
|
||||
|
||||
[project]
|
||||
name = "llama-parse"
|
||||
version = "0.6.55"
|
||||
version = "0.6.57"
|
||||
description = "Parse files into RAG-Optimized formats."
|
||||
authors = [{name = "Logan Markewich", email = "logan@llamaindex.ai"}]
|
||||
requires-python = ">=3.9,<4.0"
|
||||
readme = "README.md"
|
||||
license = "MIT"
|
||||
dependencies = ["llama-cloud-services>=0.6.55"]
|
||||
dependencies = ["llama-cloud-services>=0.6.56"]
|
||||
|
||||
[project.scripts]
|
||||
llama-parse = "llama_parse.cli.main:parse"
|
||||
|
||||
+3
-2
@@ -12,12 +12,13 @@ dev = [
|
||||
"deepdiff>=8.1.1,<9",
|
||||
"ipython>=8.12.3,<9",
|
||||
"jupyter>=1.1.1,<2",
|
||||
"mypy>=1.14.1,<2"
|
||||
"mypy>=1.14.1,<2",
|
||||
"pydantic-settings>=2.10.1"
|
||||
]
|
||||
|
||||
[project]
|
||||
name = "llama-cloud-services"
|
||||
version = "0.6.55"
|
||||
version = "0.6.57"
|
||||
description = "Tailored SDK clients for LlamaCloud services."
|
||||
authors = [{name = "Logan Markewich", email = "logan@runllama.ai"}]
|
||||
requires-python = ">=3.9,<4.0"
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
[pytest]
|
||||
asyncio_mode = auto
|
||||
@@ -0,0 +1,25 @@
|
||||
import pytest
|
||||
from llama_index.core.constants import DEFAULT_BASE_URL
|
||||
from pydantic import Field, SecretStr
|
||||
from pydantic_settings import BaseSettings
|
||||
|
||||
|
||||
class EndToEndTestSettings(BaseSettings):
|
||||
LLAMA_CLOUD_BASE_URL: str = Field(
|
||||
description="The base URL of the LlamaCloud API", default=DEFAULT_BASE_URL
|
||||
)
|
||||
LLAMA_CLOUD_API_KEY: SecretStr = Field(
|
||||
description="The API key for the LlamaCloud API"
|
||||
)
|
||||
LLAMA_CLOUD_ORGANIZATION_ID: str | None = Field(
|
||||
default=None, description="The organization ID for the LlamaCloud API"
|
||||
)
|
||||
LLAMA_CLOUD_PROJECT_NAME: str = Field(
|
||||
description="The project name for the LlamaCloud API",
|
||||
default="framework_integration_test",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def e2e_test_settings() -> EndToEndTestSettings:
|
||||
return EndToEndTestSettings()
|
||||
@@ -27,8 +27,6 @@ api_key = os.environ.get("LLAMA_CLOUD_API_KEY", None)
|
||||
organization_id = os.environ.get("LLAMA_CLOUD_ORGANIZATION_ID", None)
|
||||
project_name = os.environ.get("LLAMA_CLOUD_PROJECT_NAME", "framework_integration_test")
|
||||
|
||||
print("api-key", api_key, "base-url", base_url)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def remote_file() -> Tuple[str, str]:
|
||||
|
||||
@@ -202,3 +202,12 @@ async def test_get_result(markdown_parser: LlamaParse) -> None:
|
||||
result = await markdown_parser.aget_result(expected.job_id)
|
||||
assert result.job_id == expected.job_id
|
||||
assert len(result.pages) == len(expected.pages)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parse_audio() -> None:
|
||||
parser = LlamaParse()
|
||||
filepath = "tests/test_files/hello_world.m4a"
|
||||
|
||||
result = await parser.aparse(filepath)
|
||||
assert result.job_id is not None
|
||||
|
||||
Binary file not shown.
@@ -0,0 +1,132 @@
|
||||
import os
|
||||
import pytest
|
||||
from io import BytesIO
|
||||
from llama_cloud.client import AsyncLlamaCloud
|
||||
from llama_cloud.types import Project, File
|
||||
from llama_cloud_services.files.client import FileClient
|
||||
from tests.conftest import EndToEndTestSettings
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def llama_cloud_client(
|
||||
e2e_test_settings: EndToEndTestSettings,
|
||||
) -> AsyncLlamaCloud:
|
||||
return AsyncLlamaCloud(
|
||||
token=e2e_test_settings.LLAMA_CLOUD_API_KEY.get_secret_value(),
|
||||
base_url=e2e_test_settings.LLAMA_CLOUD_BASE_URL,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def project(
|
||||
llama_cloud_client: AsyncLlamaCloud, e2e_test_settings: EndToEndTestSettings
|
||||
) -> Project:
|
||||
projects = await llama_cloud_client.projects.list_projects(
|
||||
project_name=e2e_test_settings.LLAMA_CLOUD_PROJECT_NAME,
|
||||
organization_id=e2e_test_settings.LLAMA_CLOUD_ORGANIZATION_ID,
|
||||
)
|
||||
assert len(projects) == 1
|
||||
return projects[0]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def file_client(
|
||||
llama_cloud_client: AsyncLlamaCloud, project: Project, use_presigned_url: bool
|
||||
) -> FileClient:
|
||||
return FileClient(
|
||||
llama_cloud_client,
|
||||
project_id=project.id,
|
||||
organization_id=project.organization_id,
|
||||
use_presigned_url=use_presigned_url,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_file() -> str:
|
||||
return "tests/test_files/index/Simple PDF Slides.pdf"
|
||||
|
||||
|
||||
parametrize_use_presigned_url = pytest.mark.parametrize(
|
||||
"use_presigned_url", [True, False]
|
||||
)
|
||||
|
||||
|
||||
@parametrize_use_presigned_url
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_file_from_path(file_client: FileClient, test_file: str):
|
||||
"""Test uploading a file from file path"""
|
||||
external_file_id = f"test_upload_path_{os.getpid()}"
|
||||
uploaded_file = await file_client.upload_file(test_file, external_file_id)
|
||||
|
||||
assert isinstance(uploaded_file, File)
|
||||
expected_name = os.path.basename(test_file)
|
||||
assert uploaded_file.name == expected_name
|
||||
assert uploaded_file.external_file_id == external_file_id
|
||||
|
||||
|
||||
@parametrize_use_presigned_url
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_bytes(
|
||||
file_client: FileClient, test_file: str, use_presigned_url: bool
|
||||
):
|
||||
"""Test uploading a file from bytes"""
|
||||
# Read file as bytes
|
||||
with open(test_file, "rb") as f:
|
||||
file_bytes = f.read()
|
||||
|
||||
external_file_id = f"test_upload_bytes_{os.getpid()}"
|
||||
uploaded_file = await file_client.upload_bytes(file_bytes, external_file_id)
|
||||
|
||||
assert isinstance(uploaded_file, File)
|
||||
expected_name = external_file_id if use_presigned_url else "upload"
|
||||
assert uploaded_file.name == expected_name
|
||||
assert uploaded_file.external_file_id == external_file_id
|
||||
|
||||
|
||||
@parametrize_use_presigned_url
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_buffer(
|
||||
file_client: FileClient, test_file: str, use_presigned_url: bool
|
||||
):
|
||||
"""Test uploading a file from buffer"""
|
||||
# Read file as bytes and create buffer
|
||||
with open(test_file, "rb") as f:
|
||||
file_bytes = f.read()
|
||||
|
||||
buffer = BytesIO(file_bytes)
|
||||
file_size = len(file_bytes)
|
||||
external_file_id = f"test_upload_buffer_{os.getpid()}"
|
||||
|
||||
uploaded_file = await file_client.upload_buffer(buffer, external_file_id, file_size)
|
||||
|
||||
assert isinstance(uploaded_file, File)
|
||||
expected_name = external_file_id if use_presigned_url else "upload"
|
||||
assert uploaded_file.name == expected_name
|
||||
assert uploaded_file.external_file_id == external_file_id
|
||||
|
||||
|
||||
@parametrize_use_presigned_url
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_file(file_client: FileClient, test_file: str):
|
||||
"""Test retrieving a file by ID"""
|
||||
# Upload a file first
|
||||
external_file_id = f"test_get_file_{os.getpid()}"
|
||||
uploaded_file = await file_client.upload_file(test_file, external_file_id)
|
||||
|
||||
# Retrieve the file by ID
|
||||
retrieved_file = await file_client.get_file(uploaded_file.id)
|
||||
|
||||
assert isinstance(retrieved_file, File)
|
||||
assert retrieved_file == uploaded_file
|
||||
|
||||
|
||||
@parametrize_use_presigned_url
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_with_default_external_id(file_client: FileClient, test_file: str):
|
||||
"""Test uploading file with default external_file_id"""
|
||||
# Upload file without specifying external_file_id
|
||||
uploaded_file = await file_client.upload_file(test_file)
|
||||
|
||||
assert isinstance(uploaded_file, File)
|
||||
assert uploaded_file.name == os.path.basename(test_file)
|
||||
assert uploaded_file.external_file_id == test_file
|
||||
Generated
+2266
-2250
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"type": "module",
|
||||
"main": "./dist/index.cjs",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": "./dist/index.js",
|
||||
"private": true
|
||||
}
|
||||
@@ -12855,6 +12855,72 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/extraction/run": {
|
||||
"post": {
|
||||
"tags": ["LlamaExtract"],
|
||||
"summary": "Extract Stateless",
|
||||
"description": "Stateless extraction endpoint that uses a default extraction agent in the user's default project. Requires data_schema, config, and either file_id, text, or base64 encoded file data.",
|
||||
"operationId": "extract_stateless_api_v1_extraction_run_post",
|
||||
"security": [
|
||||
{
|
||||
"HTTPBearer": []
|
||||
},
|
||||
{
|
||||
"HTTPBearer": []
|
||||
}
|
||||
],
|
||||
"parameters": [
|
||||
{
|
||||
"name": "session",
|
||||
"in": "cookie",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"title": "Session"
|
||||
}
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"required": true,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/StatelessExtractionRequest"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Successful Response",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/ExtractJob"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"422": {
|
||||
"description": "Validation Error",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/HTTPValidationError"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/extraction/jobs": {
|
||||
"get": {
|
||||
"tags": ["LlamaExtract"],
|
||||
@@ -35483,6 +35549,64 @@
|
||||
"title": "WebhookConfiguration",
|
||||
"description": "Allows the user to configure webhook options for notifications and callbacks."
|
||||
},
|
||||
"StatelessExtractionRequest": {
|
||||
"type": "object",
|
||||
"required": ["data_schema"],
|
||||
"properties": {
|
||||
"data_schema": {
|
||||
"anyOf": [
|
||||
{
|
||||
"additionalProperties": {
|
||||
"anyOf": [
|
||||
{
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"items": {},
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "boolean"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"config": {
|
||||
"$ref": "#/components/schemas/ExtractConfig",
|
||||
"description": "The configuration parameters for the extraction agent."
|
||||
},
|
||||
"file_id": {
|
||||
"type": "string",
|
||||
"format": "uuid",
|
||||
"title": "File Id",
|
||||
"description": "ID of an uploaded file to extract from"
|
||||
}
|
||||
},
|
||||
"title": "StatelessExtractionRequest",
|
||||
"description": "Request body for stateless extraction. Must include either file_id, text, or base64."
|
||||
},
|
||||
"llama_index__core__base__llms__types__ChatMessage": {
|
||||
"properties": {
|
||||
"role": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "llama-cloud-services",
|
||||
"version": "0.2.0",
|
||||
"version": "0.3.1",
|
||||
"type": "module",
|
||||
"license": "MIT",
|
||||
"scripts": {
|
||||
@@ -20,7 +20,8 @@
|
||||
"./api",
|
||||
"./reader",
|
||||
"./parse",
|
||||
"./beta/agent"
|
||||
"./beta/agent",
|
||||
"./extract"
|
||||
],
|
||||
"exports": {
|
||||
"./openapi.json": "./openapi.json",
|
||||
@@ -68,6 +69,17 @@
|
||||
},
|
||||
"default": "./parse/dist/index.js"
|
||||
},
|
||||
"./extract": {
|
||||
"require": {
|
||||
"types": "./extract/dist/index.d.cts",
|
||||
"default": "./extract/dist/index.cjs"
|
||||
},
|
||||
"import": {
|
||||
"types": "./extract/dist/index.d.ts",
|
||||
"default": "./extract/dist/index.js"
|
||||
},
|
||||
"default": "./extract/dist/index.js"
|
||||
},
|
||||
".": {
|
||||
"require": {
|
||||
"types": "./dist/index.d.cts",
|
||||
@@ -113,6 +125,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"ajv": "^8.17.1",
|
||||
"file-type": "^21.0.0",
|
||||
"p-retry": "^6.2.1",
|
||||
"zod": "^3.25.76"
|
||||
},
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
import { createClient, createConfig, type Client } from "@hey-api/client-fetch";
|
||||
import { File } from "buffer";
|
||||
import * as extract from "./extract";
|
||||
import type { ExtractAgent, ExtractConfig } from "./extract";
|
||||
import { getEnv } from "@llamaindex/env";
|
||||
import type { ExtractResult } from "./type";
|
||||
|
||||
const URLS = {
|
||||
us: "https://api.cloud.llamaindex.ai",
|
||||
eu: "https://api.cloud.eu.llamaindex.ai",
|
||||
"us-staging": "https://api.staging.llamaindex.ai",
|
||||
} as const;
|
||||
|
||||
function getUrl(baseUrl: string | undefined, region: string | undefined) {
|
||||
if (typeof baseUrl != "undefined") {
|
||||
return baseUrl;
|
||||
}
|
||||
if (typeof region === "undefined") {
|
||||
return URLS["us"];
|
||||
} else if (region === "us" || region === "eu" || region === "us-staging") {
|
||||
return URLS[region];
|
||||
} else {
|
||||
throw new Error(`Unsupported region: ${region}`);
|
||||
}
|
||||
}
|
||||
|
||||
export class LlamaExtractAgent {
|
||||
private agent: ExtractAgent;
|
||||
private client: Client;
|
||||
id: string;
|
||||
name: string;
|
||||
dataSchema: {
|
||||
[key: string]:
|
||||
| string
|
||||
| number
|
||||
| boolean
|
||||
| {
|
||||
[key: string]: unknown;
|
||||
}
|
||||
| unknown[]
|
||||
| null;
|
||||
};
|
||||
|
||||
constructor(agent: ExtractAgent, client: Client) {
|
||||
this.agent = agent;
|
||||
this.client = client;
|
||||
this.id = agent.id;
|
||||
this.name = agent.name;
|
||||
this.dataSchema = agent.data_schema;
|
||||
}
|
||||
|
||||
async extract(
|
||||
filePath: string | undefined = undefined,
|
||||
fileContent:
|
||||
| Buffer<ArrayBufferLike>
|
||||
| Uint8Array<ArrayBuffer>
|
||||
| string
|
||||
| File
|
||||
| undefined = undefined,
|
||||
fileName: string | undefined = undefined,
|
||||
project_id: string | null = null,
|
||||
organization_id: string | null = null,
|
||||
fromUi: boolean | undefined = undefined,
|
||||
pollingInterval: number = 1,
|
||||
maxPollingIterations: number = 1800,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<ExtractResult | undefined> {
|
||||
return await extract.extract(
|
||||
this.agent.id,
|
||||
filePath,
|
||||
fileContent,
|
||||
fileName,
|
||||
project_id,
|
||||
organization_id,
|
||||
this.client,
|
||||
fromUi,
|
||||
pollingInterval,
|
||||
maxPollingIterations,
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export class LlamaExtract {
|
||||
private client: Client;
|
||||
|
||||
constructor(
|
||||
apiKey: string | undefined = undefined,
|
||||
baseUrl: string | undefined = undefined,
|
||||
region: string | undefined = undefined,
|
||||
) {
|
||||
const key = apiKey ?? getEnv("LLAMA_CLOUD_API_KEY");
|
||||
if (typeof key === "undefined") {
|
||||
throw new Error(
|
||||
"No API key provided and no API key found in environment. Please pass the API key or set `LLAMA_CLOUD_API_KEY` as an environment variable.",
|
||||
);
|
||||
}
|
||||
const url = getUrl(baseUrl, region);
|
||||
this.client = createClient(
|
||||
createConfig({
|
||||
baseUrl: url,
|
||||
headers: {
|
||||
Authorization: `Bearer ${key}`,
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async createAgent(
|
||||
name: string,
|
||||
dataSchema:
|
||||
| {
|
||||
[key: string]:
|
||||
| { [key: string]: unknown }
|
||||
| Array<unknown>
|
||||
| string
|
||||
| number
|
||||
| number
|
||||
| boolean
|
||||
| null;
|
||||
}
|
||||
| string,
|
||||
config: ExtractConfig | undefined = undefined,
|
||||
project_id: string | null = null,
|
||||
organization_id: string | null = null,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<LlamaExtractAgent | undefined> {
|
||||
const agent = await extract.createAgent(
|
||||
name,
|
||||
dataSchema,
|
||||
config,
|
||||
project_id,
|
||||
organization_id,
|
||||
this.client,
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
);
|
||||
if (typeof agent != "undefined") {
|
||||
return new LlamaExtractAgent(agent, this.client);
|
||||
}
|
||||
}
|
||||
|
||||
async getAgent(
|
||||
name: string | undefined = undefined,
|
||||
id: string | undefined = undefined,
|
||||
project_id: string | null = null,
|
||||
organization_id: string | null = null,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<LlamaExtractAgent | undefined> {
|
||||
const agent = await extract.getAgent(
|
||||
id,
|
||||
name,
|
||||
project_id,
|
||||
organization_id,
|
||||
this.client,
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
);
|
||||
if (typeof agent != "undefined") {
|
||||
return new LlamaExtractAgent(agent, this.client);
|
||||
}
|
||||
}
|
||||
|
||||
async deleteAgent(
|
||||
id: string,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 500,
|
||||
): Promise<boolean | undefined> {
|
||||
return await extract.deleteAgent(
|
||||
id,
|
||||
this.client,
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
);
|
||||
}
|
||||
|
||||
async extract(
|
||||
dataSchema:
|
||||
| {
|
||||
[key: string]:
|
||||
| { [key: string]: unknown }
|
||||
| Array<unknown>
|
||||
| string
|
||||
| number
|
||||
| number
|
||||
| boolean
|
||||
| null;
|
||||
}
|
||||
| string,
|
||||
config: ExtractConfig | undefined = undefined,
|
||||
filePath: string | undefined = undefined,
|
||||
fileContent:
|
||||
| Buffer<ArrayBufferLike>
|
||||
| Uint8Array<ArrayBuffer>
|
||||
| string
|
||||
| File
|
||||
| undefined = undefined,
|
||||
fileName: string | undefined = undefined,
|
||||
project_id: string | null = null,
|
||||
organization_id: string | null = null,
|
||||
pollingInterval: number = 1,
|
||||
maxPollingIterations: number = 1800,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<ExtractResult | undefined> {
|
||||
return await extract.extractStateless(
|
||||
dataSchema,
|
||||
config,
|
||||
filePath,
|
||||
fileContent,
|
||||
fileName,
|
||||
project_id,
|
||||
organization_id,
|
||||
this.client,
|
||||
pollingInterval,
|
||||
maxPollingIterations,
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -18644,6 +18644,66 @@ export const WebhookConfigurationSchema = {
|
||||
"Allows the user to configure webhook options for notifications and callbacks.",
|
||||
} as const;
|
||||
|
||||
export const StatelessExtractionRequestSchema = {
|
||||
type: "object",
|
||||
required: ["data_schema"],
|
||||
properties: {
|
||||
data_schema: {
|
||||
anyOf: [
|
||||
{
|
||||
additionalProperties: {
|
||||
anyOf: [
|
||||
{
|
||||
additionalProperties: true,
|
||||
type: "object",
|
||||
},
|
||||
{
|
||||
items: {},
|
||||
type: "array",
|
||||
},
|
||||
{
|
||||
type: "string",
|
||||
},
|
||||
{
|
||||
type: "integer",
|
||||
},
|
||||
{
|
||||
type: "number",
|
||||
},
|
||||
{
|
||||
type: "boolean",
|
||||
},
|
||||
{
|
||||
type: "null",
|
||||
},
|
||||
],
|
||||
},
|
||||
type: "object",
|
||||
},
|
||||
{
|
||||
type: "string",
|
||||
},
|
||||
{
|
||||
type: "null",
|
||||
},
|
||||
],
|
||||
},
|
||||
config: {
|
||||
$ref: "#/components/schemas/ExtractConfig",
|
||||
description: "The configuration parameters for the extraction agent.",
|
||||
},
|
||||
file_id: {
|
||||
type: "string",
|
||||
format: "uuid",
|
||||
title: "File Id",
|
||||
description: "ID of an uploaded file to extract from",
|
||||
},
|
||||
},
|
||||
title: "StatelessExtractionRequest",
|
||||
description:
|
||||
"Request body for stateless extraction. Must include either file_id, text, or base64.",
|
||||
} as const;
|
||||
|
||||
export const llama_index__core__base__llms__types__ChatMessageSchema = {
|
||||
properties: {
|
||||
role: {
|
||||
|
||||
@@ -452,6 +452,9 @@ import type {
|
||||
UpdateExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdPutData,
|
||||
UpdateExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdPutResponse,
|
||||
UpdateExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdPutError,
|
||||
ExtractStatelessApiV1ExtractionRunPostData,
|
||||
ExtractStatelessApiV1ExtractionRunPostResponse,
|
||||
ExtractStatelessApiV1ExtractionRunPostError,
|
||||
ListJobsApiV1ExtractionJobsGetData,
|
||||
ListJobsApiV1ExtractionJobsGetResponse,
|
||||
ListJobsApiV1ExtractionJobsGetError,
|
||||
@@ -5701,6 +5704,39 @@ export const updateExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgent
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Extract Stateless
|
||||
* Stateless extraction endpoint that uses a default extraction agent in the user's default project. Requires data_schema, config, and either file_id, text, or base64 encoded file data.
|
||||
*/
|
||||
export const extractStatelessApiV1ExtractionRunPost = <
|
||||
ThrowOnError extends boolean = false,
|
||||
>(
|
||||
options: Options<ExtractStatelessApiV1ExtractionRunPostData, ThrowOnError>,
|
||||
) => {
|
||||
return (options.client ?? _heyApiClient).post<
|
||||
ExtractStatelessApiV1ExtractionRunPostResponse,
|
||||
ExtractStatelessApiV1ExtractionRunPostError,
|
||||
ThrowOnError
|
||||
>({
|
||||
security: [
|
||||
{
|
||||
scheme: "bearer",
|
||||
type: "http",
|
||||
},
|
||||
{
|
||||
scheme: "bearer",
|
||||
type: "http",
|
||||
},
|
||||
],
|
||||
url: "/api/v1/extraction/run",
|
||||
...options,
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...options?.headers,
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* List Jobs
|
||||
*/
|
||||
|
||||
@@ -8272,6 +8272,35 @@ export type WebhookConfiguration = {
|
||||
> | null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Request body for stateless extraction. Must include either file_id, text, or base64.
|
||||
*/
|
||||
export type StatelessExtractionRequest = {
|
||||
data_schema:
|
||||
| {
|
||||
[key: string]:
|
||||
| {
|
||||
[key: string]: unknown;
|
||||
}
|
||||
| Array<unknown>
|
||||
| string
|
||||
| number
|
||||
| number
|
||||
| boolean
|
||||
| null;
|
||||
}
|
||||
| string
|
||||
| null;
|
||||
/**
|
||||
* The configuration parameters for the extraction agent.
|
||||
*/
|
||||
config?: ExtractConfig;
|
||||
/**
|
||||
* ID of an uploaded file to extract from
|
||||
*/
|
||||
file_id?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Chat message.
|
||||
*/
|
||||
@@ -13078,6 +13107,33 @@ export type UpdateExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentI
|
||||
export type UpdateExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdPutResponse =
|
||||
UpdateExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdPutResponses[keyof UpdateExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdPutResponses];
|
||||
|
||||
export type ExtractStatelessApiV1ExtractionRunPostData = {
|
||||
body: StatelessExtractionRequest;
|
||||
path?: never;
|
||||
query?: never;
|
||||
url: "/api/v1/extraction/run";
|
||||
};
|
||||
|
||||
export type ExtractStatelessApiV1ExtractionRunPostErrors = {
|
||||
/**
|
||||
* Validation Error
|
||||
*/
|
||||
422: HttpValidationError;
|
||||
};
|
||||
|
||||
export type ExtractStatelessApiV1ExtractionRunPostError =
|
||||
ExtractStatelessApiV1ExtractionRunPostErrors[keyof ExtractStatelessApiV1ExtractionRunPostErrors];
|
||||
|
||||
export type ExtractStatelessApiV1ExtractionRunPostResponses = {
|
||||
/**
|
||||
* Successful Response
|
||||
*/
|
||||
200: ExtractJob;
|
||||
};
|
||||
|
||||
export type ExtractStatelessApiV1ExtractionRunPostResponse =
|
||||
ExtractStatelessApiV1ExtractionRunPostResponses[keyof ExtractStatelessApiV1ExtractionRunPostResponses];
|
||||
|
||||
export type ListJobsApiV1ExtractionJobsGetData = {
|
||||
body?: never;
|
||||
path?: never;
|
||||
|
||||
@@ -3472,6 +3472,12 @@ export const zUserOrganizationRoleCreate = z.object({
|
||||
role_id: z.string().uuid(),
|
||||
});
|
||||
|
||||
export const zStatelessExtractionRequest = z.object({
|
||||
data_schema: z.union([z.object({}), z.string(), z.null()]),
|
||||
config: zExtractConfig.optional(),
|
||||
file_id: z.string().uuid().optional(),
|
||||
});
|
||||
|
||||
export const zListKeysApiV1ApiKeysGetResponse = z.array(zApiKey);
|
||||
|
||||
export const zGenerateKeyApiV1ApiKeysPostResponse = zApiKey;
|
||||
@@ -3829,6 +3835,8 @@ export const zGetExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentId
|
||||
export const zUpdateExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdPutResponse =
|
||||
zExtractAgent;
|
||||
|
||||
export const zExtractStatelessApiV1ExtractionRunPostResponse = zExtractJob;
|
||||
|
||||
export const zListJobsApiV1ExtractionJobsGetResponse = z.array(zExtractJob);
|
||||
|
||||
export const zRunJobApiV1ExtractionJobsPostResponse = zExtractJob;
|
||||
|
||||
@@ -0,0 +1,651 @@
|
||||
import { emitWarning } from "process";
|
||||
import fs from "fs/promises";
|
||||
import { Blob } from "buffer";
|
||||
import * as path from "path";
|
||||
import type { ExtractResult } from "./type";
|
||||
import { randomUUID } from "@llamaindex/env";
|
||||
import { File } from "buffer";
|
||||
import {
|
||||
type Options,
|
||||
type ExtractAgentCreate,
|
||||
type ExtractConfig,
|
||||
type ExtractJobCreate,
|
||||
type ExtractAgent,
|
||||
type ExtractJob,
|
||||
type CreateExtractionAgentApiV1ExtractionExtractionAgentsPostData,
|
||||
type GetExtractionAgentByNameApiV1ExtractionExtractionAgentsByNameNameGetData,
|
||||
type GetExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdGetData,
|
||||
type RunJobApiV1ExtractionJobsPostData,
|
||||
type GetJobApiV1ExtractionJobsJobIdGetData,
|
||||
type GetJobResultApiV1ExtractionJobsJobIdResultGetData,
|
||||
StatusEnum,
|
||||
type UploadFileApiV1FilesPostData,
|
||||
type StatelessExtractionRequest,
|
||||
type ExtractStatelessApiV1ExtractionRunPostData,
|
||||
type DeleteExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdDeleteData,
|
||||
createExtractionAgentApiV1ExtractionExtractionAgentsPost,
|
||||
getExtractionAgentByNameApiV1ExtractionExtractionAgentsByNameNameGet,
|
||||
getExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdGet,
|
||||
runJobApiV1ExtractionJobsPost,
|
||||
getJobApiV1ExtractionJobsJobIdGet,
|
||||
getJobResultApiV1ExtractionJobsJobIdResultGet,
|
||||
uploadFileApiV1FilesPost,
|
||||
extractStatelessApiV1ExtractionRunPost,
|
||||
deleteExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdDelete,
|
||||
} from "./api";
|
||||
import type { Client } from "@hey-api/client-fetch";
|
||||
import { sleep } from "./utils";
|
||||
import { fileTypeFromBuffer } from "file-type";
|
||||
|
||||
type BodyUploadFileApiV1FilesPost = {
|
||||
upload_file: Blob | File;
|
||||
};
|
||||
|
||||
export async function createAgent(
|
||||
name: string,
|
||||
dataSchema:
|
||||
| {
|
||||
[key: string]:
|
||||
| { [key: string]: unknown }
|
||||
| Array<unknown>
|
||||
| string
|
||||
| number
|
||||
| number
|
||||
| boolean
|
||||
| null;
|
||||
}
|
||||
| string,
|
||||
config: ExtractConfig = {} as ExtractConfig,
|
||||
project_id: string | null = null,
|
||||
organization_id: string | null = null,
|
||||
client: Client | undefined = undefined,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<ExtractAgent | undefined> {
|
||||
const agentData = {
|
||||
name: name,
|
||||
data_schema: dataSchema,
|
||||
config: config,
|
||||
} as ExtractAgentCreate;
|
||||
const agentDataCreation = {
|
||||
body: agentData,
|
||||
query: { project_id: project_id, organization_id: organization_id },
|
||||
} as CreateExtractionAgentApiV1ExtractionExtractionAgentsPostData;
|
||||
const options =
|
||||
agentDataCreation as Options<CreateExtractionAgentApiV1ExtractionExtractionAgentsPostData>;
|
||||
if (typeof client != "undefined") {
|
||||
options.client = client;
|
||||
}
|
||||
let retries: number = 0;
|
||||
while (true) {
|
||||
if (retries > maxRetriesOnError) {
|
||||
throw new Error(
|
||||
"Error while creating the agent: Exceeded maximum number of retries, the API keeps returning errors.",
|
||||
);
|
||||
}
|
||||
const response =
|
||||
await createExtractionAgentApiV1ExtractionExtractionAgentsPost(options);
|
||||
if (!response.response.ok) {
|
||||
if ("error" in response) {
|
||||
console.log(
|
||||
`An error occurred while creating the extraction agent.\nDetails:\n\n${JSON.stringify(
|
||||
response.error,
|
||||
)}\n\nRetrying...`,
|
||||
);
|
||||
}
|
||||
retries++;
|
||||
await sleep(retryInterval * 1000);
|
||||
} else {
|
||||
return response.data as ExtractAgent;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function getAgent(
|
||||
id: string | undefined = undefined,
|
||||
name: string | undefined = undefined,
|
||||
project_id: string | null = null,
|
||||
organization_id: string | null = null,
|
||||
client: Client | undefined = undefined,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<ExtractAgent | undefined> {
|
||||
if (typeof id === "undefined" && typeof name === "undefined") {
|
||||
throw new Error("One of `id` and `string` must be passed.");
|
||||
} else if (typeof id != "undefined" && typeof name != "undefined") {
|
||||
emitWarning("You passed both `id` and `name`, using only id...");
|
||||
const data = {
|
||||
path: { extraction_agent_id: id },
|
||||
} as GetExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdGetData;
|
||||
const options =
|
||||
data as Options<GetExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdGetData>;
|
||||
if (typeof client != "undefined") {
|
||||
options.client = client;
|
||||
}
|
||||
let retries: number = 0;
|
||||
while (true) {
|
||||
if (retries > maxRetriesOnError) {
|
||||
throw new Error(
|
||||
"Error while getting the agent: Exceeded maximum number of retries, the API keeps returning errors.",
|
||||
);
|
||||
}
|
||||
const response =
|
||||
await getExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdGet(
|
||||
options,
|
||||
);
|
||||
if (!response.response.ok) {
|
||||
if ("error" in response) {
|
||||
console.log(
|
||||
`An error occurred while getting the extraction agent by ID.\nDetails:\n\n${JSON.stringify(
|
||||
response.error,
|
||||
)}\n\nRetrying...`,
|
||||
);
|
||||
}
|
||||
retries++;
|
||||
await sleep(retryInterval * 1000);
|
||||
} else {
|
||||
return response.data as ExtractAgent;
|
||||
}
|
||||
}
|
||||
} else if (typeof name != "undefined" && typeof id === "undefined") {
|
||||
const data = {
|
||||
path: { name: name },
|
||||
query: { organization_id: organization_id, project_id: project_id },
|
||||
} as GetExtractionAgentByNameApiV1ExtractionExtractionAgentsByNameNameGetData;
|
||||
const options =
|
||||
data as Options<GetExtractionAgentByNameApiV1ExtractionExtractionAgentsByNameNameGetData>;
|
||||
if (typeof client != "undefined") {
|
||||
options.client = client;
|
||||
}
|
||||
let retries: number = 0;
|
||||
while (true) {
|
||||
if (retries > maxRetriesOnError) {
|
||||
throw new Error(
|
||||
"Error while getting the agent: Exceeded maximum number of retries, the API keeps returning errors.",
|
||||
);
|
||||
}
|
||||
const response =
|
||||
await getExtractionAgentByNameApiV1ExtractionExtractionAgentsByNameNameGet(
|
||||
options,
|
||||
);
|
||||
if (!response.response.ok) {
|
||||
if ("error" in response) {
|
||||
console.log(
|
||||
`An error occurred while getting the extraction agent by name.\nDetails:\n\n${JSON.stringify(
|
||||
response.error,
|
||||
)}\n\nRetrying...`,
|
||||
);
|
||||
}
|
||||
retries++;
|
||||
await sleep(retryInterval * 1000);
|
||||
} else {
|
||||
return response.data as ExtractAgent;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const data = {
|
||||
path: { extraction_agent_id: id },
|
||||
} as GetExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdGetData;
|
||||
const options =
|
||||
data as Options<GetExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdGetData>;
|
||||
if (typeof client != "undefined") {
|
||||
options.client = client;
|
||||
}
|
||||
let retries: number = 0;
|
||||
while (true) {
|
||||
if (retries > maxRetriesOnError) {
|
||||
throw new Error(
|
||||
"Error while getting the agent: Exceeded maximum number of retries, the API keeps returning errors.",
|
||||
);
|
||||
}
|
||||
const response =
|
||||
await getExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdGet(
|
||||
options,
|
||||
);
|
||||
if (!response.response.ok) {
|
||||
if (!response.response.ok) {
|
||||
if ("error" in response) {
|
||||
console.log(
|
||||
`An error occurred while getting the extraction agent by ID.\nDetails:\n\n${JSON.stringify(
|
||||
response.error,
|
||||
)}\n\nRetrying...`,
|
||||
);
|
||||
}
|
||||
retries++;
|
||||
await sleep(retryInterval * 1000);
|
||||
}
|
||||
} else {
|
||||
return response.data as ExtractAgent;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function textToFile(text: string, fileName: string | null = null) {
|
||||
return new File(
|
||||
[text],
|
||||
fileName ?? "uploadedFile_" + randomUUID().replaceAll("-", "_") + ".txt",
|
||||
);
|
||||
}
|
||||
|
||||
async function uploadFile(
|
||||
filePath: string | undefined = undefined,
|
||||
fileContent:
|
||||
| Buffer<ArrayBufferLike>
|
||||
| File
|
||||
| Uint8Array<ArrayBuffer>
|
||||
| string
|
||||
| undefined = undefined,
|
||||
fileName: string | undefined = undefined,
|
||||
project_id: string | null = null,
|
||||
organization_id: string | null = null,
|
||||
client: Client | undefined = undefined,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<string | undefined> {
|
||||
let file: File | undefined = undefined;
|
||||
if (typeof filePath === "undefined" && typeof fileContent === "undefined") {
|
||||
throw new Error(
|
||||
"One between filePath and fileContent needs to be provided",
|
||||
);
|
||||
} else if (typeof filePath != "undefined") {
|
||||
const buffer = await fs.readFile(filePath);
|
||||
const actualFileName = fileName ?? path.basename(filePath);
|
||||
const uint8Array = new Uint8Array(buffer);
|
||||
file = new File([uint8Array], actualFileName);
|
||||
} else if (typeof fileContent != "undefined") {
|
||||
if (fileContent instanceof File) {
|
||||
file = fileContent;
|
||||
} else if (fileContent instanceof Buffer) {
|
||||
const fileType = await fileTypeFromBuffer(fileContent);
|
||||
const ext = fileType?.ext ?? "pdf";
|
||||
const uint8Array = new Uint8Array(fileContent);
|
||||
file = new File(
|
||||
[uint8Array],
|
||||
fileName ??
|
||||
"uploadedFile_" + randomUUID().replaceAll("-", "_") + "." + ext,
|
||||
);
|
||||
} else if (fileContent instanceof Uint8Array) {
|
||||
const fileType = await fileTypeFromBuffer(fileContent);
|
||||
const ext = fileType?.ext ?? "pdf";
|
||||
file = new File(
|
||||
[fileContent],
|
||||
fileName ??
|
||||
"uploadedFile_" + randomUUID().replaceAll("-", "_") + "." + ext,
|
||||
);
|
||||
} else if (typeof fileContent === "string") {
|
||||
file = textToFile(fileContent, fileName);
|
||||
} else {
|
||||
throw new Error("Unsupported fileContent type");
|
||||
}
|
||||
}
|
||||
const fileToUpload = {
|
||||
upload_file: file,
|
||||
} as BodyUploadFileApiV1FilesPost;
|
||||
const uploadData = {
|
||||
body: fileToUpload,
|
||||
query: { organization_id: organization_id, project_id: project_id },
|
||||
} as UploadFileApiV1FilesPostData;
|
||||
const uploadOptions = uploadData as Options<UploadFileApiV1FilesPostData>;
|
||||
if (typeof client != "undefined") {
|
||||
uploadOptions.client = client;
|
||||
}
|
||||
let retries: number = 0;
|
||||
while (true) {
|
||||
if (retries > maxRetriesOnError) {
|
||||
throw new Error(
|
||||
"Error while processing your file: Exceeded maximum number of retries, the API keeps returning errors.",
|
||||
);
|
||||
}
|
||||
const uploadResponse = await uploadFileApiV1FilesPost(uploadOptions);
|
||||
let fileId: string | undefined = undefined;
|
||||
if (!uploadResponse.response.ok) {
|
||||
retries++;
|
||||
await sleep(retryInterval * 1000);
|
||||
}
|
||||
if (typeof uploadResponse.data != "undefined") {
|
||||
fileId = uploadResponse.data.id as string;
|
||||
return fileId;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function createExtractJob(
|
||||
options:
|
||||
| Options<RunJobApiV1ExtractionJobsPostData>
|
||||
| Options<ExtractStatelessApiV1ExtractionRunPostData>,
|
||||
stateless: boolean = false,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<string | undefined> {
|
||||
let retries: number = 0;
|
||||
while (true) {
|
||||
if (retries > maxRetriesOnError) {
|
||||
throw new Error(
|
||||
"Error while creating the extraction job: Exceeded maximum number of retries, the API keeps returning errors.",
|
||||
);
|
||||
}
|
||||
let response:
|
||||
| {
|
||||
data: ExtractJob | undefined;
|
||||
request: Request;
|
||||
response: Response;
|
||||
}
|
||||
| undefined = undefined;
|
||||
if (!stateless) {
|
||||
response = (await runJobApiV1ExtractionJobsPost(
|
||||
options as Options<RunJobApiV1ExtractionJobsPostData>,
|
||||
)) as {
|
||||
data: ExtractJob | undefined;
|
||||
request: Request;
|
||||
response: Response;
|
||||
};
|
||||
} else {
|
||||
response = (await extractStatelessApiV1ExtractionRunPost(
|
||||
options as Options<ExtractStatelessApiV1ExtractionRunPostData>,
|
||||
)) as {
|
||||
data: ExtractJob | undefined;
|
||||
request: Request;
|
||||
response: Response;
|
||||
};
|
||||
}
|
||||
if (!response.response.ok) {
|
||||
if ("error" in response) {
|
||||
console.log(
|
||||
"An error occurred: ",
|
||||
JSON.stringify(response.error),
|
||||
"\nRetrying...",
|
||||
);
|
||||
}
|
||||
retries++;
|
||||
await sleep(retryInterval * 1000);
|
||||
}
|
||||
if (typeof response.data != "undefined") {
|
||||
const jobStatus = response.data.status as StatusEnum;
|
||||
if (jobStatus == "CANCELLED") {
|
||||
retries++;
|
||||
await sleep(retryInterval * 1000);
|
||||
} else if (jobStatus == "ERROR") {
|
||||
retries++;
|
||||
await sleep(retryInterval * 1000);
|
||||
} else {
|
||||
return response.data.id as string;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function pollForJobCompletion(
|
||||
jobId: string,
|
||||
interval: number = 1,
|
||||
maxIterations: number = 1800,
|
||||
client: Client | undefined = undefined,
|
||||
): Promise<boolean> {
|
||||
let status: StatusEnum | undefined = undefined;
|
||||
const jobData = {
|
||||
path: { job_id: jobId },
|
||||
} as GetJobApiV1ExtractionJobsJobIdGetData;
|
||||
const jobOptions = jobData as Options<GetJobApiV1ExtractionJobsJobIdGetData>;
|
||||
if (typeof client != "undefined") {
|
||||
jobOptions.client = client;
|
||||
}
|
||||
let numIterations: number = 0;
|
||||
while (true) {
|
||||
if (numIterations > maxIterations) {
|
||||
return false;
|
||||
}
|
||||
const response = await getJobApiV1ExtractionJobsJobIdGet(jobOptions);
|
||||
if (!response.response.ok) {
|
||||
numIterations++;
|
||||
}
|
||||
if (typeof response.data != "undefined") {
|
||||
status = response.data.status as StatusEnum;
|
||||
if (status == StatusEnum.CANCELLED || status == StatusEnum.ERROR) {
|
||||
throw new Error("There was an error extracting data from your file.");
|
||||
} else if (status == StatusEnum.SUCCESS) {
|
||||
return true;
|
||||
} else {
|
||||
numIterations++;
|
||||
await sleep(interval * 1000);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function getJobResult(
|
||||
jobId: string,
|
||||
client: Client | undefined = undefined,
|
||||
project_id: string | null = null,
|
||||
organization_id: string | null = null,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<ExtractResult | undefined> {
|
||||
const jobData = {
|
||||
path: { job_id: jobId },
|
||||
query: { organization_id: organization_id, project_id: project_id },
|
||||
} as GetJobResultApiV1ExtractionJobsJobIdResultGetData;
|
||||
const jobOptions =
|
||||
jobData as Options<GetJobResultApiV1ExtractionJobsJobIdResultGetData>;
|
||||
if (typeof client != "undefined") {
|
||||
jobOptions.client = client;
|
||||
}
|
||||
let retries: number = 0;
|
||||
while (true) {
|
||||
if (retries > maxRetriesOnError) {
|
||||
throw new Error(
|
||||
"Error while getting the result of the extraction job: Exceeded maximum number of retries, the API keeps returning errors.",
|
||||
);
|
||||
}
|
||||
const response =
|
||||
await getJobResultApiV1ExtractionJobsJobIdResultGet(jobOptions);
|
||||
if (!response.response.ok) {
|
||||
if ("error" in response) {
|
||||
console.log(
|
||||
"An error occurred: ",
|
||||
JSON.stringify(response.error),
|
||||
"\nRetrying...",
|
||||
);
|
||||
}
|
||||
retries++;
|
||||
await sleep(retryInterval * 1000);
|
||||
}
|
||||
if (typeof response.data != "undefined") {
|
||||
return {
|
||||
data: response.data.data,
|
||||
extractionMetadata: response.data.extraction_metadata,
|
||||
} as ExtractResult;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function extract(
|
||||
agentId: string,
|
||||
filePath: string | undefined = undefined,
|
||||
fileContent:
|
||||
| Buffer<ArrayBufferLike>
|
||||
| File
|
||||
| Uint8Array<ArrayBuffer>
|
||||
| string
|
||||
| undefined = undefined,
|
||||
fileName: string | undefined = undefined,
|
||||
project_id: string | null = null,
|
||||
organization_id: string | null = null,
|
||||
client: Client | undefined = undefined,
|
||||
fromUi: boolean | undefined = undefined,
|
||||
pollingInterval: number = 1,
|
||||
maxPollingIterations: number = 1800,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<ExtractResult | undefined> {
|
||||
const fileId = (await uploadFile(
|
||||
filePath,
|
||||
fileContent,
|
||||
fileName,
|
||||
project_id,
|
||||
organization_id,
|
||||
client,
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
)) as string;
|
||||
const extractJobCreate = {
|
||||
extraction_agent_id: agentId,
|
||||
file_id: fileId,
|
||||
} as ExtractJobCreate;
|
||||
const extractData = {
|
||||
body: extractJobCreate,
|
||||
query: { from_ui: fromUi },
|
||||
} as RunJobApiV1ExtractionJobsPostData;
|
||||
const extractOptions =
|
||||
extractData as Options<RunJobApiV1ExtractionJobsPostData>;
|
||||
if (typeof client != "undefined") {
|
||||
extractOptions.client = client;
|
||||
}
|
||||
const jobId = (await createExtractJob(
|
||||
extractOptions,
|
||||
false,
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
)) as string;
|
||||
const success = await pollForJobCompletion(
|
||||
jobId,
|
||||
pollingInterval,
|
||||
maxPollingIterations,
|
||||
client,
|
||||
);
|
||||
if (!success) {
|
||||
throw new Error("Your job is taking longer than 10 minutes, timing out...");
|
||||
} else {
|
||||
return (await getJobResult(
|
||||
jobId,
|
||||
client,
|
||||
project_id,
|
||||
organization_id,
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
)) as ExtractResult;
|
||||
}
|
||||
}
|
||||
|
||||
export async function extractStateless(
|
||||
dataSchema:
|
||||
| {
|
||||
[key: string]:
|
||||
| { [key: string]: unknown }
|
||||
| Array<unknown>
|
||||
| string
|
||||
| number
|
||||
| number
|
||||
| boolean
|
||||
| null;
|
||||
}
|
||||
| string,
|
||||
config: ExtractConfig = {} as ExtractConfig,
|
||||
filePath: string | undefined = undefined,
|
||||
fileContent:
|
||||
| Buffer<ArrayBufferLike>
|
||||
| File
|
||||
| Uint8Array<ArrayBuffer>
|
||||
| string
|
||||
| undefined = undefined,
|
||||
fileName: string | undefined = undefined,
|
||||
project_id: string | null = null,
|
||||
organization_id: string | null = null,
|
||||
client: Client | undefined = undefined,
|
||||
pollingInterval: number = 1,
|
||||
maxPollingIterations: number = 1800,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<ExtractResult | undefined> {
|
||||
const fileId = (await uploadFile(
|
||||
filePath,
|
||||
fileContent,
|
||||
fileName,
|
||||
project_id,
|
||||
organization_id,
|
||||
client,
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
)) as string;
|
||||
const extractStatetelessCreate = {
|
||||
data_schema: dataSchema,
|
||||
file_id: fileId,
|
||||
config: config,
|
||||
} as StatelessExtractionRequest;
|
||||
const extractStatetelessData = {
|
||||
body: extractStatetelessCreate,
|
||||
} as ExtractStatelessApiV1ExtractionRunPostData;
|
||||
const extractOptions =
|
||||
extractStatetelessData as Options<ExtractStatelessApiV1ExtractionRunPostData>;
|
||||
if (typeof client != "undefined") {
|
||||
extractOptions.client = client;
|
||||
}
|
||||
const jobId = (await createExtractJob(
|
||||
extractOptions,
|
||||
true,
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
)) as string;
|
||||
const success = await pollForJobCompletion(
|
||||
jobId,
|
||||
pollingInterval,
|
||||
maxPollingIterations,
|
||||
client,
|
||||
);
|
||||
if (!success) {
|
||||
throw new Error("Your job is taking longer than 10 minutes, timing out...");
|
||||
} else {
|
||||
return (await getJobResult(
|
||||
jobId,
|
||||
client,
|
||||
project_id,
|
||||
organization_id,
|
||||
maxRetriesOnError,
|
||||
retryInterval,
|
||||
)) as ExtractResult;
|
||||
}
|
||||
}
|
||||
|
||||
export async function deleteAgent(
|
||||
id: string,
|
||||
client: Client | undefined = undefined,
|
||||
maxRetriesOnError: number = 10,
|
||||
retryInterval: number = 0.5,
|
||||
): Promise<boolean | undefined> {
|
||||
const deleteData = {
|
||||
path: { extraction_agent_id: id },
|
||||
} as DeleteExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdDeleteData;
|
||||
const deleteOptions =
|
||||
deleteData as Options<DeleteExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdDeleteData>;
|
||||
if (typeof client != "undefined") {
|
||||
deleteOptions.client = client;
|
||||
}
|
||||
let retries: number = 0;
|
||||
while (true) {
|
||||
if (retries > maxRetriesOnError) {
|
||||
throw new Error(
|
||||
"Maximum number of attempts for deleting agent " +
|
||||
id +
|
||||
" reached, but the API continues to return errors.",
|
||||
);
|
||||
}
|
||||
const response =
|
||||
await deleteExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdDelete(
|
||||
deleteOptions,
|
||||
);
|
||||
if (!response.response.ok) {
|
||||
if ("error" in response) {
|
||||
console.log(
|
||||
`An error occurred while deleting the agent: ${JSON.stringify(
|
||||
response.error,
|
||||
)}\nRetrying...`,
|
||||
);
|
||||
}
|
||||
retries++;
|
||||
await sleep(retryInterval * 1000);
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export { type ExtractAgent, type ExtractConfig };
|
||||
@@ -6,3 +6,5 @@ export {
|
||||
} from "./LlamaCloudRetriever.js";
|
||||
export type { CloudConstructorParams } from "./type.js";
|
||||
export { LlamaParseReader } from "./reader.js";
|
||||
export { LlamaExtract, LlamaExtractAgent } from "./LlamaExtract.js";
|
||||
export type { ExtractConfig } from "./extract.js";
|
||||
|
||||
@@ -14,7 +14,8 @@ import {
|
||||
getJobTextResultApiV1ParsingJobJobIdResultTextGet,
|
||||
uploadFileApiV1ParsingUploadPost,
|
||||
} from "./api";
|
||||
import { sleep } from "./utils";
|
||||
import { sleep, getSavePath } from "./utils";
|
||||
import type { ParseResult } from "./type";
|
||||
|
||||
export type Language = ParserLanguages;
|
||||
export type ResultType = "text" | "markdown" | "json";
|
||||
@@ -594,15 +595,15 @@ export class LlamaParseReader extends FileReader {
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads data from a file and returns an array of JSON objects.
|
||||
* Loads data from a file and returns an array of ParseResult objects.
|
||||
* To be used with resultType "json".
|
||||
*
|
||||
* @param filePathOrContent - The file path or the file content as a Uint8Array.
|
||||
* @returns A Promise that resolves to an array of JSON objects.
|
||||
* @returns A Promise that resolves to an array of ParseResult objects, encapsulating the JSON array resulting from the parsing within the pages attribute.
|
||||
*/
|
||||
async loadJson(
|
||||
filePathOrContent: string | Uint8Array,
|
||||
): Promise<Record<string, any>[]> {
|
||||
): Promise<ParseResult[]> {
|
||||
let jobId;
|
||||
const isFilePath =
|
||||
typeof filePathOrContent === "string" &&
|
||||
@@ -628,7 +629,7 @@ export class LlamaParseReader extends FileReader {
|
||||
const resultJson = await this.getJobResult(jobId, "json");
|
||||
resultJson.job_id = jobId;
|
||||
resultJson.file_path = isFilePath ? filePathOrContent : undefined;
|
||||
return [resultJson];
|
||||
return [resultJson] as ParseResult[];
|
||||
} catch (e) {
|
||||
console.error(`Error while parsing the file under job id ${jobId}`, e);
|
||||
if (this.ignoreErrors) {
|
||||
@@ -639,6 +640,80 @@ export class LlamaParseReader extends FileReader {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads data from a file or file bytes (or an array of those) and returns an array of ParseResult objects.
|
||||
*
|
||||
* @param filePathOrContent - The file path or the file content as a Uint8Array, or an array of one of those two types.
|
||||
* @returns A Promise that resolves to an array of ParseResult objects, encapsulating the JSON array resulting from the parsing within the pages attribute.
|
||||
*/
|
||||
async parse(
|
||||
filePathOrContent: string | Uint8Array | string[] | Uint8Array[],
|
||||
): Promise<ParseResult[]> {
|
||||
const jsonResults: Record<string, any>[][] = [];
|
||||
if (!Array.isArray(filePathOrContent)) {
|
||||
const jsonResult = await this.loadJson(filePathOrContent);
|
||||
jsonResults.push(jsonResult);
|
||||
} else {
|
||||
for (let i = 0; i < filePathOrContent.length; i++) {
|
||||
console.log(
|
||||
`Processing file ${i + 1} of ${filePathOrContent.length}...`,
|
||||
);
|
||||
const jsonResult = await this.loadJson(
|
||||
filePathOrContent[i] as string | Uint8Array,
|
||||
);
|
||||
jsonResults.push(jsonResult);
|
||||
}
|
||||
}
|
||||
const parseResults: ParseResult[] = [];
|
||||
for (const jsonResult of jsonResults) {
|
||||
for (const result of jsonResult) {
|
||||
const parseResult = {
|
||||
pages: result.pages,
|
||||
job_metadata: result.job_metadata,
|
||||
job_id: result.job_id,
|
||||
file_path: result?.file_path ?? "",
|
||||
is_completed: true,
|
||||
} as ParseResult;
|
||||
parseResults.push(parseResult);
|
||||
}
|
||||
}
|
||||
return parseResults;
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads and saves tables from a given array of ParseResult to a specified download path.
|
||||
*
|
||||
* @param jsonResults - The array of ParseResult containing table information.
|
||||
* @param downloadPath - The path where the downloaded tables will be saved as CSV files.
|
||||
* @returns A Promise that resolves to an array of strings representing the paths to the tables.
|
||||
*/
|
||||
async getTables(
|
||||
jsonResults: ParseResult[],
|
||||
downloadPath: string,
|
||||
): Promise<string[]> {
|
||||
const tables: string[] = [];
|
||||
for (const result of jsonResults) {
|
||||
for (const page of result.pages) {
|
||||
if ("items" in page && Array.isArray(page.items)) {
|
||||
for (let i = 0; i < page.items.length; i++) {
|
||||
if (
|
||||
"type" in page.items[i] &&
|
||||
page.items[i].type === "table" &&
|
||||
"csv" in page.items[i] &&
|
||||
typeof page.items[i].csv === "string" &&
|
||||
page.items[i].csv != ""
|
||||
) {
|
||||
const savePath = getSavePath(downloadPath, i);
|
||||
await fs.writeFile(savePath, page.items[i].csv);
|
||||
tables.push(savePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return tables;
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads and saves images from a given JSON result to a specified download path.
|
||||
* Currently only supports resultType "json".
|
||||
@@ -648,7 +723,7 @@ export class LlamaParseReader extends FileReader {
|
||||
* @returns A Promise that resolves to an array of image objects.
|
||||
*/
|
||||
async getImages(
|
||||
jsonResult: Record<string, any>[],
|
||||
jsonResult: ParseResult[],
|
||||
downloadPath: string,
|
||||
): Promise<Record<string, any>[]> {
|
||||
try {
|
||||
|
||||
@@ -8,3 +8,54 @@ export type CloudConstructorParams = {
|
||||
projectName: string;
|
||||
organizationId?: string | undefined;
|
||||
} & ClientParams;
|
||||
|
||||
export type ExtractResult = {
|
||||
data:
|
||||
| {
|
||||
[key: string]:
|
||||
| {
|
||||
[key: string]: unknown;
|
||||
}
|
||||
| Array<unknown>
|
||||
| string
|
||||
| number
|
||||
| number
|
||||
| boolean
|
||||
| null;
|
||||
}
|
||||
| Array<{
|
||||
[key: string]:
|
||||
| {
|
||||
[key: string]: unknown;
|
||||
}
|
||||
| Array<unknown>
|
||||
| string
|
||||
| number
|
||||
| number
|
||||
| boolean
|
||||
| null;
|
||||
}>
|
||||
| null;
|
||||
extractionMetadata: {
|
||||
[key: string]:
|
||||
| {
|
||||
[key: string]: unknown;
|
||||
}
|
||||
| Array<unknown>
|
||||
| string
|
||||
| number
|
||||
| number
|
||||
| boolean
|
||||
| null;
|
||||
};
|
||||
};
|
||||
|
||||
export type ParseResult = {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
pages: Record<string, any>[];
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
job_metadata: Record<string, any>;
|
||||
job_id: string;
|
||||
is_completed: boolean;
|
||||
file_path: string;
|
||||
};
|
||||
|
||||
@@ -6,6 +6,8 @@ import {
|
||||
import { DEFAULT_BASE_URL } from "@llamaindex/core/global";
|
||||
import { getEnv } from "@llamaindex/env";
|
||||
import type { ClientParams } from "./type.js";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
|
||||
function getBaseUrl(baseUrl?: string): string {
|
||||
return baseUrl ?? getEnv("LLAMA_CLOUD_BASE_URL") ?? DEFAULT_BASE_URL;
|
||||
@@ -99,3 +101,19 @@ export async function getPipelineId(
|
||||
export async function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
export function getSavePath(downloadPath: string, i: number): string {
|
||||
const now = new Date();
|
||||
const formatted =
|
||||
now.toISOString().replace(/[-:T]/g, "_").replace(/\..+/, "") +
|
||||
"_" +
|
||||
now.getMilliseconds().toString().padStart(3, "0");
|
||||
|
||||
let savePath = path.join(downloadPath, `table_${formatted}.csv`);
|
||||
|
||||
if (fs.existsSync(savePath)) {
|
||||
savePath = savePath.replace(".csv", "_") + i.toString() + ".csv";
|
||||
}
|
||||
|
||||
return savePath;
|
||||
}
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
import { describe, it, expect, beforeEach, beforeAll } from "vitest";
|
||||
import { LlamaParseReader } from "../src/reader.js";
|
||||
import { LlamaCloudIndex } from "../src/LlamaCloudIndex.js";
|
||||
import { LlamaExtract, LlamaExtractAgent } from "../src/LlamaExtract.js";
|
||||
import { Document } from "@llamaindex/core/schema";
|
||||
import { fs } from "@llamaindex/env";
|
||||
import { ExtractConfig } from "../src/api.js";
|
||||
import { ParseResult } from "../src/type.js";
|
||||
|
||||
// Integration tests that require actual API keys and files
|
||||
describe("Integration Tests", () => {
|
||||
@@ -282,6 +285,78 @@ describe("Integration Tests", () => {
|
||||
},
|
||||
60000,
|
||||
);
|
||||
|
||||
it.skipIf(skipIfNoApiKey)(
|
||||
"should parse a file and return a ParseResult array",
|
||||
async () => {
|
||||
const parseReader = new LlamaParseReader({
|
||||
apiKey: process.env.LLAMA_CLOUD_API_KEY!,
|
||||
verbose: false,
|
||||
});
|
||||
|
||||
const testContent = "Test document for JSON parsing";
|
||||
const testFilePath = "test-json.txt";
|
||||
|
||||
await fs.writeFile(testFilePath, new TextEncoder().encode(testContent));
|
||||
|
||||
try {
|
||||
const result = await parseReader.parse(testFilePath);
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(Array.isArray(result)).toBe(true);
|
||||
expect(result.length).toBeGreaterThan(0);
|
||||
expect(result[0]).toHaveProperty("job_id");
|
||||
expect(result[0]).toHaveProperty("job_metadata");
|
||||
expect(result[0]).toHaveProperty("file_path");
|
||||
expect(result[0]).toHaveProperty("pages");
|
||||
|
||||
await fs.unlink(testFilePath);
|
||||
} catch (error) {
|
||||
try {
|
||||
await fs.unlink(testFilePath);
|
||||
} catch {}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
60000,
|
||||
);
|
||||
|
||||
it.skipIf(skipIfNoApiKey)(
|
||||
"should extract tables correctly from a JSON result",
|
||||
async () => {
|
||||
const parseReader = new LlamaParseReader({
|
||||
apiKey: process.env.LLAMA_CLOUD_API_KEY!,
|
||||
verbose: false,
|
||||
});
|
||||
const pseudoJsonResult = [
|
||||
{
|
||||
pages: [
|
||||
{
|
||||
items: [
|
||||
{
|
||||
type: "table",
|
||||
csv: "Name,Age,Height (cm)\nAnna,12,140\nBob,22,175\nClaire,33,173\nDenis,44,185\n",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
job_id: "jobId",
|
||||
job_metadata: { job_id: "jobId" },
|
||||
file_path: "table.csv",
|
||||
is_completed: true,
|
||||
},
|
||||
] as ParseResult[];
|
||||
|
||||
const tmpdir = await fs.mkdtemp("tables");
|
||||
const result = await parseReader.getTables(pseudoJsonResult, tmpdir);
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(Array.isArray(result)).toBe(true);
|
||||
expect(result.length).toBeGreaterThan(0);
|
||||
expect(typeof result[0] === "string").toBe(true);
|
||||
},
|
||||
60000,
|
||||
);
|
||||
});
|
||||
|
||||
describe("LlamaCloudIndex Integration", () => {
|
||||
@@ -414,6 +489,121 @@ describe("Integration Tests", () => {
|
||||
);
|
||||
});
|
||||
|
||||
describe("LlamaExtract Integration", () => {
|
||||
it.skipIf(skipIfNoApiKey)(
|
||||
"should create agents correctly",
|
||||
async () => {
|
||||
const dataSchema = {
|
||||
properties: {
|
||||
text: {
|
||||
description: "Text from the file",
|
||||
type: "string",
|
||||
},
|
||||
},
|
||||
required: ["text"],
|
||||
type: "object",
|
||||
};
|
||||
const extractClient = new LlamaExtract(
|
||||
process.env.LLAMA_CLOUD_API_KEY!,
|
||||
"https://api.cloud.llamaindex.ai",
|
||||
);
|
||||
const agent = await extractClient.createAgent(
|
||||
"ExtractTestAgent",
|
||||
dataSchema,
|
||||
);
|
||||
expect(agent).instanceOf(LlamaExtractAgent);
|
||||
},
|
||||
60000,
|
||||
);
|
||||
it.skipIf(skipIfNoApiKey)(
|
||||
"should fetch agents correctly",
|
||||
async () => {
|
||||
const extractClient = new LlamaExtract(
|
||||
process.env.LLAMA_CLOUD_API_KEY!,
|
||||
"https://api.cloud.llamaindex.ai",
|
||||
);
|
||||
const agent = await extractClient.getAgent("ExtractTestAgent");
|
||||
expect(agent).instanceOf(LlamaExtractAgent);
|
||||
},
|
||||
60000,
|
||||
);
|
||||
it.skipIf(skipIfNoApiKey)(
|
||||
"should extract data correctly (file paths and file contents) with an agent and delete that agent",
|
||||
async () => {
|
||||
const extractClient = new LlamaExtract(
|
||||
process.env.LLAMA_CLOUD_API_KEY!,
|
||||
"https://api.cloud.llamaindex.ai",
|
||||
);
|
||||
const agent = await extractClient.getAgent("ExtractTestAgent");
|
||||
const testContent =
|
||||
"**Text to extract**: Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.";
|
||||
const testFilePath = "test-extract-agent.md";
|
||||
|
||||
await fs.writeFile(testFilePath, new TextEncoder().encode(testContent));
|
||||
const result = await agent!.extract("test-extract-agent.md");
|
||||
expect("data" in result!).toBeTruthy();
|
||||
expect("extractionMetadata" in result!).toBeTruthy();
|
||||
|
||||
const buffer = await fs.readFile("test-extract-agent.md");
|
||||
const resultBuffer = await agent!.extract(
|
||||
undefined,
|
||||
buffer,
|
||||
"test-extract-agent.md",
|
||||
);
|
||||
expect("data" in resultBuffer!).toBeTruthy();
|
||||
expect("extractionMetadata" in resultBuffer!).toBeTruthy();
|
||||
|
||||
const success = await extractClient.deleteAgent(agent!.id);
|
||||
expect(success).toBeTruthy();
|
||||
},
|
||||
60000,
|
||||
);
|
||||
it.skipIf(skipIfNoApiKey)(
|
||||
"should extract statelessly file paths and file contents",
|
||||
async () => {
|
||||
const dataSchema = {
|
||||
properties: {
|
||||
text: {
|
||||
description: "Text from the file",
|
||||
type: "string",
|
||||
},
|
||||
},
|
||||
required: ["text"],
|
||||
type: "object",
|
||||
};
|
||||
|
||||
const testContent =
|
||||
"**Text to extract**: Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.";
|
||||
const testFilePath = "test-extract.md";
|
||||
|
||||
await fs.writeFile(testFilePath, new TextEncoder().encode(testContent));
|
||||
|
||||
const extractClient = new LlamaExtract(
|
||||
process.env.LLAMA_CLOUD_API_KEY!,
|
||||
"https://api.cloud.llamaindex.ai",
|
||||
);
|
||||
const result = await extractClient.extract(
|
||||
dataSchema,
|
||||
{} as ExtractConfig,
|
||||
"test-extract.md",
|
||||
);
|
||||
expect("data" in result!).toBeTruthy();
|
||||
expect("extractionMetadata" in result!).toBeTruthy();
|
||||
|
||||
const buffer = await fs.readFile("test-extract.md");
|
||||
const resultBuffer = await extractClient.extract(
|
||||
dataSchema,
|
||||
{} as ExtractConfig,
|
||||
undefined,
|
||||
buffer,
|
||||
); // testing without passing a file name
|
||||
expect("data" in resultBuffer!).toBeTruthy();
|
||||
expect("extractionMetadata" in resultBuffer!).toBeTruthy();
|
||||
},
|
||||
60000,
|
||||
);
|
||||
});
|
||||
|
||||
describe("Error Handling Integration", () => {
|
||||
it.skipIf(skipIfNoApiKey)(
|
||||
"should handle malformed files gracefully",
|
||||
|
||||
Reference in New Issue
Block a user