Compare commits

...

4 Commits

Author SHA1 Message Date
Clelia (Astra) Bertelli e7df0bfc46 chore: implement claude suggestions 2025-10-21 12:12:48 +02:00
Clelia (Astra) Bertelli dd84466f3d chore: camelCase for everyone; refactor: slimmer logic for fileContents/filePaths handling 2025-10-20 17:38:52 +02:00
Clelia (Astra) Bertelli 29cee6b6bd ci: changesets 2025-10-20 16:25:00 +02:00
Clelia (Astra) Bertelli 703080f677 feat: add classify to ts sdk 2025-10-20 16:02:09 +02:00
11 changed files with 578 additions and 120 deletions
+5
View File
@@ -0,0 +1,5 @@
---
"llama-cloud-services": patch
---
Adding LlamaClassify among the available LlamaCloud services
@@ -0,0 +1,8 @@
{
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
"types": "./dist/index.d.ts",
"exports": "./dist/index.js",
"private": true
}
+13 -1
View File
@@ -24,7 +24,8 @@
"./reader",
"./parse",
"./beta/agent",
"./extract"
"./extract",
"./classify"
],
"exports": {
"./openapi.json": "./openapi.json",
@@ -83,6 +84,17 @@
},
"default": "./extract/dist/index.js"
},
"./classify": {
"require": {
"types": "./classify/dist/index.d.cts",
"default": "./classify/dist/index.cjs"
},
"import": {
"types": "./classify/dist/index.d.ts",
"default": "./classify/dist/index.js"
},
"default": "./classify/dist/index.js"
},
".": {
"require": {
"types": "./dist/index.d.cts",
@@ -0,0 +1,69 @@
import { createClient, createConfig, type Client } from "@hey-api/client-fetch";
import {
classify,
type ClassifyParsingConfiguration,
type ClassifierRule,
type ClassifyJobResults,
} from "./classify";
import { getUrl } from "./utils";
import { getEnv } from "@llamaindex/env";
import { File } from "buffer";
export class LlamaClassify {
private client: Client;
constructor(
apiKey: string | undefined = undefined,
baseUrl: string | undefined = undefined,
region: string | undefined = undefined,
) {
const key = apiKey ?? getEnv("LLAMA_CLOUD_API_KEY");
if (typeof key === "undefined") {
throw new Error(
"No API key provided and no API key found in environment. Please pass the API key or set `LLAMA_CLOUD_API_KEY` as an environment variable.",
);
}
const url = getUrl(baseUrl, region);
this.client = createClient(
createConfig({
baseUrl: url,
headers: {
Authorization: `Bearer ${key}`,
},
}),
);
}
async classify(
rules: ClassifierRule[],
parsingConfiguration: ClassifyParsingConfiguration,
fileContents:
| Buffer<ArrayBufferLike>[]
| File[]
| Uint8Array<ArrayBuffer>[]
| string[]
| undefined = undefined,
filePaths: string[] | undefined = undefined,
projectId: string | null = null,
organizationId: string | null = null,
pollingInterval: number = 1,
maxPollingIterations: number = 1800,
maxRetriesOnError: number = 10,
retryInterval: number = 0.5,
): Promise<ClassifyJobResults> {
const result = await classify(
rules,
parsingConfiguration,
fileContents,
filePaths,
projectId,
organizationId,
this.client,
pollingInterval,
maxPollingIterations,
maxRetriesOnError,
retryInterval,
);
return result;
}
}
+1 -19
View File
@@ -4,25 +4,7 @@ import * as extract from "./extract";
import type { ExtractAgent, ExtractConfig } from "./extract";
import { getEnv } from "@llamaindex/env";
import type { ExtractResult } from "./type";
const URLS = {
us: "https://api.cloud.llamaindex.ai",
eu: "https://api.cloud.eu.llamaindex.ai",
"us-staging": "https://api.staging.llamaindex.ai",
} as const;
function getUrl(baseUrl: string | undefined, region: string | undefined) {
if (typeof baseUrl != "undefined") {
return baseUrl;
}
if (typeof region === "undefined") {
return URLS["us"];
} else if (region === "us" || region === "eu" || region === "us-staging") {
return URLS[region];
} else {
throw new Error(`Unsupported region: ${region}`);
}
}
import { getUrl } from "./utils";
export class LlamaExtractAgent {
private agent: ExtractAgent;
+289
View File
@@ -0,0 +1,289 @@
import type {
Options,
CreateClassifyJobApiV1ClassifierJobsPostData,
ClassifyJobCreate,
ClassifierRule,
ClassifyParsingConfiguration,
GetClassifyJobApiV1ClassifierJobsClassifyJobIdGetData,
GetClassificationJobResultsApiV1ClassifierJobsClassifyJobIdResultsGetData,
ClassifyJobResults,
} from "./api";
import {
StatusEnum,
createClassifyJobApiV1ClassifierJobsPost,
getClassifyJobApiV1ClassifierJobsClassifyJobIdGet,
getClassificationJobResultsApiV1ClassifierJobsClassifyJobIdResultsGet,
} from "./api";
import type { Client } from "@hey-api/client-fetch";
import { sleep } from "./utils";
import { uploadFile } from "./fileUpload";
import { File } from "buffer";
async function createClassifyJob(
fileIds: string[],
rules: ClassifierRule[],
parsingConfiguration: ClassifyParsingConfiguration,
organizationId: null | string,
projectId: null | string,
client: Client | undefined,
maxRetriesOnError: number = 10,
retryInterval: number = 0.5,
): Promise<string> {
const rawData = {
file_ids: fileIds,
rules: rules,
parsing_configuration: parsingConfiguration,
} as ClassifyJobCreate;
const data = {
body: rawData,
query: {
project_id: projectId,
organization_id: organizationId,
},
} as CreateClassifyJobApiV1ClassifierJobsPostData;
const options = data as Options<CreateClassifyJobApiV1ClassifierJobsPostData>;
if (typeof client != "undefined") {
options.client = client;
}
let retries = 0;
while (true) {
if (retries > maxRetriesOnError) {
throw new Error(
"Error while creating the classify job: Exceeded maximum number of retries, the API keeps returning errors.",
);
}
const response = await createClassifyJobApiV1ClassifierJobsPost(options);
if (!response.response.ok) {
if ("error" in response) {
console.log(
`An error occurred while creating the classification job.\nDetails:\n\n${JSON.stringify(
response.error,
)}\n\nRetrying...`,
);
}
retries++;
await sleep(retryInterval * 1000);
} else {
if (typeof response.data != "undefined") {
return response.data.id;
} else {
throw new Error(
"Error while creating the classify job: the job creation succeeded but no data where returned",
);
}
}
}
}
async function pollForJobCompletion(
jobId: string,
interval: number = 1,
maxIterations: number = 1800,
client: Client | undefined = undefined,
): Promise<boolean> {
let status: StatusEnum | undefined = undefined;
const jobData = {
path: { classify_job_id: jobId },
} as GetClassifyJobApiV1ClassifierJobsClassifyJobIdGetData;
const jobOptions =
jobData as Options<GetClassifyJobApiV1ClassifierJobsClassifyJobIdGetData>;
if (typeof client != "undefined") {
jobOptions.client = client;
}
let numIterations: number = 0;
while (true) {
if (numIterations > maxIterations) {
return false;
}
const response =
await getClassifyJobApiV1ClassifierJobsClassifyJobIdGet(jobOptions);
if (!response.response.ok) {
numIterations++;
}
if (typeof response.data != "undefined") {
status = response.data.status as StatusEnum;
if (status == StatusEnum.CANCELLED || status == StatusEnum.ERROR) {
throw new Error("There was an error during the classification job.");
} else if (status == StatusEnum.SUCCESS) {
return true;
} else {
numIterations++;
await sleep(interval * 1000);
}
}
}
}
async function getJobResult(
jobId: string,
client: Client | undefined = undefined,
projectId: string | null = null,
organizationId: string | null = null,
maxRetriesOnError: number = 10,
retryInterval: number = 0.5,
): Promise<ClassifyJobResults> {
const jobData = {
path: { classify_job_id: jobId },
query: { organization_id: organizationId, project_id: projectId },
} as GetClassificationJobResultsApiV1ClassifierJobsClassifyJobIdResultsGetData;
const jobOptions =
jobData as Options<GetClassificationJobResultsApiV1ClassifierJobsClassifyJobIdResultsGetData>;
if (typeof client != "undefined") {
jobOptions.client = client;
}
let retries: number = 0;
while (true) {
if (retries > maxRetriesOnError) {
throw new Error(
"Error while getting the result of the classification job: Exceeded maximum number of retries, the API keeps returning errors.",
);
}
const response =
await getClassificationJobResultsApiV1ClassifierJobsClassifyJobIdResultsGet(
jobOptions,
);
if (!response.response.ok) {
if ("error" in response) {
console.log(
"An error occurred: ",
JSON.stringify(response.error),
"\nRetrying...",
);
}
retries++;
await sleep(retryInterval * 1000);
}
if (typeof response.data != "undefined") {
return response.data as ClassifyJobResults;
} else {
throw new Error(
"Error while retrieving results for the classify job: the result was successfully obtained but no data were returned",
);
}
}
}
export async function classify(
rules: ClassifierRule[],
parsingConfiguration: ClassifyParsingConfiguration,
fileContents:
| Buffer<ArrayBufferLike>[]
| File[]
| Uint8Array<ArrayBuffer>[]
| string[]
| undefined = undefined,
filePaths: string[] | undefined = undefined,
projectId: string | null = null,
organizationId: string | null = null,
client: Client | undefined = undefined,
pollingInterval: number = 1,
maxPollingIterations: number = 1800,
maxRetriesOnError: number = 10,
retryInterval: number = 0.5,
): Promise<ClassifyJobResults> {
const fileIds: string[] = [];
if (!filePaths && !fileContents) {
throw new Error(
"One between filePath and fileContent needs to be provided",
);
}
if (filePaths) {
const uploadPromises = filePaths.map(async (name) => {
try {
const fileId = await uploadFile(
name,
undefined,
undefined,
projectId,
organizationId,
client,
maxRetriesOnError,
retryInterval,
);
if (fileId) {
return fileId;
} else {
console.error(`Unable to upload ${name}, skipping...`);
return null;
}
} catch (error) {
console.error(`Error uploading ${name}:`, error);
return null;
}
});
const results = await Promise.all(uploadPromises);
fileIds.push(...results.filter((id) => id !== null));
}
if (fileContents) {
const uploadPromises = fileContents.map(async (content) => {
try {
const fileId = await uploadFile(
undefined,
content,
undefined,
projectId,
organizationId,
client,
maxRetriesOnError,
retryInterval,
);
if (fileId) {
return fileId;
} else {
console.error(`Unable to upload file (content), skipping...`);
return null;
}
} catch (error) {
console.error(`Error uploading file (content):`, error);
return null;
}
});
const results = await Promise.all(uploadPromises);
fileIds.push(...results.filter((id) => id !== null));
}
if (fileIds.length == 0) {
throw new Error(
"None of the provided files was successfully uploaded, it is not possible to create a classification job.",
);
}
const jobId = await createClassifyJob(
fileIds,
rules,
parsingConfiguration,
organizationId,
projectId,
client,
maxRetriesOnError,
retryInterval,
);
const success = await pollForJobCompletion(
jobId,
pollingInterval,
maxPollingIterations,
client,
);
if (!success) {
throw new Error("Your job is taking longer than 10 minutes, timing out...");
} else {
return (await getJobResult(
jobId,
client,
projectId,
organizationId,
maxRetriesOnError,
retryInterval,
)) as ClassifyJobResults;
}
}
export {
type ClassifierRule,
type ClassifyJobResults,
type ClassifyParsingConfiguration,
};
+1 -100
View File
@@ -1,9 +1,5 @@
import { emitWarning } from "process";
import fs from "fs/promises";
import { Blob } from "buffer";
import * as path from "path";
import type { ExtractResult } from "./type";
import { randomUUID } from "@llamaindex/env";
import { File } from "buffer";
import {
type Options,
@@ -19,7 +15,6 @@ import {
type GetJobApiV1ExtractionJobsJobIdGetData,
type GetJobResultApiV1ExtractionJobsJobIdResultGetData,
StatusEnum,
type UploadFileApiV1FilesPostData,
type StatelessExtractionRequest,
type ExtractStatelessApiV1ExtractionRunPostData,
type DeleteExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdDeleteData,
@@ -29,17 +24,12 @@ import {
runJobApiV1ExtractionJobsPost,
getJobApiV1ExtractionJobsJobIdGet,
getJobResultApiV1ExtractionJobsJobIdResultGet,
uploadFileApiV1FilesPost,
extractStatelessApiV1ExtractionRunPost,
deleteExtractionAgentApiV1ExtractionExtractionAgentsExtractionAgentIdDelete,
} from "./api";
import type { Client } from "@hey-api/client-fetch";
import { sleep } from "./utils";
import { fileTypeFromBuffer } from "file-type";
type BodyUploadFileApiV1FilesPost = {
upload_file: Blob | File;
};
import { uploadFile } from "./fileUpload";
export async function createAgent(
name: string,
@@ -221,95 +211,6 @@ export async function getAgent(
}
}
function textToFile(text: string, fileName: string | null = null) {
return new File(
[text],
fileName ?? "uploadedFile_" + randomUUID().replaceAll("-", "_") + ".txt",
);
}
async function uploadFile(
filePath: string | undefined = undefined,
fileContent:
| Buffer<ArrayBufferLike>
| File
| Uint8Array<ArrayBuffer>
| string
| undefined = undefined,
fileName: string | undefined = undefined,
project_id: string | null = null,
organization_id: string | null = null,
client: Client | undefined = undefined,
maxRetriesOnError: number = 10,
retryInterval: number = 0.5,
): Promise<string | undefined> {
let file: File | undefined = undefined;
if (typeof filePath === "undefined" && typeof fileContent === "undefined") {
throw new Error(
"One between filePath and fileContent needs to be provided",
);
} else if (typeof filePath != "undefined") {
const buffer = await fs.readFile(filePath);
const actualFileName = fileName ?? path.basename(filePath);
const uint8Array = new Uint8Array(buffer);
file = new File([uint8Array], actualFileName);
} else if (typeof fileContent != "undefined") {
if (fileContent instanceof File) {
file = fileContent;
} else if (fileContent instanceof Buffer) {
const fileType = await fileTypeFromBuffer(fileContent);
const ext = fileType?.ext ?? "pdf";
const uint8Array = new Uint8Array(fileContent);
file = new File(
[uint8Array],
fileName ??
"uploadedFile_" + randomUUID().replaceAll("-", "_") + "." + ext,
);
} else if (fileContent instanceof Uint8Array) {
const fileType = await fileTypeFromBuffer(fileContent);
const ext = fileType?.ext ?? "pdf";
file = new File(
[fileContent],
fileName ??
"uploadedFile_" + randomUUID().replaceAll("-", "_") + "." + ext,
);
} else if (typeof fileContent === "string") {
file = textToFile(fileContent, fileName);
} else {
throw new Error("Unsupported fileContent type");
}
}
const fileToUpload = {
upload_file: file,
} as BodyUploadFileApiV1FilesPost;
const uploadData = {
body: fileToUpload,
query: { organization_id: organization_id, project_id: project_id },
} as UploadFileApiV1FilesPostData;
const uploadOptions = uploadData as Options<UploadFileApiV1FilesPostData>;
if (typeof client != "undefined") {
uploadOptions.client = client;
}
let retries: number = 0;
while (true) {
if (retries > maxRetriesOnError) {
throw new Error(
"Error while processing your file: Exceeded maximum number of retries, the API keeps returning errors.",
);
}
const uploadResponse = await uploadFileApiV1FilesPost(uploadOptions);
let fileId: string | undefined = undefined;
if (!uploadResponse.response.ok) {
retries++;
await sleep(retryInterval * 1000);
}
if (typeof uploadResponse.data != "undefined") {
fileId = uploadResponse.data.id as string;
return fileId;
}
}
}
async function createExtractJob(
options:
| Options<RunJobApiV1ExtractionJobsPostData>
+109
View File
@@ -0,0 +1,109 @@
import fs from "fs/promises";
import { Blob } from "buffer";
import * as path from "path";
import { randomUUID } from "@llamaindex/env";
import { File } from "buffer";
import {
type Options,
type UploadFileApiV1FilesPostData,
uploadFileApiV1FilesPost,
} from "./api";
import type { Client } from "@hey-api/client-fetch";
import { sleep } from "./utils";
import { fileTypeFromBuffer } from "file-type";
type BodyUploadFileApiV1FilesPost = {
upload_file: Blob | File;
};
function textToFile(text: string, fileName: string | null = null) {
return new File(
[text],
fileName ?? "uploadedFile_" + randomUUID().replaceAll("-", "_") + ".txt",
);
}
export async function uploadFile(
filePath: string | undefined = undefined,
fileContent:
| Buffer<ArrayBufferLike>
| File
| Uint8Array<ArrayBuffer>
| string
| undefined = undefined,
fileName: string | undefined = undefined,
project_id: string | null = null,
organization_id: string | null = null,
client: Client | undefined = undefined,
maxRetriesOnError: number = 10,
retryInterval: number = 0.5,
): Promise<string | undefined> {
let file: File | undefined = undefined;
if (typeof filePath === "undefined" && typeof fileContent === "undefined") {
throw new Error(
"One between filePath and fileContent needs to be provided",
);
} else if (typeof filePath != "undefined") {
const buffer = await fs.readFile(filePath);
const actualFileName = fileName ?? path.basename(filePath);
const uint8Array = new Uint8Array(buffer);
file = new File([uint8Array], actualFileName);
} else if (typeof fileContent != "undefined") {
if (fileContent instanceof File) {
file = fileContent;
} else if (fileContent instanceof Buffer) {
const fileType = await fileTypeFromBuffer(fileContent);
const ext = fileType?.ext ?? "pdf";
const uint8Array = new Uint8Array(fileContent);
file = new File(
[uint8Array],
fileName ??
"uploadedFile_" + randomUUID().replaceAll("-", "_") + "." + ext,
);
} else if (fileContent instanceof Uint8Array) {
const fileType = await fileTypeFromBuffer(fileContent);
const ext = fileType?.ext ?? "pdf";
file = new File(
[fileContent],
fileName ??
"uploadedFile_" + randomUUID().replaceAll("-", "_") + "." + ext,
);
} else if (typeof fileContent === "string") {
file = textToFile(fileContent, fileName);
} else {
throw new Error("Unsupported fileContent type");
}
}
const fileToUpload = {
upload_file: file,
} as BodyUploadFileApiV1FilesPost;
const uploadData = {
body: fileToUpload,
query: { organization_id: organization_id, project_id: project_id },
} as UploadFileApiV1FilesPostData;
const uploadOptions = uploadData as Options<UploadFileApiV1FilesPostData>;
if (typeof client != "undefined") {
uploadOptions.client = client;
}
let retries: number = 0;
while (true) {
if (retries > maxRetriesOnError) {
throw new Error(
"Error while processing your file: Exceeded maximum number of retries, the API keeps returning errors.",
);
}
const uploadResponse = await uploadFileApiV1FilesPost(uploadOptions);
let fileId: string | undefined = undefined;
if (!uploadResponse.response.ok) {
retries++;
await sleep(retryInterval * 1000);
}
if (
uploadResponse.response.ok &&
typeof uploadResponse.data != "undefined"
) {
fileId = uploadResponse.data.id as string;
return fileId;
}
}
}
+6
View File
@@ -8,3 +8,9 @@ export type { CloudConstructorParams } from "./type.js";
export { LlamaParseReader } from "./reader.js";
export { LlamaExtract, LlamaExtractAgent } from "./LlamaExtract.js";
export type { ExtractConfig } from "./extract.js";
export { LlamaClassify } from "./LlamaClassify.js";
export type {
ClassifierRule,
ClassifyJobResults,
ClassifyParsingConfiguration,
} from "./classify.js";
+22
View File
@@ -117,3 +117,25 @@ export function getSavePath(downloadPath: string, i: number): string {
return savePath;
}
const URLS = {
us: "https://api.cloud.llamaindex.ai",
eu: "https://api.cloud.eu.llamaindex.ai",
"us-staging": "https://api.staging.llamaindex.ai",
} as const;
export function getUrl(
baseUrl: string | undefined,
region: string | undefined,
) {
if (typeof baseUrl != "undefined") {
return baseUrl;
}
if (typeof region === "undefined") {
return URLS["us"];
} else if (region === "us" || region === "eu" || region === "us-staging") {
return URLS[region];
} else {
throw new Error(`Unsupported region: ${region}`);
}
}
@@ -2,6 +2,8 @@ import { describe, it, expect, beforeEach, beforeAll } from "vitest";
import { LlamaParseReader } from "../src/reader.js";
import { LlamaCloudIndex } from "../src/LlamaCloudIndex.js";
import { LlamaExtract, LlamaExtractAgent } from "../src/LlamaExtract.js";
import { LlamaClassify } from "../src/LlamaClassify.js";
import { ClassifierRule, ClassifyParsingConfiguration } from "../src/classify.js";
import { Document } from "@llamaindex/core/schema";
import { fs } from "@llamaindex/env";
import { ExtractConfig } from "../src/api.js";
@@ -489,6 +491,59 @@ describe("Integration Tests", () => {
);
});
describe("LlamaClassify Integration", () => {
it.skipIf(skipIfNoApiKey)(
"should classify data correctly (file paths and file contents) ",
async () => {
const classifyClient = new LlamaClassify(
process.env.LLAMA_CLOUD_API_KEY!,
"https://api.cloud.llamaindex.ai",
);
const testContent =
`A Fox one day spied a beautiful bunch of ripe grapes hanging from a vine trained along the branches of a tree. The grapes seemed ready to burst with juice, and the Fox's mouth watered as he gazed longingly at them. The bunch hung from a high branch, and the Fox had to jump for it. The first time he jumped he missed it by a long way. So he walked off a short distance and took a running leap at it, only to fall short once more. Again and again he tried, but in vain. Now he sat down and looked at the grapes in disgust. "What a fool I am," he said. "Here I am wearing myself out to get a bunch of sour grapes that are not worth gaping for." And off he walked very, very scornfully.There are many who pretend to despise and belittle that which is beyond their reach.`;
const testFilePath = "the_fox_and_the_grapes.md";
await fs.writeFile(testFilePath, new TextEncoder().encode(testContent));
const rules: ClassifierRule[] = [
{type: "fable", description: "A short story featuring animals whose aim is to teach the reader a lesson (the moral of the story)"},
{type: "fairy_tale", description: "A mid-to-long story featuring humans, magic creatures and other characters, whose main aim is to entertain the readers."}
]
const parsingConfig: ClassifyParsingConfiguration = {lang: "en"}
const result = await classifyClient.classify(
rules,
parsingConfig,
undefined,
["the_fox_and_the_grapes.md"]
);
expect("items" in result).toBeTruthy();
expect(result.items.length).toBeGreaterThan(0);
expect("result" in result.items[0]).toBeTruthy();
expect(result.items[0].result!.type === "fable").toBeTruthy();
const buffer = await fs.readFile("the_fox_and_the_grapes.md");
const resultBuffer = await classifyClient.classify(
rules,
parsingConfig,
[buffer],
);
expect("items" in resultBuffer).toBeTruthy();
expect(resultBuffer.items.length).toBeGreaterThan(0);
expect("result" in resultBuffer.items[0]).toBeTruthy();
expect(resultBuffer.items[0].result!.type === "fable").toBeTruthy();
try {
await fs.unlink("the_fox_and_the_grapes.md")
} catch(err) {
console.log(`Unable to delete file the_fox_and_the_grapes.md because of ${err}`)
}
},
60000,
);
});
describe("LlamaExtract Integration", () => {
it.skipIf(skipIfNoApiKey)(
"should create agents correctly",