Compare commits

..

2 Commits

Author SHA1 Message Date
github-actions[bot] 8474ca970e Release 0.11.1 (#1961)
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
2025-05-20 22:18:57 -07:00
Alex Yang 3703f907d9 fix(parse): upload API (#1960) 2025-05-20 17:39:39 -07:00
35 changed files with 4558 additions and 1189 deletions
+8
View File
@@ -1,5 +1,13 @@
# @llamaindex/doc
## 0.2.20
### Patch Changes
- Updated dependencies [3703f90]
- @llamaindex/cloud@4.0.9
- llamaindex@0.11.1
## 0.2.19
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/doc",
"version": "0.2.19",
"version": "0.2.20",
"private": true,
"scripts": {
"postinstall": "fumadocs-mdx",
@@ -1,5 +1,11 @@
# @llamaindex/cloudflare-worker-agent-test
## 0.0.162
### Patch Changes
- llamaindex@0.11.1
## 0.0.161
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/cloudflare-worker-agent-test",
"version": "0.0.161",
"version": "0.0.162",
"type": "module",
"private": true,
"scripts": {
@@ -1,5 +1,12 @@
# @llamaindex/llama-parse-browser-test
## 0.0.64
### Patch Changes
- Updated dependencies [3703f90]
- @llamaindex/cloud@4.0.9
## 0.0.63
### Patch Changes
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/llama-parse-browser-test",
"private": true,
"version": "0.0.63",
"version": "0.0.64",
"type": "module",
"scripts": {
"dev": "vite",
+6
View File
@@ -1,5 +1,11 @@
# @llamaindex/next-agent-test
## 0.1.162
### Patch Changes
- llamaindex@0.11.1
## 0.1.161
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/next-agent-test",
"version": "0.1.161",
"version": "0.1.162",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,5 +1,11 @@
# test-edge-runtime
## 0.1.161
### Patch Changes
- llamaindex@0.11.1
## 0.1.160
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/nextjs-edge-runtime-test",
"version": "0.1.160",
"version": "0.1.161",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,5 +1,11 @@
# @llamaindex/next-node-runtime
## 0.1.29
### Patch Changes
- llamaindex@0.11.1
## 0.1.28
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/next-node-runtime-test",
"version": "0.1.28",
"version": "0.1.29",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,5 +1,11 @@
# vite-import-llamaindex
## 0.0.28
### Patch Changes
- llamaindex@0.11.1
## 0.0.27
### Patch Changes
@@ -1,7 +1,7 @@
{
"name": "vite-import-llamaindex",
"private": true,
"version": "0.0.27",
"version": "0.0.28",
"type": "module",
"scripts": {
"build": "vite build",
@@ -1,5 +1,11 @@
# @llamaindex/waku-query-engine-test
## 0.0.162
### Patch Changes
- llamaindex@0.11.1
## 0.0.161
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/waku-query-engine-test",
"version": "0.0.161",
"version": "0.0.162",
"type": "module",
"private": true,
"scripts": {
+6
View File
@@ -1,5 +1,11 @@
# @llamaindex/autotool
## 8.0.1
### Patch Changes
- llamaindex@0.11.1
## 8.0.0
### Patch Changes
@@ -1,5 +1,12 @@
# @llamaindex/autotool-01-node-example
## 0.0.109
### Patch Changes
- llamaindex@0.11.1
- @llamaindex/autotool@8.0.1
## 0.0.108
### Patch Changes
@@ -13,5 +13,5 @@
"scripts": {
"start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
},
"version": "0.0.108"
"version": "0.0.109"
}
+1 -1
View File
@@ -6,7 +6,7 @@
"url": "git+https://github.com/run-llama/LlamaIndexTS.git",
"directory": "packages/autotool"
},
"version": "8.0.0",
"version": "8.0.1",
"description": "auto transpile your JS function to LLM Agent compatible",
"files": [
"dist",
+6
View File
@@ -1,5 +1,11 @@
# @llamaindex/cloud
## 4.0.9
### Patch Changes
- 3703f90: feat(parse): add upload API
## 4.0.8
### Patch Changes
+6 -1
View File
@@ -12,8 +12,13 @@ export default defineConfig({
plugins: [
...defaultPlugins,
"@hey-api/client-fetch",
"zod",
"@hey-api/schemas",
"@hey-api/sdk",
{
name: "@hey-api/sdk",
enums: "javascript",
identifierCase: "PascalCase",
name: "@hey-api/typescript",
},
],
});
+2868 -953
View File
File diff suppressed because it is too large Load Diff
+18 -4
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/cloud",
"version": "4.0.8",
"version": "4.0.9",
"type": "module",
"license": "MIT",
"scripts": {
@@ -37,6 +37,17 @@
},
"default": "./reader/dist/index.js"
},
"./parse": {
"require": {
"types": "./parse/dist/index.d.cts",
"default": "./parse/dist/index.cjs"
},
"import": {
"types": "./parse/dist/index.d.ts",
"default": "./parse/dist/index.js"
},
"default": "./parse/dist/index.js"
},
".": {
"require": {
"types": "./reader/dist/index.d.cts",
@@ -55,16 +66,19 @@
"directory": "packages/cloud"
},
"devDependencies": {
"@hey-api/client-fetch": "^0.10.0",
"@hey-api/openapi-ts": "^0.66.7",
"@hey-api/client-fetch": "^0.10.1",
"@hey-api/openapi-ts": "^0.67.5",
"@llama-flow/core": "^0.4.1",
"@llamaindex/core": "workspace:*",
"@llamaindex/env": "workspace:*"
},
"peerDependencies": {
"@llama-flow/core": "^0.4.1",
"@llamaindex/core": "workspace:*",
"@llamaindex/env": "workspace:*"
},
"dependencies": {
"p-retry": "^6.2.1"
"p-retry": "^6.2.1",
"zod": "^3.25.7"
}
}
+8
View File
@@ -0,0 +1,8 @@
{
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
"types": "./dist/index.d.ts",
"exports": "./dist/index.js",
"private": true
}
+55
View File
@@ -0,0 +1,55 @@
import { workflowEvent } from "@llama-flow/core";
import { zodEvent } from "@llama-flow/core/util/zod";
import { z } from "zod";
import { parseFormSchema } from "./schema";
export const uploadEvent = zodEvent(
parseFormSchema.merge(
z.object({
file: z
.string()
.or(z.instanceof(File))
.or(z.instanceof(Blob))
.or(z.instanceof(Uint8Array))
.optional()
.describe("input"),
}),
),
{
debugLabel: "upload",
uniqueId: "52099967-34a8-419b-8950-c859eab60145",
},
);
export const checkStatusEvent = workflowEvent<string>({
debugLabel: "check-status",
uniqueId: "462157fc-1ded-4ba7-9bc4-3e870395bd20",
});
export const checkStatusSuccessEvent = workflowEvent<string>({
debugLabel: "check-status-success",
uniqueId: "360b7641-30f7-456e-851d-104bb5e3f8d2",
});
export const requestMarkdownEvent = workflowEvent<string>({
debugLabel: "markdown-request",
uniqueId: "aa4c2e3c-0f09-4035-aab6-c72719c877cc",
});
export const requestTextEvent = workflowEvent<string>({
debugLabel: "text-request",
uniqueId: "6976536e-5399-4285-9455-0b70f1dfc92b",
});
export const requestJsonEvent = workflowEvent<string>({
debugLabel: "json-request",
uniqueId: "9fc4a330-52ad-4aac-8092-a650998b7f6f",
});
export const markdownResultEvent = workflowEvent<string>({
debugLabel: "markdown-result",
uniqueId: "2dfb57c8-73d1-4394-bea8-f05246d934d4",
});
export const textResultEvent = workflowEvent<string>({
debugLabel: "text-result",
uniqueId: "a27deec6-b24f-4eda-a5ac-ba2fb2bf37c8",
});
export const jsonResultEvent = workflowEvent<unknown>({
debugLabel: "json-result",
uniqueId: "e086e6bd-a612-4f25-ab41-9b31dcb8af86",
});
+225
View File
@@ -0,0 +1,225 @@
import { createClient, createConfig } from "@hey-api/client-fetch";
import { createWorkflow, type InferWorkflowEventData } from "@llama-flow/core";
import { createStatefulMiddleware } from "@llama-flow/core/middleware/state";
import { withTraceEvents } from "@llama-flow/core/middleware/trace-events";
import { pRetryHandler } from "@llama-flow/core/util/p-retry";
import { fs, getEnv, path } from "@llamaindex/env";
import {
type BodyUploadFileApiV1ParsingUploadPost,
getJobApiV1ParsingJobJobIdGet,
getJobJsonResultApiV1ParsingJobJobIdResultJsonGet,
getJobResultApiV1ParsingJobJobIdResultMarkdownGet,
getJobTextResultApiV1ParsingJobJobIdResultTextGet,
type StatusEnum,
uploadFileApiV1ParsingUploadPost,
} from "./client";
import {
checkStatusEvent,
checkStatusSuccessEvent,
jsonResultEvent,
markdownResultEvent,
requestJsonEvent,
requestMarkdownEvent,
requestTextEvent,
textResultEvent,
uploadEvent,
} from "./events";
export type LlamaParseWorkflowParams = {
region?: "us" | "eu" | "us-staging";
apiKey?: string;
};
const URLS = {
us: "https://api.cloud.llamaindex.ai",
eu: "https://api.cloud.eu.llamaindex.ai",
"us-staging": "https://api.staging.llamaindex.ai",
} as const;
const { withState, getContext } = createStatefulMiddleware(
(params: LlamaParseWorkflowParams) => {
const apiKey = params.apiKey ?? getEnv("LLAMA_CLOUD_API_KEY");
const region = params.region ?? "us";
if (!apiKey) {
throw new Error("LLAMA_CLOUD_API_KEY is not set");
}
return {
cache: {} as Record<string, StatusEnum>,
client: createClient(
createConfig({
baseUrl: URLS[region],
headers: {
Authorization: `Bearer ${apiKey}`,
},
}),
),
};
},
);
const llamaParseWorkflow = withState(withTraceEvents(createWorkflow()));
llamaParseWorkflow.handle([uploadEvent], async ({ data: form }) => {
const { state } = getContext();
const finalForm = { ...form };
if ("file" in form) {
// support loads from the file system
const file = form?.file;
const isFilePath = typeof file === "string";
const data = isFilePath ? await fs.readFile(file) : file;
const filename: string | undefined = isFilePath
? path.basename(file)
: undefined;
finalForm.file = data
? globalThis.File && filename
? new File([data], filename)
: new Blob([data])
: undefined;
}
const {
data: { id, status },
} = await uploadFileApiV1ParsingUploadPost({
throwOnError: true,
body: {
...finalForm,
} as BodyUploadFileApiV1ParsingUploadPost,
client: state.client,
});
state.cache[id] = status;
return checkStatusEvent.with(id);
});
llamaParseWorkflow.handle(
[checkStatusEvent],
pRetryHandler(
async ({ data: uuid }) => {
const { state } = getContext();
if (state.cache[uuid] === "SUCCESS") {
return checkStatusSuccessEvent.with(uuid);
}
const {
data: { status },
} = await getJobApiV1ParsingJobJobIdGet({
throwOnError: true,
path: {
job_id: uuid,
},
client: state.client,
});
state.cache[uuid] = status;
if (status === "SUCCESS") {
return checkStatusSuccessEvent.with(uuid);
}
throw new Error(`LLamaParse status: ${status}`);
},
{
retries: 100,
},
),
);
//#region sub workflow
llamaParseWorkflow.handle([requestMarkdownEvent], async ({ data: job_id }) => {
const { state } = getContext();
const { data } = await getJobResultApiV1ParsingJobJobIdResultMarkdownGet({
throwOnError: true,
path: {
job_id,
},
client: state.client,
});
return markdownResultEvent.with(data.markdown);
});
llamaParseWorkflow.handle([requestTextEvent], async ({ data: job_id }) => {
const { state } = getContext();
const { data } = await getJobTextResultApiV1ParsingJobJobIdResultTextGet({
throwOnError: true,
path: {
job_id,
},
client: state.client,
});
return textResultEvent.with(data.text);
});
llamaParseWorkflow.handle([requestJsonEvent], async ({ data: job_id }) => {
const { state } = getContext();
const { data } = await getJobJsonResultApiV1ParsingJobJobIdResultJsonGet({
throwOnError: true,
path: {
job_id,
},
client: state.client,
});
return jsonResultEvent.with(data.pages);
});
//#endregion
export type ParseJob = {
get jobId(): string;
get signal(): AbortSignal;
get context(): ReturnType<typeof llamaParseWorkflow.createContext>;
get form(): InferWorkflowEventData<typeof uploadEvent>;
markdown(): Promise<string>;
text(): Promise<string>;
//eslint-disable-next-line @typescript-eslint/no-explicit-any
json(): Promise<any[]>;
};
export const upload = async (
params: InferWorkflowEventData<typeof uploadEvent> & LlamaParseWorkflowParams,
): Promise<ParseJob> => {
//#region upload event
const context = llamaParseWorkflow.createContext(params);
const { stream, sendEvent } = context;
const ev = uploadEvent.with(params);
sendEvent(ev);
const uploadThread = await llamaParseWorkflow
.substream(ev, stream)
.until((ev) => checkStatusSuccessEvent.include(ev))
.toArray();
//#region
const jobId: string = uploadThread.at(-1)!.data;
return {
get signal() {
// lazy load
return context.signal;
},
get jobId() {
return jobId;
},
get form() {
return ev.data;
},
get context() {
return context;
},
async markdown(): Promise<string> {
const requestEv = requestMarkdownEvent.with(jobId);
const { sendEvent, stream } = llamaParseWorkflow.createContext(params);
sendEvent(requestEv);
const markdownThread = await stream.until(markdownResultEvent).toArray();
return markdownThread.at(-1)!.data;
},
async text(): Promise<string> {
const requestEv = requestTextEvent.with(jobId);
const { sendEvent, stream } = llamaParseWorkflow.createContext(params);
sendEvent(requestEv);
const textThread = await stream.until(textResultEvent).toArray();
return textThread.at(-1)!.data;
},
//eslint-disable-next-line @typescript-eslint/no-explicit-any
async json(): Promise<any[]> {
const requestEv = requestJsonEvent.with(jobId);
const { sendEvent, stream } = llamaParseWorkflow.createContext(params);
sendEvent(requestEv);
const jsonThread = await stream
.until((ev) => jsonResultEvent.include(ev))
.toArray();
return jsonThread.at(-1)!.data;
},
};
};
+131
View File
@@ -0,0 +1,131 @@
import { FailPageMode, ParserLanguages, ParsingMode } from "./client";
import { z } from "zod";
type Language = ParserLanguages;
const VALUES: [Language, ...Language[]] = [
ParserLanguages.EN,
...Object.values(ParserLanguages),
];
const languageSchema = z.enum(VALUES);
const PARSE_PRESETS = [
"fast",
"balanced",
"premium",
"structured",
"auto",
"scientific",
"invoice",
"slides",
"_carlyle",
] as const;
export const parsePresetSchema = z.enum(PARSE_PRESETS);
export const parseFormSchema = z.object({
adaptive_long_table: z.boolean().optional(),
annotate_links: z.boolean().optional(),
auto_mode: z.boolean().optional(),
auto_mode_trigger_on_image_in_page: z.boolean().optional(),
auto_mode_trigger_on_table_in_page: z.boolean().optional(),
auto_mode_trigger_on_text_in_page: z.string().optional(),
auto_mode_trigger_on_regexp_in_page: z.string().optional(),
auto_mode_configuration_json: z.string().optional(),
azure_openai_api_version: z.string().optional(),
azure_openai_deployment_name: z.string().optional(),
azure_openai_endpoint: z.string().optional(),
azure_openai_key: z.string().optional(),
bbox_bottom: z.number().min(0).max(1).optional(),
bbox_left: z.number().min(0).max(1).optional(),
bbox_right: z.number().min(0).max(1).optional(),
bbox_top: z.number().min(0).max(1).optional(),
disable_ocr: z.boolean().optional(),
disable_reconstruction: z.boolean().optional(),
disable_image_extraction: z.boolean().optional(),
do_not_cache: z.coerce.boolean().optional(),
do_not_unroll_columns: z.coerce.boolean().optional(),
extract_charts: z.boolean().optional(),
guess_xlsx_sheet_name: z.boolean().optional(),
html_make_all_elements_visible: z.boolean().optional(),
html_remove_fixed_elements: z.boolean().optional(),
html_remove_navigation_elements: z.boolean().optional(),
http_proxy: z
.string()
.url(
'Set a valid URL for the HTTP proxy, e.g., "http://proxy.example.com:8080"',
)
.refine(
(url) => {
try {
const parsedUrl = new URL(url);
return (
parsedUrl.protocol === "http:" || parsedUrl.protocol === "https:"
);
} catch {
return false;
}
},
{
message: "Invalid HTTP proxy URL",
},
)
.optional(),
input_s3_path: z.string().optional(),
input_s3_region: z.string().optional(),
input_url: z.string().optional(),
invalidate_cache: z.boolean().optional(),
language: z.array(languageSchema).optional(),
extract_layout: z.boolean().optional(),
max_pages: z.number().nullable().optional(),
output_pdf_of_document: z.boolean().optional(),
output_s3_path_prefix: z.string().optional(),
output_s3_region: z.string().optional(),
page_prefix: z.string().optional(),
page_separator: z.string().optional(),
page_suffix: z.string().optional(),
preserve_layout_alignment_across_pages: z.boolean().optional(),
skip_diagonal_text: z.boolean().optional(),
spreadsheet_extract_sub_tables: z.boolean().optional(),
structured_output: z.boolean().optional(),
structured_output_json_schema: z.string().optional(),
structured_output_json_schema_name: z.string().optional(),
take_screenshot: z.boolean().optional(),
target_pages: z.string().optional(),
vendor_multimodal_api_key: z.string().optional(),
vendor_multimodal_model_name: z.string().optional(),
model: z.string().optional(),
webhook_url: z.string().url().optional(),
parse_mode: z.nativeEnum(ParsingMode).nullable().optional(),
system_prompt: z.string().optional(),
system_prompt_append: z.string().optional(),
user_prompt: z.string().optional(),
job_timeout_in_seconds: z.number().optional(),
job_timeout_extra_time_per_page_in_seconds: z.number().optional(),
strict_mode_image_extraction: z.boolean().optional(),
strict_mode_image_ocr: z.boolean().optional(),
strict_mode_reconstruction: z.boolean().optional(),
strict_mode_buggy_font: z.boolean().optional(),
save_images: z.boolean().optional(),
ignore_document_elements_for_layout_detection: z.boolean().optional(),
output_tables_as_HTML: z.boolean().optional(),
use_vendor_multimodal_model: z.boolean().optional(),
bounding_box: z.string().optional(),
gpt4o_mode: z.boolean().optional(),
gpt4o_api_key: z.string().optional(),
complemental_formatting_instruction: z.string().optional(),
content_guideline_instruction: z.string().optional(),
premium_mode: z.boolean().optional(),
is_formatting_instruction: z.boolean().optional(),
continuous_mode: z.boolean().optional(),
parsing_instruction: z.string().optional(),
fast_mode: z.boolean().optional(),
formatting_instruction: z.string().optional(),
preset: parsePresetSchema.optional(),
compact_markdown_table: z.boolean().optional(),
markdown_table_multiline_header_separator: z.string().optional(),
page_error_tolerance: z.number().min(0).max(1).optional(),
replace_failed_page_mode: z.nativeEnum(FailPageMode).nullable().optional(),
replace_failed_page_with_error_message_prefix: z.string().optional(),
replace_failed_page_with_error_message_suffix: z.string().optional(),
});
+6
View File
@@ -1,5 +1,11 @@
# @llamaindex/experimental
## 0.0.178
### Patch Changes
- llamaindex@0.11.1
## 0.0.177
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/experimental",
"description": "Experimental package for LlamaIndexTS",
"version": "0.0.177",
"version": "0.0.178",
"type": "module",
"types": "dist/type/index.d.ts",
"main": "dist/cjs/index.js",
+7
View File
@@ -1,5 +1,12 @@
# llamaindex
## 0.11.1
### Patch Changes
- Updated dependencies [3703f90]
- @llamaindex/cloud@4.0.9
## 0.11.0
### Minor Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "llamaindex",
"version": "0.11.0",
"version": "0.11.1",
"license": "MIT",
"type": "module",
"keywords": [
+1143 -218
View File
File diff suppressed because it is too large Load Diff
+8
View File
@@ -1,5 +1,13 @@
# @llamaindex/unit-test
## 0.1.29
### Patch Changes
- Updated dependencies [3703f90]
- @llamaindex/cloud@4.0.9
- llamaindex@0.11.1
## 0.1.28
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/unit-test",
"private": true,
"version": "0.1.28",
"version": "0.1.29",
"type": "module",
"scripts": {
"test": "vitest run"