Compare commits

..

2 Commits

Author SHA1 Message Date
github-actions[bot] 8474ca970e Release 0.11.1 (#1961)
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
2025-05-20 22:18:57 -07:00
Alex Yang 3703f907d9 fix(parse): upload API (#1960) 2025-05-20 17:39:39 -07:00
39 changed files with 4555 additions and 1903 deletions
+8
View File
@@ -1,5 +1,13 @@
# @llamaindex/doc
## 0.2.20
### Patch Changes
- Updated dependencies [3703f90]
- @llamaindex/cloud@4.0.9
- llamaindex@0.11.1
## 0.2.19
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/doc",
"version": "0.2.19",
"version": "0.2.20",
"private": true,
"scripts": {
"postinstall": "fumadocs-mdx",
@@ -1,5 +1,11 @@
# @llamaindex/cloudflare-worker-agent-test
## 0.0.162
### Patch Changes
- llamaindex@0.11.1
## 0.0.161
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/cloudflare-worker-agent-test",
"version": "0.0.161",
"version": "0.0.162",
"type": "module",
"private": true,
"scripts": {
@@ -1,5 +1,12 @@
# @llamaindex/llama-parse-browser-test
## 0.0.64
### Patch Changes
- Updated dependencies [3703f90]
- @llamaindex/cloud@4.0.9
## 0.0.63
### Patch Changes
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/llama-parse-browser-test",
"private": true,
"version": "0.0.63",
"version": "0.0.64",
"type": "module",
"scripts": {
"dev": "vite",
+6
View File
@@ -1,5 +1,11 @@
# @llamaindex/next-agent-test
## 0.1.162
### Patch Changes
- llamaindex@0.11.1
## 0.1.161
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/next-agent-test",
"version": "0.1.161",
"version": "0.1.162",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,5 +1,11 @@
# test-edge-runtime
## 0.1.161
### Patch Changes
- llamaindex@0.11.1
## 0.1.160
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/nextjs-edge-runtime-test",
"version": "0.1.160",
"version": "0.1.161",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,5 +1,11 @@
# @llamaindex/next-node-runtime
## 0.1.29
### Patch Changes
- llamaindex@0.11.1
## 0.1.28
### Patch Changes
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/next-node-runtime-test",
"version": "0.1.28",
"version": "0.1.29",
"private": true,
"scripts": {
"dev": "next dev",
@@ -1,5 +1,11 @@
# vite-import-llamaindex
## 0.0.28
### Patch Changes
- llamaindex@0.11.1
## 0.0.27
### Patch Changes
@@ -1,7 +1,7 @@
{
"name": "vite-import-llamaindex",
"private": true,
"version": "0.0.27",
"version": "0.0.28",
"type": "module",
"scripts": {
"build": "vite build",
@@ -1,5 +1,11 @@
# @llamaindex/waku-query-engine-test
## 0.0.162
### Patch Changes
- llamaindex@0.11.1
## 0.0.161
### Patch Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/waku-query-engine-test",
"version": "0.0.161",
"version": "0.0.162",
"type": "module",
"private": true,
"scripts": {
-332
View File
@@ -1,332 +0,0 @@
import { ChatMemoryBuffer, ChatMessage, LLM, MessageContent } from "llamaindex";
import {
agentStreamEvent,
createStatefulMiddleware,
createWorkflow,
startAgentEvent,
stopAgentEvent,
workflowEvent,
} from "@llamaindex/workflow";
import { z } from "zod";
export const DocumentRequirementSchema = z.object({
type: z.enum(["markdown", "html"]),
title: z.string(),
requirement: z.string(),
});
export type DocumentRequirement = z.infer<typeof DocumentRequirementSchema>;
export const UIEventSchema = z.object({
type: z.literal("ui_event"),
data: z.object({
state: z
.enum(["plan", "generate", "completed"])
.describe(
"The current state of the workflow: 'plan', 'generate', or 'completed'.",
),
requirement: z
.string()
.optional()
.describe(
"An optional requirement creating or updating a document, if applicable.",
),
}),
});
export type UIEvent = z.infer<typeof UIEventSchema>;
export const uiEvent = workflowEvent<UIEvent>();
const planEvent = workflowEvent<{
userInput: MessageContent;
context?: string | undefined;
}>();
const generateArtifactEvent = workflowEvent<{
requirement: DocumentRequirement;
}>();
const synthesizeAnswerEvent = workflowEvent<{
requirement: DocumentRequirement;
generatedArtifact: string;
}>();
const ArtifactSchema = z.object({
type: z.literal("artifact"),
data: z.object({
type: z.literal("document"),
data: z.object({
title: z.string(),
content: z.string(),
type: z.string(),
}),
created_at: z.number(),
}),
});
export type Artifact = z.infer<typeof ArtifactSchema>;
export const artifactEvent = workflowEvent<Artifact>();
export function createDocumentArtifactWorkflow(
llm: LLM,
chatHistory: ChatMessage[],
lastArtifact: Artifact | undefined,
) {
const { withState, getContext } = createStatefulMiddleware(() => {
return {
memory: new ChatMemoryBuffer({
llm,
chatHistory: chatHistory,
}),
lastArtifact: lastArtifact,
};
});
const workflow = withState(createWorkflow());
workflow.handle([startAgentEvent], async ({ data: { userInput } }) => {
// Prepare chat history
const { state } = getContext();
// Put user input to the memory
if (!userInput) {
throw new Error("Missing user input to start the workflow");
}
state.memory.put({
role: "user",
content: userInput,
});
return planEvent.with({
userInput,
context: state.lastArtifact
? JSON.stringify(state.lastArtifact)
: undefined,
});
});
workflow.handle([planEvent], async ({ data: planData }) => {
const { sendEvent } = getContext();
const { state } = getContext();
sendEvent(
uiEvent.with({
type: "ui_event",
data: {
state: "plan",
},
}),
);
const user_msg = planData.userInput;
const context = planData.context
? `## The context is: \n${planData.context}\n`
: "";
const prompt = `
You are a documentation analyst responsible for analyzing the user's request and providing requirements for document generation or update.
Follow these instructions:
1. Carefully analyze the conversation history and the user's request to determine what has been done and what the next step should be.
2. From the user's request, provide requirements for the next step of the document generation or update.
3. Do not be verbose; only return the requirements for the next step of the document generation or update.
4. Only the following document types are allowed: "markdown", "html".
5. The requirement should be in the following format:
\`\`\`json
{
"type": "markdown" | "html",
"title": string,
"requirement": string
}
\`\`\`
## Example:
User request: Create a project guideline document.
You should return:
\`\`\`json
{
"type": "markdown",
"title": "Project Guideline",
"requirement": "Generate a Markdown document that outlines the project goals, deliverables, and timeline. Include sections for introduction, objectives, deliverables, and timeline."
}
\`\`\`
User request: Add a troubleshooting section to the guideline.
You should return:
\`\`\`json
{
"type": "markdown",
"title": "Project Guideline",
"requirement": "Add a 'Troubleshooting' section at the end of the document with common issues and solutions."
}
\`\`\`
${context}
Now, please plan for the user's request:
${user_msg}
`;
const response = await llm.complete({
prompt,
});
// Parse the response to DocumentRequirement
const jsonBlock = response.text.match(/```json\s*([\s\S]*?)\s*```/);
if (!jsonBlock) {
throw new Error("No JSON block found in the response.");
}
const requirement = DocumentRequirementSchema.parse(
JSON.parse(jsonBlock[1]),
);
state.memory.put({
role: "assistant",
content: `Planning for the document generation: \n${response.text}`,
});
return generateArtifactEvent.with({
requirement,
});
});
workflow.handle(
[generateArtifactEvent],
async ({ data: { requirement } }) => {
const { sendEvent } = getContext();
const { state } = getContext();
sendEvent(
uiEvent.with({
type: "ui_event",
data: {
state: "generate",
requirement: requirement.requirement,
},
}),
);
const previousArtifact = state.lastArtifact
? JSON.stringify(state.lastArtifact)
: "";
const requirementStr = JSON.stringify(requirement);
const prompt = `
You are a skilled technical writer who can help users with documentation.
You are given a task to generate or update a document for a given requirement.
## Follow these instructions:
**1. Carefully read the user's requirements.**
If any details are ambiguous or missing, make reasonable assumptions and clearly reflect those in your output.
If the previous document is provided:
+ Carefully analyze the document with the request to make the right changes.
+ Avoid making unnecessary changes from the previous document if the request is not to rewrite it from scratch.
**2. For document requests:**
- If the user does not specify a type, default to Markdown.
- Ensure the document is clear, well-structured, and grammatically correct.
- Only generate content relevant to the user's request—do not add extra boilerplate.
**3. Do not be verbose in your response.**
- No other text or comments; only return the document content wrapped by the appropriate code block (\`\`\`markdown or \`\`\`html).
- If the user's request is to update the document, only return the updated document.
**4. Only the following types are allowed: "markdown", "html".**
**5. If there is no change to the document, return the reason without any code block.**
## Example:
\`\`\`markdown
# Project Guideline
## Introduction
...
\`\`\`
The previous content is:
${previousArtifact}
Now, please generate the document for the following requirement:
${requirementStr}
`;
const response = await llm.complete({
prompt,
});
// Extract the document from the response
const docMatch = response.text.match(/```(markdown|html)([\s\S]*)```/);
const generatedContent = response.text;
if (docMatch) {
const content = docMatch[2].trim();
const docType = docMatch[1] as "markdown" | "html";
// Put the generated document to the memory
state.memory.put({
role: "assistant",
content: `Generated document: \n${response.text}`,
});
// To show the Canvas panel for the artifact
sendEvent(
artifactEvent.with({
type: "artifact",
data: {
type: "document",
created_at: Date.now(),
data: {
title: requirement.title,
content: content,
type: docType,
},
},
}),
);
}
return synthesizeAnswerEvent.with({
requirement,
generatedArtifact: generatedContent,
});
},
);
workflow.handle([synthesizeAnswerEvent], async ({ data }) => {
const { sendEvent } = getContext();
const { state } = getContext();
const chatHistory = await state.memory.getMessages();
const messages = [
...chatHistory,
{
role: "system" as const,
content: `
Your responsibility is to explain the work to the user.
If there is no document to update, explain the reason.
If the document is updated, just summarize what changed. Don't need to include the whole document again in the response.
`,
},
];
const responseStream = await llm.chat({
messages,
stream: true,
});
sendEvent(
uiEvent.with({
type: "ui_event",
data: {
state: "completed",
requirement: data.requirement.requirement,
},
}),
);
let response = "";
for await (const chunk of responseStream) {
response += chunk.delta;
sendEvent(
agentStreamEvent.with({
delta: chunk.delta,
response: "",
currentAgentName: "assistant",
raw: chunk,
}),
);
}
return stopAgentEvent.with({
result: response,
});
});
return workflow;
}
-162
View File
@@ -1,162 +0,0 @@
import { openai } from "@llamaindex/openai";
import { Box, render, Text } from "ink";
import React from "react";
import {
agentStreamEvent,
startAgentEvent,
stopAgentEvent,
} from "@llamaindex/workflow";
import {
artifactEvent,
createDocumentArtifactWorkflow,
uiEvent,
UIEvent,
} from "./doc-workflow";
// Import Artifact type (assuming it's exported from doc-workflow.ts)
// If not exported, we'd define a simplified version here or use 'any' temporarily.
// Based on the read file, Artifact is a Zod schema inference.
// We can either import it or define a similar structure for props.
// For now, let's assume it's available from './doc-workflow'.
import type { Artifact } from "./doc-workflow"; // Assuming Artifact is exported
const llm = openai({ model: "gpt-4.1-mini" });
// Define the props for our UI component
interface WorkflowUIProps {
uiEvent: UIEvent;
}
// React component to render the UI using Ink
const WorkflowUI: React.FC<WorkflowUIProps> = ({ uiEvent }) => {
const contentWidth = 60;
const stateColorMap: { [key: string]: string } = {
plan: "yellow",
generate: "blue",
completed: "green",
};
const state = uiEvent.data.state;
const coloredStateText = (
<Text color={stateColorMap[state] || "white"}>{state.toUpperCase()}</Text>
);
const title = "💡 Workflow Event";
const requirementLabel = "Requirement: ";
return (
<Box
borderStyle="round"
borderColor="gray"
flexDirection="column"
paddingX={1}
width={contentWidth + 2 + 2}
>
<Box justifyContent="center" paddingTop={0} paddingBottom={0}>
<Text>{title}</Text>
</Box>
<Box borderStyle="single" borderColor="gray" height={1} marginY={1} />
<Box flexDirection="column">
<Box>
<Text>State: </Text>
{coloredStateText}
</Box>
{uiEvent.data.requirement !== undefined && (
<Box flexDirection="row" marginTop={1}>
<Text>{requirementLabel}</Text>
<Box flexGrow={1} flexShrink={1} marginLeft={1}>
<Text>{uiEvent.data.requirement}</Text>
</Box>
</Box>
)}
</Box>
</Box>
);
};
// --- BEGIN NEW ARTIFACT UI COMPONENT ---
interface ArtifactUIProps {
artifact: Artifact; // Use the imported Artifact type
}
const ArtifactUI: React.FC<ArtifactUIProps> = ({ artifact }) => {
const artifactData = artifact.data.data; // Access nested data
const contentWidth = 80; // Allow more width for artifact content
return (
<Box
borderStyle="round"
borderColor="cyan"
flexDirection="column"
padding={1}
width={contentWidth + 2 + 2} // Account for padding and border
>
<Box justifyContent="center" paddingBottom={1}>
<Text bold color="cyan">
📄 Artifact Generated: {artifactData.title}
</Text>
</Box>
<Box borderStyle="single" borderColor="gray" marginY={1} />
<Box flexDirection="column" gap={1}>
<Box>
<Text bold>Title: </Text>
<Text>{artifactData.title}</Text>
</Box>
<Box>
<Text bold>Type: </Text>
<Text>{artifactData.type}</Text>
</Box>
<Box flexDirection="column">
<Text bold>Content:</Text>
{/* Add a Box for content to allow it to take up space and wrap */}
<Box borderStyle="round" borderColor="gray" padding={1} marginTop={1}>
<Text>{artifactData.content}</Text>
</Box>
</Box>
<Box marginTop={1}>
<Text dimColor>
Created At: {new Date(artifact.data.created_at).toLocaleString()}
</Text>
</Box>
</Box>
</Box>
);
};
async function main() {
const workflow = createDocumentArtifactWorkflow(llm, [], undefined);
const { stream, sendEvent } = workflow.createContext();
const { rerender, unmount, clear } = render(
<Text>Waiting for workflow to start...</Text>,
);
sendEvent(
startAgentEvent.with({ userInput: "Create a project guideline document." }),
);
for await (const event of stream.until(stopAgentEvent)) {
if (agentStreamEvent.include(event)) {
process.stdout.write(event.data.delta);
} else if (uiEvent.include(event)) {
if (event.data.data.state !== "completed") {
rerender(<WorkflowUI uiEvent={event.data} />);
}
} else if (artifactEvent.include(event)) {
rerender(<ArtifactUI artifact={event.data} />);
}
}
unmount();
console.log("\nWorkflow finished.");
}
main().catch(console.error);
-4
View File
@@ -2,7 +2,6 @@
"name": "@llamaindex/examples",
"version": "0.3.16",
"private": true,
"type": "module",
"scripts": {
"lint": "eslint .",
"start": "tsx ./starter.ts"
@@ -64,18 +63,15 @@
"ajv": "^8.17.1",
"commander": "^12.1.0",
"dotenv": "^16.4.5",
"ink": "^5.2.1",
"js-tiktoken": "^1.0.14",
"llamaindex": "^0.11.0",
"mongodb": "6.7.0",
"postgres": "^3.4.4",
"react": "^18.3.1",
"wikipedia": "^2.1.2",
"zod": "^3.23.8"
},
"devDependencies": {
"@types/node": "^22.9.0",
"@types/react": "^18.3.21",
"tsx": "^4.19.3",
"typescript": "^5.7.3"
},
+8 -3
View File
@@ -10,8 +10,13 @@
"outDir": "./lib",
"tsBuildInfoFile": "./lib/.tsbuildinfo",
"incremental": true,
"composite": true,
"jsx": "react-jsx"
"composite": true
},
"include": ["./**/*.ts", "./**/*.tsx"]
"ts-node": {
"files": true,
"compilerOptions": {
"module": "commonjs"
}
},
"include": ["./**/*.ts"]
}
+6
View File
@@ -1,5 +1,11 @@
# @llamaindex/autotool
## 8.0.1
### Patch Changes
- llamaindex@0.11.1
## 8.0.0
### Patch Changes
@@ -1,5 +1,12 @@
# @llamaindex/autotool-01-node-example
## 0.0.109
### Patch Changes
- llamaindex@0.11.1
- @llamaindex/autotool@8.0.1
## 0.0.108
### Patch Changes
@@ -13,5 +13,5 @@
"scripts": {
"start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
},
"version": "0.0.108"
"version": "0.0.109"
}
+1 -1
View File
@@ -6,7 +6,7 @@
"url": "git+https://github.com/run-llama/LlamaIndexTS.git",
"directory": "packages/autotool"
},
"version": "8.0.0",
"version": "8.0.1",
"description": "auto transpile your JS function to LLM Agent compatible",
"files": [
"dist",
+6
View File
@@ -1,5 +1,11 @@
# @llamaindex/cloud
## 4.0.9
### Patch Changes
- 3703f90: feat(parse): add upload API
## 4.0.8
### Patch Changes
+6 -1
View File
@@ -12,8 +12,13 @@ export default defineConfig({
plugins: [
...defaultPlugins,
"@hey-api/client-fetch",
"zod",
"@hey-api/schemas",
"@hey-api/sdk",
{
name: "@hey-api/sdk",
enums: "javascript",
identifierCase: "PascalCase",
name: "@hey-api/typescript",
},
],
});
+2868 -953
View File
File diff suppressed because it is too large Load Diff
+18 -4
View File
@@ -1,6 +1,6 @@
{
"name": "@llamaindex/cloud",
"version": "4.0.8",
"version": "4.0.9",
"type": "module",
"license": "MIT",
"scripts": {
@@ -37,6 +37,17 @@
},
"default": "./reader/dist/index.js"
},
"./parse": {
"require": {
"types": "./parse/dist/index.d.cts",
"default": "./parse/dist/index.cjs"
},
"import": {
"types": "./parse/dist/index.d.ts",
"default": "./parse/dist/index.js"
},
"default": "./parse/dist/index.js"
},
".": {
"require": {
"types": "./reader/dist/index.d.cts",
@@ -55,16 +66,19 @@
"directory": "packages/cloud"
},
"devDependencies": {
"@hey-api/client-fetch": "^0.10.0",
"@hey-api/openapi-ts": "^0.66.7",
"@hey-api/client-fetch": "^0.10.1",
"@hey-api/openapi-ts": "^0.67.5",
"@llama-flow/core": "^0.4.1",
"@llamaindex/core": "workspace:*",
"@llamaindex/env": "workspace:*"
},
"peerDependencies": {
"@llama-flow/core": "^0.4.1",
"@llamaindex/core": "workspace:*",
"@llamaindex/env": "workspace:*"
},
"dependencies": {
"p-retry": "^6.2.1"
"p-retry": "^6.2.1",
"zod": "^3.25.7"
}
}
+8
View File
@@ -0,0 +1,8 @@
{
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
"types": "./dist/index.d.ts",
"exports": "./dist/index.js",
"private": true
}
+55
View File
@@ -0,0 +1,55 @@
import { workflowEvent } from "@llama-flow/core";
import { zodEvent } from "@llama-flow/core/util/zod";
import { z } from "zod";
import { parseFormSchema } from "./schema";
export const uploadEvent = zodEvent(
parseFormSchema.merge(
z.object({
file: z
.string()
.or(z.instanceof(File))
.or(z.instanceof(Blob))
.or(z.instanceof(Uint8Array))
.optional()
.describe("input"),
}),
),
{
debugLabel: "upload",
uniqueId: "52099967-34a8-419b-8950-c859eab60145",
},
);
export const checkStatusEvent = workflowEvent<string>({
debugLabel: "check-status",
uniqueId: "462157fc-1ded-4ba7-9bc4-3e870395bd20",
});
export const checkStatusSuccessEvent = workflowEvent<string>({
debugLabel: "check-status-success",
uniqueId: "360b7641-30f7-456e-851d-104bb5e3f8d2",
});
export const requestMarkdownEvent = workflowEvent<string>({
debugLabel: "markdown-request",
uniqueId: "aa4c2e3c-0f09-4035-aab6-c72719c877cc",
});
export const requestTextEvent = workflowEvent<string>({
debugLabel: "text-request",
uniqueId: "6976536e-5399-4285-9455-0b70f1dfc92b",
});
export const requestJsonEvent = workflowEvent<string>({
debugLabel: "json-request",
uniqueId: "9fc4a330-52ad-4aac-8092-a650998b7f6f",
});
export const markdownResultEvent = workflowEvent<string>({
debugLabel: "markdown-result",
uniqueId: "2dfb57c8-73d1-4394-bea8-f05246d934d4",
});
export const textResultEvent = workflowEvent<string>({
debugLabel: "text-result",
uniqueId: "a27deec6-b24f-4eda-a5ac-ba2fb2bf37c8",
});
export const jsonResultEvent = workflowEvent<unknown>({
debugLabel: "json-result",
uniqueId: "e086e6bd-a612-4f25-ab41-9b31dcb8af86",
});
+225
View File
@@ -0,0 +1,225 @@
import { createClient, createConfig } from "@hey-api/client-fetch";
import { createWorkflow, type InferWorkflowEventData } from "@llama-flow/core";
import { createStatefulMiddleware } from "@llama-flow/core/middleware/state";
import { withTraceEvents } from "@llama-flow/core/middleware/trace-events";
import { pRetryHandler } from "@llama-flow/core/util/p-retry";
import { fs, getEnv, path } from "@llamaindex/env";
import {
type BodyUploadFileApiV1ParsingUploadPost,
getJobApiV1ParsingJobJobIdGet,
getJobJsonResultApiV1ParsingJobJobIdResultJsonGet,
getJobResultApiV1ParsingJobJobIdResultMarkdownGet,
getJobTextResultApiV1ParsingJobJobIdResultTextGet,
type StatusEnum,
uploadFileApiV1ParsingUploadPost,
} from "./client";
import {
checkStatusEvent,
checkStatusSuccessEvent,
jsonResultEvent,
markdownResultEvent,
requestJsonEvent,
requestMarkdownEvent,
requestTextEvent,
textResultEvent,
uploadEvent,
} from "./events";
export type LlamaParseWorkflowParams = {
region?: "us" | "eu" | "us-staging";
apiKey?: string;
};
const URLS = {
us: "https://api.cloud.llamaindex.ai",
eu: "https://api.cloud.eu.llamaindex.ai",
"us-staging": "https://api.staging.llamaindex.ai",
} as const;
const { withState, getContext } = createStatefulMiddleware(
(params: LlamaParseWorkflowParams) => {
const apiKey = params.apiKey ?? getEnv("LLAMA_CLOUD_API_KEY");
const region = params.region ?? "us";
if (!apiKey) {
throw new Error("LLAMA_CLOUD_API_KEY is not set");
}
return {
cache: {} as Record<string, StatusEnum>,
client: createClient(
createConfig({
baseUrl: URLS[region],
headers: {
Authorization: `Bearer ${apiKey}`,
},
}),
),
};
},
);
const llamaParseWorkflow = withState(withTraceEvents(createWorkflow()));
llamaParseWorkflow.handle([uploadEvent], async ({ data: form }) => {
const { state } = getContext();
const finalForm = { ...form };
if ("file" in form) {
// support loads from the file system
const file = form?.file;
const isFilePath = typeof file === "string";
const data = isFilePath ? await fs.readFile(file) : file;
const filename: string | undefined = isFilePath
? path.basename(file)
: undefined;
finalForm.file = data
? globalThis.File && filename
? new File([data], filename)
: new Blob([data])
: undefined;
}
const {
data: { id, status },
} = await uploadFileApiV1ParsingUploadPost({
throwOnError: true,
body: {
...finalForm,
} as BodyUploadFileApiV1ParsingUploadPost,
client: state.client,
});
state.cache[id] = status;
return checkStatusEvent.with(id);
});
llamaParseWorkflow.handle(
[checkStatusEvent],
pRetryHandler(
async ({ data: uuid }) => {
const { state } = getContext();
if (state.cache[uuid] === "SUCCESS") {
return checkStatusSuccessEvent.with(uuid);
}
const {
data: { status },
} = await getJobApiV1ParsingJobJobIdGet({
throwOnError: true,
path: {
job_id: uuid,
},
client: state.client,
});
state.cache[uuid] = status;
if (status === "SUCCESS") {
return checkStatusSuccessEvent.with(uuid);
}
throw new Error(`LLamaParse status: ${status}`);
},
{
retries: 100,
},
),
);
//#region sub workflow
llamaParseWorkflow.handle([requestMarkdownEvent], async ({ data: job_id }) => {
const { state } = getContext();
const { data } = await getJobResultApiV1ParsingJobJobIdResultMarkdownGet({
throwOnError: true,
path: {
job_id,
},
client: state.client,
});
return markdownResultEvent.with(data.markdown);
});
llamaParseWorkflow.handle([requestTextEvent], async ({ data: job_id }) => {
const { state } = getContext();
const { data } = await getJobTextResultApiV1ParsingJobJobIdResultTextGet({
throwOnError: true,
path: {
job_id,
},
client: state.client,
});
return textResultEvent.with(data.text);
});
llamaParseWorkflow.handle([requestJsonEvent], async ({ data: job_id }) => {
const { state } = getContext();
const { data } = await getJobJsonResultApiV1ParsingJobJobIdResultJsonGet({
throwOnError: true,
path: {
job_id,
},
client: state.client,
});
return jsonResultEvent.with(data.pages);
});
//#endregion
export type ParseJob = {
get jobId(): string;
get signal(): AbortSignal;
get context(): ReturnType<typeof llamaParseWorkflow.createContext>;
get form(): InferWorkflowEventData<typeof uploadEvent>;
markdown(): Promise<string>;
text(): Promise<string>;
//eslint-disable-next-line @typescript-eslint/no-explicit-any
json(): Promise<any[]>;
};
export const upload = async (
params: InferWorkflowEventData<typeof uploadEvent> & LlamaParseWorkflowParams,
): Promise<ParseJob> => {
//#region upload event
const context = llamaParseWorkflow.createContext(params);
const { stream, sendEvent } = context;
const ev = uploadEvent.with(params);
sendEvent(ev);
const uploadThread = await llamaParseWorkflow
.substream(ev, stream)
.until((ev) => checkStatusSuccessEvent.include(ev))
.toArray();
//#region
const jobId: string = uploadThread.at(-1)!.data;
return {
get signal() {
// lazy load
return context.signal;
},
get jobId() {
return jobId;
},
get form() {
return ev.data;
},
get context() {
return context;
},
async markdown(): Promise<string> {
const requestEv = requestMarkdownEvent.with(jobId);
const { sendEvent, stream } = llamaParseWorkflow.createContext(params);
sendEvent(requestEv);
const markdownThread = await stream.until(markdownResultEvent).toArray();
return markdownThread.at(-1)!.data;
},
async text(): Promise<string> {
const requestEv = requestTextEvent.with(jobId);
const { sendEvent, stream } = llamaParseWorkflow.createContext(params);
sendEvent(requestEv);
const textThread = await stream.until(textResultEvent).toArray();
return textThread.at(-1)!.data;
},
//eslint-disable-next-line @typescript-eslint/no-explicit-any
async json(): Promise<any[]> {
const requestEv = requestJsonEvent.with(jobId);
const { sendEvent, stream } = llamaParseWorkflow.createContext(params);
sendEvent(requestEv);
const jsonThread = await stream
.until((ev) => jsonResultEvent.include(ev))
.toArray();
return jsonThread.at(-1)!.data;
},
};
};
+131
View File
@@ -0,0 +1,131 @@
import { FailPageMode, ParserLanguages, ParsingMode } from "./client";
import { z } from "zod";
type Language = ParserLanguages;
const VALUES: [Language, ...Language[]] = [
ParserLanguages.EN,
...Object.values(ParserLanguages),
];
const languageSchema = z.enum(VALUES);
const PARSE_PRESETS = [
"fast",
"balanced",
"premium",
"structured",
"auto",
"scientific",
"invoice",
"slides",
"_carlyle",
] as const;
export const parsePresetSchema = z.enum(PARSE_PRESETS);
export const parseFormSchema = z.object({
adaptive_long_table: z.boolean().optional(),
annotate_links: z.boolean().optional(),
auto_mode: z.boolean().optional(),
auto_mode_trigger_on_image_in_page: z.boolean().optional(),
auto_mode_trigger_on_table_in_page: z.boolean().optional(),
auto_mode_trigger_on_text_in_page: z.string().optional(),
auto_mode_trigger_on_regexp_in_page: z.string().optional(),
auto_mode_configuration_json: z.string().optional(),
azure_openai_api_version: z.string().optional(),
azure_openai_deployment_name: z.string().optional(),
azure_openai_endpoint: z.string().optional(),
azure_openai_key: z.string().optional(),
bbox_bottom: z.number().min(0).max(1).optional(),
bbox_left: z.number().min(0).max(1).optional(),
bbox_right: z.number().min(0).max(1).optional(),
bbox_top: z.number().min(0).max(1).optional(),
disable_ocr: z.boolean().optional(),
disable_reconstruction: z.boolean().optional(),
disable_image_extraction: z.boolean().optional(),
do_not_cache: z.coerce.boolean().optional(),
do_not_unroll_columns: z.coerce.boolean().optional(),
extract_charts: z.boolean().optional(),
guess_xlsx_sheet_name: z.boolean().optional(),
html_make_all_elements_visible: z.boolean().optional(),
html_remove_fixed_elements: z.boolean().optional(),
html_remove_navigation_elements: z.boolean().optional(),
http_proxy: z
.string()
.url(
'Set a valid URL for the HTTP proxy, e.g., "http://proxy.example.com:8080"',
)
.refine(
(url) => {
try {
const parsedUrl = new URL(url);
return (
parsedUrl.protocol === "http:" || parsedUrl.protocol === "https:"
);
} catch {
return false;
}
},
{
message: "Invalid HTTP proxy URL",
},
)
.optional(),
input_s3_path: z.string().optional(),
input_s3_region: z.string().optional(),
input_url: z.string().optional(),
invalidate_cache: z.boolean().optional(),
language: z.array(languageSchema).optional(),
extract_layout: z.boolean().optional(),
max_pages: z.number().nullable().optional(),
output_pdf_of_document: z.boolean().optional(),
output_s3_path_prefix: z.string().optional(),
output_s3_region: z.string().optional(),
page_prefix: z.string().optional(),
page_separator: z.string().optional(),
page_suffix: z.string().optional(),
preserve_layout_alignment_across_pages: z.boolean().optional(),
skip_diagonal_text: z.boolean().optional(),
spreadsheet_extract_sub_tables: z.boolean().optional(),
structured_output: z.boolean().optional(),
structured_output_json_schema: z.string().optional(),
structured_output_json_schema_name: z.string().optional(),
take_screenshot: z.boolean().optional(),
target_pages: z.string().optional(),
vendor_multimodal_api_key: z.string().optional(),
vendor_multimodal_model_name: z.string().optional(),
model: z.string().optional(),
webhook_url: z.string().url().optional(),
parse_mode: z.nativeEnum(ParsingMode).nullable().optional(),
system_prompt: z.string().optional(),
system_prompt_append: z.string().optional(),
user_prompt: z.string().optional(),
job_timeout_in_seconds: z.number().optional(),
job_timeout_extra_time_per_page_in_seconds: z.number().optional(),
strict_mode_image_extraction: z.boolean().optional(),
strict_mode_image_ocr: z.boolean().optional(),
strict_mode_reconstruction: z.boolean().optional(),
strict_mode_buggy_font: z.boolean().optional(),
save_images: z.boolean().optional(),
ignore_document_elements_for_layout_detection: z.boolean().optional(),
output_tables_as_HTML: z.boolean().optional(),
use_vendor_multimodal_model: z.boolean().optional(),
bounding_box: z.string().optional(),
gpt4o_mode: z.boolean().optional(),
gpt4o_api_key: z.string().optional(),
complemental_formatting_instruction: z.string().optional(),
content_guideline_instruction: z.string().optional(),
premium_mode: z.boolean().optional(),
is_formatting_instruction: z.boolean().optional(),
continuous_mode: z.boolean().optional(),
parsing_instruction: z.string().optional(),
fast_mode: z.boolean().optional(),
formatting_instruction: z.string().optional(),
preset: parsePresetSchema.optional(),
compact_markdown_table: z.boolean().optional(),
markdown_table_multiline_header_separator: z.string().optional(),
page_error_tolerance: z.number().min(0).max(1).optional(),
replace_failed_page_mode: z.nativeEnum(FailPageMode).nullable().optional(),
replace_failed_page_with_error_message_prefix: z.string().optional(),
replace_failed_page_with_error_message_suffix: z.string().optional(),
});
+6
View File
@@ -1,5 +1,11 @@
# @llamaindex/experimental
## 0.0.178
### Patch Changes
- llamaindex@0.11.1
## 0.0.177
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/experimental",
"description": "Experimental package for LlamaIndexTS",
"version": "0.0.177",
"version": "0.0.178",
"type": "module",
"types": "dist/type/index.d.ts",
"main": "dist/cjs/index.js",
+7
View File
@@ -1,5 +1,12 @@
# llamaindex
## 0.11.1
### Patch Changes
- Updated dependencies [3703f90]
- @llamaindex/cloud@4.0.9
## 0.11.0
### Minor Changes
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "llamaindex",
"version": "0.11.0",
"version": "0.11.1",
"license": "MIT",
"type": "module",
"keywords": [
+1132 -431
View File
File diff suppressed because it is too large Load Diff
+8
View File
@@ -1,5 +1,13 @@
# @llamaindex/unit-test
## 0.1.29
### Patch Changes
- Updated dependencies [3703f90]
- @llamaindex/cloud@4.0.9
- llamaindex@0.11.1
## 0.1.28
### Patch Changes
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "@llamaindex/unit-test",
"private": true,
"version": "0.1.28",
"version": "0.1.29",
"type": "module",
"scripts": {
"test": "vitest run"