Merge branch 'main' into feat/build-wasm-with-extism

.changeset/quick-brooms-sniff.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+feat: update JinaAIEmbedding, support embedding v3

.changeset/shy-bottles-shop.md

@@ -0,0 +1,5 @@
+---
+"@llamaindex/extism-tools": patch
+---
+
+Build WASM tools with extism

.github/workflows/lint_on_push_or_pull.yml

@@ -19,6 +19,10 @@ jobs:
         with:
           node-version-file: ".nvmrc"
           cache: "pnpm"
+      - name: Install Extism
+        run: |
+          curl -O https://raw.githubusercontent.com/extism/js-pdk/main/install.sh
+          sh install.sh
       - name: Install dependencies
         run: pnpm install
       - name: Run lint

.github/workflows/preview.yml

@@ -18,6 +18,11 @@ jobs:
           node-version-file: ".nvmrc"
           cache: "pnpm"
 
+      - name: Install Extism
+        run: |
+          curl -O https://raw.githubusercontent.com/extism/js-pdk/main/install.sh
+          sh install.sh
+
       - name: Install dependencies
         run: pnpm install
 

.github/workflows/release.yml

@@ -23,6 +23,11 @@ jobs:
           node-version-file: ".nvmrc"
           cache: "pnpm"
 
+      - name: Install Extism
+        run: |
+          curl -O https://raw.githubusercontent.com/extism/js-pdk/main/install.sh
+          sh install.sh
+
       - name: Install dependencies
         run: pnpm install
 

.github/workflows/test.yml

@@ -13,6 +13,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
+  POSTGRES_USER: runneradmin
   POSTGRES_HOST_AUTH_METHOD: trust
 
 jobs:
@@ -41,6 +42,10 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
           cache: "pnpm"
+      - name: Install Extism
+        run: |
+          curl -O https://raw.githubusercontent.com/extism/js-pdk/main/install.sh
+          sh install.sh
       - name: Install dependencies
         run: pnpm install
       - name: Run E2E Tests
@@ -61,6 +66,10 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
           cache: "pnpm"
+      - name: Install Extism
+        run: |
+          curl -O https://raw.githubusercontent.com/extism/js-pdk/main/install.sh
+          sh install.sh
       - name: Install dependencies
         run: pnpm install
       - name: Run tests
@@ -76,6 +85,10 @@ jobs:
         with:
           node-version-file: ".nvmrc"
           cache: "pnpm"
+      - name: Install Extism
+        run: |
+          curl -O https://raw.githubusercontent.com/extism/js-pdk/main/install.sh
+          sh install.sh
       - name: Install dependencies
         run: pnpm install
       - name: Build
@@ -103,7 +116,6 @@ jobs:
           - nextjs-edge-runtime
           - nextjs-node-runtime
           - waku-query-engine
-          - llama-parse-browser
     runs-on: ubuntu-latest
     name: Build LlamaIndex Example (${{ matrix.packages }})
     steps:
@@ -114,6 +126,10 @@ jobs:
         with:
           node-version-file: ".nvmrc"
           cache: "pnpm"
+      - name: Install Extism
+        run: |
+          curl -O https://raw.githubusercontent.com/extism/js-pdk/main/install.sh
+          sh install.sh
       - name: Install dependencies
         run: pnpm install
       - name: Build llamaindex
@@ -133,6 +149,10 @@ jobs:
         with:
           node-version-file: ".nvmrc"
           cache: "pnpm"
+      - name: Install Extism
+        run: |
+          curl -O https://raw.githubusercontent.com/extism/js-pdk/main/install.sh
+          sh install.sh
       - name: Install dependencies
         run: pnpm install
       - name: Build
@@ -145,9 +165,6 @@ jobs:
       - name: Pack @llamaindex/openai
         run: pnpm pack --pack-destination ${{ runner.temp }}
         working-directory: packages/llm/openai
-      - name: Pack @llamaindex/groq
-        run: pnpm pack --pack-destination ${{ runner.temp }}
-        working-directory: packages/llm/groq
       - name: Pack @llamaindex/core
         run: pnpm pack --pack-destination ${{ runner.temp }}
         working-directory: packages/core

apps/docs/CHANGELOG.md

@@ -1,30 +1,5 @@
 # docs
 
-## 0.0.72
-
-### Patch Changes
-
-- Updated dependencies [2cd1383]
-- Updated dependencies [5c4badb]
-  - llamaindex@0.6.3
-
-## 0.0.71
-
-### Patch Changes
-
-- Updated dependencies [749b43a]
-  - llamaindex@0.6.2
-
-## 0.0.70
-
-### Patch Changes
-
-- Updated dependencies [fbd5e01]
-- Updated dependencies [6b70c54]
-- Updated dependencies [1a6137b]
-- Updated dependencies [85c2e19]
-  - llamaindex@0.6.1
-
 ## 0.0.69
 
 ### Patch Changes

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.72",
+  "version": "0.0.69",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",

examples/anthropic/chat_interactive.ts

@@ -1,4 +1,4 @@
-import { Anthropic, ChatMemoryBuffer, SimpleChatEngine } from "llamaindex";
+import { Anthropic, SimpleChatEngine, SimpleChatHistory } from "llamaindex";
 import { stdin as input, stdout as output } from "node:process";
 import readline from "node:readline/promises";
 
@@ -8,8 +8,8 @@ import readline from "node:readline/promises";
     model: "claude-3-opus",
   });
   // chatHistory will store all the messages in the conversation
-  const chatHistory = new ChatMemoryBuffer({
-    chatHistory: [
+  const chatHistory = new SimpleChatHistory({
+    messages: [
       {
         content: "You want to talk in rhymes.",
         role: "system",

examples/chatHistory.ts

@@ -2,10 +2,10 @@ import { stdin as input, stdout as output } from "node:process";
 import readline from "node:readline/promises";
 
 import {
-  ChatSummaryMemoryBuffer,
   OpenAI,
   Settings,
   SimpleChatEngine,
+  SummaryChatHistory,
 } from "llamaindex";
 
 if (process.env.NODE_ENV === "development") {
@@ -18,7 +18,7 @@ async function main() {
   // Set maxTokens to 75% of the context window size of 4096
   // This will trigger the summarizer once the chat history reaches 25% of the context window size (1024 tokens)
   const llm = new OpenAI({ model: "gpt-3.5-turbo", maxTokens: 4096 * 0.75 });
-  const chatHistory = new ChatSummaryMemoryBuffer({ llm });
+  const chatHistory = new SummaryChatHistory({ llm });
   const chatEngine = new SimpleChatEngine({ llm });
   const rl = readline.createInterface({ input, output });
 

examples/groq.ts

@@ -1,23 +1,12 @@
 import fs from "node:fs/promises";
 
-import {
-  Document,
-  Groq,
-  HuggingFaceEmbedding,
-  Settings,
-  VectorStoreIndex,
-} from "llamaindex";
+import { Document, Groq, Settings, VectorStoreIndex } from "llamaindex";
 
 // Update llm to use Groq
 Settings.llm = new Groq({
   apiKey: process.env.GROQ_API_KEY,
 });
 
-// Use HuggingFace for embeddings
-Settings.embedModel = new HuggingFaceEmbedding({
-  modelType: "Xenova/all-mpnet-base-v2",
-});
-
 async function main() {
   // Load essay from abramov.txt in Node
   const path = "node_modules/llamaindex/examples/abramov.txt";

examples/huggingface/embedding.ts

@@ -27,12 +27,10 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const stream = await queryEngine.query(
-    {
-      query: "What did the author do in college?",
-    },
-    true,
-  );
+  const stream = await queryEngine.query({
+    query: "What did the author do in college?",
+    stream: true,
+  });
 
   // Output response
   for await (const chunk of stream) {

examples/huggingface/embeddingApi.ts

@@ -37,12 +37,10 @@ async function main() {
 
   // Query the index
   const queryEngine = index.asQueryEngine();
-  const stream = await queryEngine.query(
-    {
-      query: "What did the author do in college?",
-    },
-    true,
-  );
+  const stream = await queryEngine.query({
+    query: "What did the author do in college?",
+    stream: true,
+  });
 
   // Output response
   for await (const chunk of stream) {

examples/lowlevel.ts

@@ -1,7 +1,7 @@
 import {
   Document,
-  getResponseSynthesizer,
   NodeWithScore,
+  ResponseSynthesizer,
   SentenceSplitter,
   TextNode,
 } from "llamaindex";
@@ -14,7 +14,7 @@ import {
 
   console.log(nodes);
 
-  const responseSynthesizer = getResponseSynthesizer("compact");
+  const responseSynthesizer = new ResponseSynthesizer();
 
   const nodesWithScore: NodeWithScore[] = [
     {
@@ -30,7 +30,7 @@ import {
   const stream = await responseSynthesizer.synthesize(
     {
       query: "What age am I?",
-      nodes: nodesWithScore,
+      nodesWithScore,
     },
     true,
   );

examples/multimodal/rag.ts

@@ -1,5 +1,5 @@
 import {
-  getResponseSynthesizer,
+  MultiModalResponseSynthesizer,
   OpenAI,
   Settings,
   VectorStoreIndex,
@@ -27,15 +27,13 @@ async function main() {
   });
 
   const queryEngine = index.asQueryEngine({
-    responseSynthesizer: getResponseSynthesizer("multi_modal"),
+    responseSynthesizer: new MultiModalResponseSynthesizer(),
     retriever: index.asRetriever({ topK: { TEXT: 3, IMAGE: 1 } }),
   });
-  const stream = await queryEngine.query(
-    {
-      query: "Tell me more about Vincent van Gogh's famous paintings",
-    },
-    true,
-  );
+  const stream = await queryEngine.query({
+    query: "Tell me more about Vincent van Gogh's famous paintings",
+    stream: true,
+  });
   for await (const chunk of stream) {
     process.stdout.write(chunk.response);
   }

examples/prompts/promptMixin.ts

@@ -1,7 +1,8 @@
 import {
   Document,
-  getResponseSynthesizer,
   PromptTemplate,
+  ResponseSynthesizer,
+  TreeSummarize,
   TreeSummarizePrompt,
   VectorStoreIndex,
 } from "llamaindex";
@@ -26,7 +27,9 @@ async function main() {
 
   const query = "The quick brown fox jumps over the lazy dog";
 
-  const responseSynthesizer = getResponseSynthesizer("tree_summarize");
+  const responseSynthesizer = new ResponseSynthesizer({
+    responseBuilder: new TreeSummarize(),
+  });
 
   const queryEngine = index.asQueryEngine({
     responseSynthesizer,

examples/readers/src/csv.ts

@@ -1,7 +1,8 @@
 import {
-  getResponseSynthesizer,
+  CompactAndRefine,
   OpenAI,
   PromptTemplate,
+  ResponseSynthesizer,
   Settings,
   VectorStoreIndex,
 } from "llamaindex";
@@ -28,8 +29,8 @@ Given the CSV file, generate me Typescript code to answer the question: {query}.
 `,
   });
 
-  const responseSynthesizer = getResponseSynthesizer("compact", {
-    textQATemplate: csvPrompt,
+  const responseSynthesizer = new ResponseSynthesizer({
+    responseBuilder: new CompactAndRefine(undefined, csvPrompt),
   });
 
   const queryEngine = index.asQueryEngine({ responseSynthesizer });

examples/readers/src/llamaparse-json.ts

@@ -1,4 +1,3 @@
-import { createMessageContent } from "@llamaindex/core/utils";
 import {
   Document,
   ImageNode,
@@ -7,6 +6,7 @@ import {
   PromptTemplate,
   VectorStoreIndex,
 } from "llamaindex";
+import { createMessageContent } from "llamaindex/synthesizers/utils";
 
 const reader = new LlamaParseReader();
 async function main() {

examples/vectorIndexAnthropic.ts

@@ -2,10 +2,12 @@ import fs from "node:fs/promises";
 
 import {
   Anthropic,
+  CompactAndRefine,
   Document,
+  ResponseSynthesizer,
   Settings,
   VectorStoreIndex,
-  getResponseSynthesizer,
+  anthropicTextQaPrompt,
 } from "llamaindex";
 
 // Update llm to use Anthropic
@@ -21,7 +23,9 @@ async function main() {
   const document = new Document({ text: essay, id_: path });
 
   // Split text and create embeddings. Store them in a VectorStoreIndex
-  const responseSynthesizer = getResponseSynthesizer("compact");
+  const responseSynthesizer = new ResponseSynthesizer({
+    responseBuilder: new CompactAndRefine(undefined, anthropicTextQaPrompt),
+  });
 
   const index = await VectorStoreIndex.fromDocuments([document]);
 

examples/vectorIndexFromVectorStore.ts

@@ -1,10 +1,11 @@
 import {
-  getResponseSynthesizer,
   OpenAI,
   OpenAIEmbedding,
+  ResponseSynthesizer,
   RetrieverQueryEngine,
   Settings,
   TextNode,
+  TreeSummarize,
   VectorIndexRetriever,
   VectorStore,
   VectorStoreIndex,
@@ -164,7 +165,10 @@ async function main() {
       similarityTopK: 500,
     });
 
-    const responseSynthesizer = getResponseSynthesizer("tree_summarize");
+    const responseSynthesizer = new ResponseSynthesizer({
+      responseBuilder: new TreeSummarize(),
+    });
+
     return new RetrieverQueryEngine(retriever, responseSynthesizer, {
       filter,
     });

packages/autotool/CHANGELOG.md

@@ -1,34 +1,5 @@
 # @llamaindex/autotool
 
-## 3.0.3
-
-### Patch Changes
-
-- Updated dependencies [2cd1383]
-- Updated dependencies [5c4badb]
-  - llamaindex@0.6.3
-
-## 3.0.2
-
-### Patch Changes
-
-- Updated dependencies [749b43a]
-  - llamaindex@0.6.2
-
-## 3.0.1
-
-### Patch Changes
-
-- 1a6137b: feat: experimental support for browser
-
-  If you see bundler issue in next.js edge runtime, please bump to `next@14` latest version.
-
-- Updated dependencies [fbd5e01]
-- Updated dependencies [6b70c54]
-- Updated dependencies [1a6137b]
-- Updated dependencies [85c2e19]
-  - llamaindex@0.6.1
-
 ## 3.0.0
 
 ### Patch Changes

packages/autotool/examples/01_node/CHANGELOG.md

@@ -1,33 +1,5 @@
 # @llamaindex/autotool-01-node-example
 
-## 0.0.12
-
-### Patch Changes
-
-- Updated dependencies [2cd1383]
-- Updated dependencies [5c4badb]
-  - llamaindex@0.6.3
-  - @llamaindex/autotool@3.0.3
-
-## 0.0.11
-
-### Patch Changes
-
-- Updated dependencies [749b43a]
-  - llamaindex@0.6.2
-  - @llamaindex/autotool@3.0.2
-
-## 0.0.10
-
-### Patch Changes
-
-- Updated dependencies [fbd5e01]
-- Updated dependencies [6b70c54]
-- Updated dependencies [1a6137b]
-- Updated dependencies [85c2e19]
-  - llamaindex@0.6.1
-  - @llamaindex/autotool@3.0.1
-
 ## 0.0.9
 
 ### Patch Changes

packages/autotool/examples/01_node/package.json

@@ -13,5 +13,5 @@
   "scripts": {
     "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"
   },
-  "version": "0.0.12"
+  "version": "0.0.9"
 }

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,33 +1,5 @@
 # @llamaindex/autotool-02-next-example
 
-## 0.1.56
-
-### Patch Changes
-
-- Updated dependencies [2cd1383]
-- Updated dependencies [5c4badb]
-  - llamaindex@0.6.3
-  - @llamaindex/autotool@3.0.3
-
-## 0.1.55
-
-### Patch Changes
-
-- Updated dependencies [749b43a]
-  - llamaindex@0.6.2
-  - @llamaindex/autotool@3.0.2
-
-## 0.1.54
-
-### Patch Changes
-
-- Updated dependencies [fbd5e01]
-- Updated dependencies [6b70c54]
-- Updated dependencies [1a6137b]
-- Updated dependencies [85c2e19]
-  - llamaindex@0.6.1
-  - @llamaindex/autotool@3.0.1
-
 ## 0.1.53
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.56",
+  "version": "0.1.53",
   "scripts": {
     "dev": "next dev",
     "build": "next build",

packages/autotool/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool",
   "type": "module",
-  "version": "3.0.3",
+  "version": "3.0.0",
   "description": "auto transpile your JS function to LLM Agent compatible",
   "files": [
     "dist",
@@ -72,7 +72,7 @@
     "@types/node": "^22.5.1",
     "bunchee": "5.3.2",
     "llamaindex": "workspace:*",
-    "next": "14.2.11",
+    "next": "14.2.7",
     "rollup": "^4.21.2",
     "tsx": "^4.19.0",
     "typescript": "^5.6.2",

packages/cloud/CHANGELOG.md

@@ -1,41 +1,5 @@
 # @llamaindex/cloud
 
-## 0.2.7
-
-### Patch Changes
-
-- fb36eff: fix: backport for node.js 18
-
-  There could have one missing API in the node.js 18, so we need to backport it to make it work.
-
-- d24d3d1: fix: print warning when llama parse reader has error
-- Updated dependencies [2cd1383]
-  - @llamaindex/core@0.2.3
-
-## 0.2.6
-
-### Patch Changes
-
-- b42adeb: fix: get job result in llama parse reader
-- Updated dependencies [749b43a]
-  - @llamaindex/core@0.2.2
-
-## 0.2.5
-
-### Patch Changes
-
-- 85c2e19: feat: `@llamaindex/cloud` package update
-
-  - Bump to latest openapi schema
-  - Move LlamaParse class from llamaindex, this will allow you use llamaparse in more non-node.js environment
-
-- Updated dependencies [ac07e3c]
-- Updated dependencies [70ccb4a]
-- Updated dependencies [1a6137b]
-- Updated dependencies [ac07e3c]
-  - @llamaindex/core@0.2.1
-  - @llamaindex/env@0.1.11
-
 ## 0.2.4
 
 ### Patch Changes

packages/cloud/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloud",
-  "version": "0.2.7",
+  "version": "0.2.4",
   "type": "module",
   "license": "MIT",
   "scripts": {
@@ -26,20 +26,6 @@
         "types": "./dist/api.d.ts",
         "default": "./dist/api.js"
       }
-    },
-    "./reader": {
-      "require": {
-        "types": "./dist/reader.d.cts",
-        "default": "./dist/reader.cjs"
-      },
-      "import": {
-        "types": "./dist/reader.d.ts",
-        "default": "./dist/reader.js"
-      },
-      "default": {
-        "types": "./dist/reader.d.ts",
-        "default": "./dist/reader.js"
-      }
     }
   },
   "repository": {
@@ -50,15 +36,6 @@
   "devDependencies": {
     "@hey-api/client-fetch": "^0.2.4",
     "@hey-api/openapi-ts": "^0.53.0",
-    "@llamaindex/core": "workspace:^0.2.3",
-    "@llamaindex/env": "workspace:^0.1.11",
     "bunchee": "5.3.2"
-  },
-  "peerDependencies": {
-    "@llamaindex/core": "workspace:^0.2.3",
-    "@llamaindex/env": "workspace:^0.1.11"
-  },
-  "dependencies": {
-    "magic-bytes.js": "^1.10.0"
   }
 }

packages/cloud/src/utils.ts

@@ -1,3 +0,0 @@
-export async function sleep(ms: number): Promise<void> {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}

packages/cloud/tsconfig.json

@@ -8,17 +8,8 @@
     "moduleResolution": "Bundler",
     "skipLibCheck": true,
     "strict": true,
-    "lib": ["DOM", "ESNext"],
-    "types": []
+    "lib": ["DOM", "ESNext"]
   },
   "include": ["./src"],
-  "exclude": ["node_modules"],
-  "references": [
-    {
-      "path": "../core/tsconfig.json"
-    },
-    {
-      "path": "../env/tsconfig.json"
-    }
-  ]
+  "exclude": ["node_modules"]
 }

packages/community/CHANGELOG.md

@@ -1,30 +1,5 @@
 # @llamaindex/community
 
-## 0.0.37
-
-### Patch Changes
-
-- Updated dependencies [2cd1383]
-  - @llamaindex/core@0.2.3
-
-## 0.0.36
-
-### Patch Changes
-
-- Updated dependencies [749b43a]
-  - @llamaindex/core@0.2.2
-
-## 0.0.35
-
-### Patch Changes
-
-- Updated dependencies [ac07e3c]
-- Updated dependencies [70ccb4a]
-- Updated dependencies [1a6137b]
-- Updated dependencies [ac07e3c]
-  - @llamaindex/core@0.2.1
-  - @llamaindex/env@0.1.11
-
 ## 0.0.34
 
 ### Patch Changes

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.37",
+  "version": "0.0.34",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/core/CHANGELOG.md

@@ -1,35 +1,5 @@
 # @llamaindex/core
 
-## 0.2.3
-
-### Patch Changes
-
-- 2cd1383: refactor: align `response-synthesizers` & `chat-engine` module
-
-  - builtin event system
-  - correct class extends
-  - aligin APIs, naming with llama-index python
-  - move stream out of first parameter to second parameter for the better tyep checking
-  - remove JSONQueryEngine in `@llamaindex/experimental`, as the code quality is not satisify and we will bring it back later
-
-## 0.2.2
-
-### Patch Changes
-
-- 749b43a: fix: clip embedding transform function
-
-## 0.2.1
-
-### Patch Changes
-
-- ac07e3c: fix: replace instanceof check with `.type` check
-- 70ccb4a: Allow arbitrary types in workflow's StartEvent and StopEvent
-- ac07e3c: fix: add `console.warn` when import dual module
-- Updated dependencies [ac07e3c]
-- Updated dependencies [1a6137b]
-- Updated dependencies [ac07e3c]
-  - @llamaindex/env@0.1.11
-
 ## 0.2.0
 
 ### Minor Changes

packages/core/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/core",
   "type": "module",
-  "version": "0.2.3",
+  "version": "0.2.0",
   "description": "LlamaIndex Core Module",
   "exports": {
     "./node-parser": {
@@ -157,48 +157,6 @@
         "types": "./dist/workflow/index.d.ts",
         "default": "./dist/workflow/index.js"
       }
-    },
-    "./memory": {
-      "require": {
-        "types": "./dist/memory/index.d.cts",
-        "default": "./dist/memory/index.cjs"
-      },
-      "import": {
-        "types": "./dist/memory/index.d.ts",
-        "default": "./dist/memory/index.js"
-      },
-      "default": {
-        "types": "./dist/memory/index.d.ts",
-        "default": "./dist/memory/index.js"
-      }
-    },
-    "./storage/chat-store": {
-      "require": {
-        "types": "./dist/storage/chat-store/index.d.cts",
-        "default": "./dist/storage/chat-store/index.cjs"
-      },
-      "import": {
-        "types": "./dist/storage/chat-store/index.d.ts",
-        "default": "./dist/storage/chat-store/index.js"
-      },
-      "default": {
-        "types": "./dist/storage/chat-store/index.d.ts",
-        "default": "./dist/storage/chat-store/index.js"
-      }
-    },
-    "./response-synthesizers": {
-      "require": {
-        "types": "./dist/response-synthesizers/index.d.cts",
-        "default": "./dist/response-synthesizers/index.cjs"
-      },
-      "import": {
-        "types": "./dist/response-synthesizers/index.d.ts",
-        "default": "./dist/response-synthesizers/index.js"
-      },
-      "default": {
-        "types": "./dist/response-synthesizers/index.d.ts",
-        "default": "./dist/response-synthesizers/index.js"
-      }
     }
   },
   "files": [
@@ -224,7 +182,6 @@
   "dependencies": {
     "@llamaindex/env": "workspace:*",
     "@types/node": "^22.5.1",
-    "magic-bytes.js": "^1.10.0",
     "zod": "^3.23.8"
   }
 }

packages/core/src/embeddings/base.ts

@@ -23,34 +23,23 @@ export abstract class BaseEmbedding extends TransformComponent {
   embedBatchSize = DEFAULT_EMBED_BATCH_SIZE;
   embedInfo?: EmbeddingInfo;
 
-  protected constructor(
-    transformFn?: (
-      nodes: BaseNode[],
-      options?: BaseEmbeddingOptions,
-    ) => Promise<BaseNode[]>,
-  ) {
-    if (transformFn) {
-      super(transformFn);
-    } else {
-      super(
-        async (
-          nodes: BaseNode[],
-          options?: BaseEmbeddingOptions,
-        ): Promise<BaseNode[]> => {
-          const texts = nodes.map((node) =>
-            node.getContent(MetadataMode.EMBED),
-          );
+  constructor() {
+    super(
+      async (
+        nodes: BaseNode[],
+        options?: BaseEmbeddingOptions,
+      ): Promise<BaseNode[]> => {
+        const texts = nodes.map((node) => node.getContent(MetadataMode.EMBED));
 
-          const embeddings = await this.getTextEmbeddingsBatch(texts, options);
+        const embeddings = await this.getTextEmbeddingsBatch(texts, options);
 
-          for (let i = 0; i < nodes.length; i++) {
-            nodes[i]!.embedding = embeddings[i];
-          }
+        for (let i = 0; i < nodes.length; i++) {
+          nodes[i]!.embedding = embeddings[i];
+        }
 
-          return nodes;
-        },
-      );
-    }
+        return nodes;
+      },
+    );
   }
 
   similarity(

packages/core/src/embeddings/index.ts

@@ -1,5 +1,4 @@
 export { BaseEmbedding, batchEmbeddings } from "./base";
 export type { BaseEmbeddingOptions, EmbeddingInfo } from "./base";
-export { MultiModalEmbedding } from "./muti-model";
 export { truncateMaxTokens } from "./tokenizer";
 export { DEFAULT_SIMILARITY_TOP_K, SimilarityType, similarity } from "./utils";

packages/core/src/embeddings/muti-model.ts

@@ -1,81 +0,0 @@
-import type { MessageContentDetail } from "../llms";
-import {
-  ImageNode,
-  MetadataMode,
-  ModalityType,
-  splitNodesByType,
-  type BaseNode,
-  type ImageType,
-} from "../schema";
-import { extractImage, extractSingleText } from "../utils";
-import {
-  BaseEmbedding,
-  batchEmbeddings,
-  type BaseEmbeddingOptions,
-} from "./base";
-
-/*
- * Base class for Multi Modal embeddings.
- */
-export abstract class MultiModalEmbedding extends BaseEmbedding {
-  abstract getImageEmbedding(images: ImageType): Promise<number[]>;
-
-  protected constructor() {
-    super(
-      async (
-        nodes: BaseNode[],
-        options?: BaseEmbeddingOptions,
-      ): Promise<BaseNode[]> => {
-        const nodeMap = splitNodesByType(nodes);
-        const imageNodes = nodeMap[ModalityType.IMAGE] ?? [];
-        const textNodes = nodeMap[ModalityType.TEXT] ?? [];
-
-        const embeddings = await batchEmbeddings(
-          textNodes.map((node) => node.getContent(MetadataMode.EMBED)),
-          this.getTextEmbeddings.bind(this),
-          this.embedBatchSize,
-          options,
-        );
-        for (let i = 0; i < textNodes.length; i++) {
-          textNodes[i]!.embedding = embeddings[i];
-        }
-
-        const imageEmbeddings = await batchEmbeddings(
-          imageNodes.map((n) => (n as ImageNode).image),
-          this.getImageEmbeddings.bind(this),
-          this.embedBatchSize,
-          options,
-        );
-        for (let i = 0; i < imageNodes.length; i++) {
-          imageNodes[i]!.embedding = imageEmbeddings[i];
-        }
-
-        return nodes;
-      },
-    );
-  }
-
-  /**
-   * Optionally override this method to retrieve multiple image embeddings in a single request
-   * @param images
-   */
-  async getImageEmbeddings(images: ImageType[]): Promise<number[][]> {
-    return Promise.all(
-      images.map((imgFilePath) => this.getImageEmbedding(imgFilePath)),
-    );
-  }
-
-  async getQueryEmbedding(
-    query: MessageContentDetail,
-  ): Promise<number[] | null> {
-    const image = extractImage(query);
-    if (image) {
-      return await this.getImageEmbedding(image);
-    }
-    const text = extractSingleText(query);
-    if (text) {
-      return await this.getTextEmbedding(text);
-    }
-    return null;
-  }
-}

packages/core/src/global/settings/callback-manager.ts

@@ -6,13 +6,8 @@ import type {
   ToolCall,
   ToolOutput,
 } from "../../llms";
-import type { QueryEndEvent, QueryStartEvent } from "../../query-engine";
-import type {
-  SynthesizeEndEvent,
-  SynthesizeStartEvent,
-} from "../../response-synthesizers";
 import { TextNode } from "../../schema";
-import { EventCaller, getEventCaller } from "../../utils";
+import { EventCaller, getEventCaller } from "../../utils/event-caller";
 import type { UUID } from "../type";
 
 export type LLMStartEvent = {
@@ -65,10 +60,6 @@ export interface LlamaIndexEventMaps {
   "chunking-end": ChunkingEndEvent;
   "node-parsing-start": NodeParsingStartEvent;
   "node-parsing-end": NodeParsingEndEvent;
-  "query-start": QueryStartEvent;
-  "query-end": QueryEndEvent;
-  "synthesize-start": SynthesizeStartEvent;
-  "synthesize-end": SynthesizeEndEvent;
 }
 
 export class LlamaIndexCustomEvent<T = any> extends CustomEvent<T> {

packages/core/src/indices/prompt-helper.ts

@@ -1,13 +1,10 @@
 import { type Tokenizer, tokenizers } from "@llamaindex/env";
 import {
   DEFAULT_CHUNK_OVERLAP_RATIO,
-  DEFAULT_CHUNK_SIZE,
   DEFAULT_CONTEXT_WINDOW,
   DEFAULT_NUM_OUTPUTS,
   DEFAULT_PADDING,
-  Settings,
 } from "../global";
-import type { LLMMetadata } from "../llms";
 import { SentenceSplitter } from "../node-parser";
 import type { PromptTemplate } from "../prompts";
 
@@ -136,29 +133,4 @@ export class PromptHelper {
     const combinedStr = textChunks.join("\n\n");
     return textSplitter.splitText(combinedStr);
   }
-
-  static fromLLMMetadata(
-    metadata: LLMMetadata,
-    options?: {
-      chunkOverlapRatio?: number;
-      chunkSizeLimit?: number;
-      tokenizer?: Tokenizer;
-      separator?: string;
-    },
-  ) {
-    const {
-      chunkOverlapRatio = DEFAULT_CHUNK_OVERLAP_RATIO,
-      chunkSizeLimit = DEFAULT_CHUNK_SIZE,
-      tokenizer = Settings.tokenizer,
-      separator = " ",
-    } = options ?? {};
-    return new PromptHelper({
-      contextWindow: metadata.contextWindow,
-      numOutput: metadata.maxTokens ?? DEFAULT_NUM_OUTPUTS,
-      chunkOverlapRatio,
-      chunkSizeLimit,
-      tokenizer,
-      separator,
-    });
-  }
 }

packages/core/src/memory/base.ts

@@ -1,62 +0,0 @@
-import { Settings } from "../global";
-import type { ChatMessage, MessageContent } from "../llms";
-import { type BaseChatStore, SimpleChatStore } from "../storage/chat-store";
-import { extractText } from "../utils";
-
-export const DEFAULT_TOKEN_LIMIT_RATIO = 0.75;
-export const DEFAULT_CHAT_STORE_KEY = "chat_history";
-
-/**
- * A ChatMemory is used to keep the state of back and forth chat messages
- */
-export abstract class BaseMemory<
-  AdditionalMessageOptions extends object = object,
-> {
-  abstract getMessages(
-    input?: MessageContent | undefined,
-  ):
-    | ChatMessage<AdditionalMessageOptions>[]
-    | Promise<ChatMessage<AdditionalMessageOptions>[]>;
-  abstract getAllMessages():
-    | ChatMessage<AdditionalMessageOptions>[]
-    | Promise<ChatMessage<AdditionalMessageOptions>[]>;
-  abstract put(messages: ChatMessage<AdditionalMessageOptions>): void;
-  abstract reset(): void;
-
-  protected _tokenCountForMessages(messages: ChatMessage[]): number {
-    if (messages.length === 0) {
-      return 0;
-    }
-
-    const tokenizer = Settings.tokenizer;
-    const str = messages.map((m) => extractText(m.content)).join(" ");
-    return tokenizer.encode(str).length;
-  }
-}
-
-export abstract class BaseChatStoreMemory<
-  AdditionalMessageOptions extends object = object,
-> extends BaseMemory<AdditionalMessageOptions> {
-  protected constructor(
-    public chatStore: BaseChatStore<AdditionalMessageOptions> = new SimpleChatStore<AdditionalMessageOptions>(),
-    public chatStoreKey: string = DEFAULT_CHAT_STORE_KEY,
-  ) {
-    super();
-  }
-
-  getAllMessages(): ChatMessage<AdditionalMessageOptions>[] {
-    return this.chatStore.getMessages(this.chatStoreKey);
-  }
-
-  put(messages: ChatMessage<AdditionalMessageOptions>) {
-    this.chatStore.addMessage(this.chatStoreKey, messages);
-  }
-
-  set(messages: ChatMessage<AdditionalMessageOptions>[]) {
-    this.chatStore.setMessages(this.chatStoreKey, messages);
-  }
-
-  reset() {
-    this.chatStore.deleteMessages(this.chatStoreKey);
-  }
-}

packages/core/src/memory/chat-memory-buffer.ts

@@ -1,65 +0,0 @@
-import { Settings } from "../global";
-import type { ChatMessage, LLM, MessageContent } from "../llms";
-import { type BaseChatStore } from "../storage/chat-store";
-import { BaseChatStoreMemory, DEFAULT_TOKEN_LIMIT_RATIO } from "./base";
-
-type ChatMemoryBufferOptions<AdditionalMessageOptions extends object = object> =
-  {
-    tokenLimit?: number | undefined;
-    chatStore?: BaseChatStore<AdditionalMessageOptions> | undefined;
-    chatStoreKey?: string | undefined;
-    chatHistory?: ChatMessage<AdditionalMessageOptions>[] | undefined;
-    llm?: LLM<object, AdditionalMessageOptions> | undefined;
-  };
-
-export class ChatMemoryBuffer<
-  AdditionalMessageOptions extends object = object,
-> extends BaseChatStoreMemory<AdditionalMessageOptions> {
-  tokenLimit: number;
-
-  constructor(
-    options?: Partial<ChatMemoryBufferOptions<AdditionalMessageOptions>>,
-  ) {
-    super(options?.chatStore, options?.chatStoreKey);
-
-    const llm = options?.llm ?? Settings.llm;
-    const contextWindow = llm.metadata.contextWindow;
-    this.tokenLimit =
-      options?.tokenLimit ??
-      Math.ceil(contextWindow * DEFAULT_TOKEN_LIMIT_RATIO);
-
-    if (options?.chatHistory) {
-      this.chatStore.setMessages(this.chatStoreKey, options.chatHistory);
-    }
-  }
-
-  getMessages(
-    input?: MessageContent | undefined,
-    initialTokenCount: number = 0,
-  ) {
-    const messages = this.getAllMessages();
-
-    if (initialTokenCount > this.tokenLimit) {
-      throw new Error("Initial token count exceeds token limit");
-    }
-
-    let messageCount = messages.length;
-    let currentMessages = messages.slice(-messageCount);
-    let tokenCount = this._tokenCountForMessages(messages) + initialTokenCount;
-
-    while (tokenCount > this.tokenLimit && messageCount > 1) {
-      messageCount -= 1;
-      if (messages.at(-messageCount)!.role === "assistant") {
-        messageCount -= 1;
-      }
-      currentMessages = messages.slice(-messageCount);
-      tokenCount =
-        this._tokenCountForMessages(currentMessages) + initialTokenCount;
-    }
-
-    if (tokenCount > this.tokenLimit && messageCount <= 0) {
-      return [];
-    }
-    return messages.slice(-messageCount);
-  }
-}

packages/core/src/memory/index.ts

@@ -1,3 +0,0 @@
-export { BaseMemory } from "./base";
-export { ChatMemoryBuffer } from "./chat-memory-buffer";
-export { ChatSummaryMemoryBuffer } from "./summary-memory";

packages/core/src/query-engine/base.ts

@@ -1,9 +1,5 @@
-import { randomUUID } from "@llamaindex/env";
-import { Settings } from "../global";
 import type { MessageContent } from "../llms";
-import { PromptMixin } from "../prompts";
-import { EngineResponse } from "../schema";
-import { wrapEventCaller } from "../utils";
+import { EngineResponse, type NodeWithScore } from "../schema";
 
 /**
  * @link https://docs.llamaindex.ai/en/stable/api_reference/schema/?h=querybundle#llama_index.core.schema.QueryBundle
@@ -18,37 +14,16 @@ export type QueryBundle = {
 
 export type QueryType = string | QueryBundle;
 
-export type QueryFn = (
-  strOrQueryBundle: QueryType,
-  stream?: boolean,
-) => Promise<AsyncIterable<EngineResponse> | EngineResponse>;
-
-export abstract class BaseQueryEngine extends PromptMixin {
-  protected constructor(protected readonly _query: QueryFn) {
-    super();
-  }
-
+export interface BaseQueryEngine {
   query(
     strOrQueryBundle: QueryType,
     stream: true,
   ): Promise<AsyncIterable<EngineResponse>>;
   query(strOrQueryBundle: QueryType, stream?: false): Promise<EngineResponse>;
-  @wrapEventCaller
-  async query(
+
+  synthesize?(
     strOrQueryBundle: QueryType,
-    stream = false,
-  ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
-    const id = randomUUID();
-    const callbackManager = Settings.callbackManager;
-    callbackManager.dispatchEvent("query-start", {
-      id,
-      query: strOrQueryBundle,
-    });
-    const response = await this._query(strOrQueryBundle, stream);
-    callbackManager.dispatchEvent("query-end", {
-      id,
-      response,
-    });
-    return response;
-  }
+    nodes: NodeWithScore[],
+    additionalSources?: Iterator<NodeWithScore>,
+  ): Promise<EngineResponse>;
 }

packages/core/src/query-engine/index.ts

@@ -1,2 +1 @@
-export { BaseQueryEngine, type QueryBundle, type QueryType } from "./base";
-export type { QueryEndEvent, QueryStartEvent } from "./type";
+export type { BaseQueryEngine, QueryBundle, QueryType } from "./base";

packages/core/src/query-engine/type.ts

@@ -1,12 +0,0 @@
-import { EngineResponse } from "../schema";
-import type { QueryType } from "./base";
-
-export type QueryStartEvent = {
-  id: string;
-  query: QueryType;
-};
-
-export type QueryEndEvent = {
-  id: string;
-  response: EngineResponse | AsyncIterable<EngineResponse>;
-};

packages/core/src/response-synthesizers/base-synthesizer.ts

@@ -1,58 +0,0 @@
-import { randomUUID } from "@llamaindex/env";
-import { Settings } from "../global";
-import { PromptHelper } from "../indices";
-import type { LLM, MessageContent } from "../llms";
-import { PromptMixin } from "../prompts";
-import { EngineResponse, type NodeWithScore } from "../schema";
-import type { SynthesizeQuery } from "./type";
-
-export type BaseSynthesizerOptions = {
-  llm?: LLM;
-  promptHelper?: PromptHelper;
-};
-
-export abstract class BaseSynthesizer extends PromptMixin {
-  llm: LLM;
-  promptHelper: PromptHelper;
-
-  protected constructor(options: Partial<BaseSynthesizerOptions>) {
-    super();
-    this.llm = options.llm ?? Settings.llm;
-    this.promptHelper =
-      options.promptHelper ?? PromptHelper.fromLLMMetadata(this.llm.metadata);
-  }
-
-  protected abstract getResponse(
-    query: MessageContent,
-    textChunks: NodeWithScore[],
-    stream: boolean,
-  ): Promise<EngineResponse | AsyncIterable<EngineResponse>>;
-
-  synthesize(
-    query: SynthesizeQuery,
-    stream: true,
-  ): Promise<AsyncIterable<EngineResponse>>;
-  synthesize(query: SynthesizeQuery, stream?: false): Promise<EngineResponse>;
-  async synthesize(
-    query: SynthesizeQuery,
-    stream = false,
-  ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
-    const callbackManager = Settings.callbackManager;
-    const id = randomUUID();
-    callbackManager.dispatchEvent("synthesize-start", { id, query });
-    let response: EngineResponse | AsyncIterable<EngineResponse>;
-    if (query.nodes.length === 0) {
-      if (stream) {
-        response = EngineResponse.fromResponse("Empty Response", true);
-      } else {
-        response = EngineResponse.fromResponse("Empty Response", false);
-      }
-    } else {
-      const queryMessage: MessageContent =
-        typeof query.query === "string" ? query.query : query.query.query;
-      response = await this.getResponse(queryMessage, query.nodes, stream);
-    }
-    callbackManager.dispatchEvent("synthesize-end", { id, query, response });
-    return response;
-  }
-}

packages/core/src/response-synthesizers/index.ts

@@ -1,10 +0,0 @@
-export {
-  BaseSynthesizer,
-  type BaseSynthesizerOptions,
-} from "./base-synthesizer";
-export { getResponseSynthesizer, type ResponseMode } from "./factory";
-export type {
-  SynthesizeEndEvent,
-  SynthesizeQuery,
-  SynthesizeStartEvent,
-} from "./type";

packages/core/src/response-synthesizers/type.ts

@@ -1,19 +0,0 @@
-import type { QueryType } from "../query-engine";
-import { EngineResponse, type NodeWithScore } from "../schema";
-
-export type SynthesizeQuery = {
-  query: QueryType;
-  nodes: NodeWithScore[];
-  additionalSourceNodes?: NodeWithScore[];
-};
-
-export type SynthesizeStartEvent = {
-  id: string;
-  query: SynthesizeQuery;
-};
-
-export type SynthesizeEndEvent = {
-  id: string;
-  query: SynthesizeQuery;
-  response: EngineResponse | AsyncIterable<EngineResponse>;
-};

packages/core/src/schema/node.ts

@@ -437,16 +437,9 @@ export function splitNodesByType(nodes: BaseNode[]): NodesByType {
 
   for (const node of nodes) {
     let type: ModalityType;
-    if (
-      node.type === ObjectType.IMAGE ||
-      node.type === ObjectType.IMAGE_DOCUMENT
-    ) {
+    if (node instanceof ImageNode) {
       type = ModalityType.IMAGE;
-    } else if (
-      node.type === ObjectType.TEXT ||
-      node.type === ObjectType.DOCUMENT ||
-      node.type === ObjectType.INDEX
-    ) {
+    } else if (node instanceof TextNode) {
       type = ModalityType.TEXT;
     } else {
       throw new Error(`Unknown node type: ${node.type}`);
@@ -472,36 +465,28 @@ export function buildNodeFromSplits(
   };
 
   textSplits.forEach((textChunk, i) => {
-    if (
-      doc.type === ObjectType.IMAGE ||
-      doc.type === ObjectType.IMAGE_DOCUMENT
-    ) {
-      const imageDoc = doc as ImageNode;
+    if (doc instanceof ImageDocument) {
       const imageNode = new ImageNode({
-        id_: idGenerator(i, imageDoc),
+        id_: idGenerator(i, doc),
         text: textChunk,
-        image: imageDoc.image,
-        embedding: imageDoc.embedding,
-        excludedEmbedMetadataKeys: [...imageDoc.excludedEmbedMetadataKeys],
-        excludedLlmMetadataKeys: [...imageDoc.excludedLlmMetadataKeys],
-        metadataSeparator: imageDoc.metadataSeparator,
-        textTemplate: imageDoc.textTemplate,
+        image: doc.image,
+        embedding: doc.embedding,
+        excludedEmbedMetadataKeys: [...doc.excludedEmbedMetadataKeys],
+        excludedLlmMetadataKeys: [...doc.excludedLlmMetadataKeys],
+        metadataSeparator: doc.metadataSeparator,
+        textTemplate: doc.textTemplate,
         relationships: { ...relationships },
       });
       nodes.push(imageNode);
-    } else if (
-      doc.type === ObjectType.DOCUMENT ||
-      doc.type === ObjectType.TEXT
-    ) {
-      const textDoc = doc as TextNode;
+    } else if (doc instanceof Document || doc instanceof TextNode) {
       const node = new TextNode({
-        id_: idGenerator(i, textDoc),
+        id_: idGenerator(i, doc),
         text: textChunk,
-        embedding: textDoc.embedding,
-        excludedEmbedMetadataKeys: [...textDoc.excludedEmbedMetadataKeys],
-        excludedLlmMetadataKeys: [...textDoc.excludedLlmMetadataKeys],
-        metadataSeparator: textDoc.metadataSeparator,
-        textTemplate: textDoc.textTemplate,
+        embedding: doc.embedding,
+        excludedEmbedMetadataKeys: [...doc.excludedEmbedMetadataKeys],
+        excludedLlmMetadataKeys: [...doc.excludedLlmMetadataKeys],
+        metadataSeparator: doc.metadataSeparator,
+        textTemplate: doc.textTemplate,
         relationships: { ...relationships },
       });
       nodes.push(node);

packages/core/src/storage/chat-store/base-chat-store.ts

@@ -1,19 +0,0 @@
-import type { ChatMessage } from "../../llms";
-
-export abstract class BaseChatStore<
-  AdditionalMessageOptions extends object = object,
-> {
-  abstract setMessages(
-    key: string,
-    messages: ChatMessage<AdditionalMessageOptions>[],
-  ): void;
-  abstract getMessages(key: string): ChatMessage<AdditionalMessageOptions>[];
-  abstract addMessage(
-    key: string,
-    message: ChatMessage<AdditionalMessageOptions>,
-    idx?: number,
-  ): void;
-  abstract deleteMessages(key: string): void;
-  abstract deleteMessage(key: string, idx: number): void;
-  abstract getKeys(): IterableIterator<string>;
-}

packages/core/src/storage/chat-store/index.ts

@@ -1,2 +0,0 @@
-export { BaseChatStore } from "./base-chat-store";
-export { SimpleChatStore } from "./simple-chat-store";

packages/core/src/storage/chat-store/simple-chat-store.ts

@@ -1,43 +0,0 @@
-import type { ChatMessage } from "../../llms";
-import { BaseChatStore } from "./base-chat-store";
-
-export class SimpleChatStore<
-  AdditionalMessageOptions extends object = object,
-> extends BaseChatStore<AdditionalMessageOptions> {
-  #store = new Map<string, ChatMessage<AdditionalMessageOptions>[]>();
-  setMessages(key: string, messages: ChatMessage<AdditionalMessageOptions>[]) {
-    this.#store.set(key, messages);
-  }
-
-  getMessages(key: string) {
-    return this.#store.get(key) ?? [];
-  }
-
-  addMessage(
-    key: string,
-    message: ChatMessage<AdditionalMessageOptions>,
-    idx?: number,
-  ) {
-    const messages = this.#store.get(key) ?? [];
-    if (idx === undefined) {
-      messages.push(message);
-    } else {
-      messages.splice(idx, 0, message);
-    }
-    this.#store.set(key, messages);
-  }
-
-  deleteMessages(key: string) {
-    this.#store.delete(key);
-  }
-
-  deleteMessage(key: string, idx: number) {
-    const messages = this.#store.get(key) ?? [];
-    messages.splice(idx, 1);
-    this.#store.set(key, messages);
-  }
-
-  getKeys() {
-    return this.#store.keys();
-  }
-}

packages/core/src/utils/index.ts

@@ -1,4 +1,4 @@
-export { EventCaller, getEventCaller, wrapEventCaller } from "./event-caller";
+export { wrapEventCaller } from "./event-caller";
 
 export async function* streamConverter<S, D>(
   stream: AsyncIterable<S>,
@@ -47,12 +47,10 @@ export async function* streamReducer<S, D>(params: {
 export { wrapLLMEvent } from "./wrap-llm-event";
 
 export {
-  createMessageContent,
   extractDataUrlComponents,
   extractImage,
   extractSingleText,
   extractText,
-  imageToDataUrl,
   messagesToHistory,
   toToolDescriptions,
 } from "./llms";

packages/core/src/utils/llms.ts

@@ -1,5 +1,3 @@
-import { fs } from "@llamaindex/env";
-import { filetypemime } from "magic-bytes.js";
 import type {
   ChatMessage,
   MessageContent,
@@ -7,16 +5,8 @@ import type {
   MessageContentTextDetail,
   ToolMetadata,
 } from "../llms";
-import type { BasePromptTemplate } from "../prompts";
 import type { QueryType } from "../query-engine";
 import type { ImageType } from "../schema";
-import {
-  type BaseNode,
-  ImageNode,
-  MetadataMode,
-  ModalityType,
-  splitNodesByType,
-} from "../schema";
 
 /**
  * Extracts just the text whether from
@@ -117,99 +107,3 @@ export function toToolDescriptions(tools: ToolMetadata[]): string {
 
   return JSON.stringify(toolsObj, null, 4);
 }
-
-async function blobToDataUrl(input: Blob) {
-  const buffer = Buffer.from(await input.arrayBuffer());
-  const mimes = filetypemime(buffer);
-  if (mimes.length < 1) {
-    throw new Error("Unsupported image type");
-  }
-  return "data:" + mimes[0] + ";base64," + buffer.toString("base64");
-}
-
-export async function imageToDataUrl(
-  input: ImageType | Uint8Array,
-): Promise<string> {
-  // first ensure, that the input is a Blob
-  if (
-    (input instanceof URL && input.protocol === "file:") ||
-    typeof input === "string"
-  ) {
-    // string or file URL
-    const dataBuffer = await fs.readFile(
-      input instanceof URL ? input.pathname : input,
-    );
-    input = new Blob([dataBuffer]);
-  } else if (!(input instanceof Blob)) {
-    if (input instanceof URL) {
-      throw new Error(`Unsupported URL with protocol: ${input.protocol}`);
-    } else if (input instanceof Uint8Array) {
-      input = new Blob([input]); // convert Uint8Array to Blob
-    } else {
-      throw new Error(`Unsupported input type: ${typeof input}`);
-    }
-  }
-  return await blobToDataUrl(input);
-}
-
-// eslint-disable-next-line max-params
-async function createContentPerModality(
-  prompt: BasePromptTemplate,
-  type: ModalityType,
-  nodes: BaseNode[],
-  extraParams: Record<string, string>,
-  metadataMode: MetadataMode,
-): Promise<MessageContentDetail[]> {
-  switch (type) {
-    case ModalityType.TEXT:
-      return [
-        {
-          type: "text",
-          text: prompt.format({
-            ...extraParams,
-            context: nodes.map((r) => r.getContent(metadataMode)).join("\n\n"),
-          }),
-        },
-      ];
-    case ModalityType.IMAGE:
-      const images: MessageContentDetail[] = await Promise.all(
-        (nodes as ImageNode[]).map(async (node) => {
-          return {
-            type: "image_url",
-            image_url: {
-              url: await imageToDataUrl(node.image),
-            },
-          } satisfies MessageContentDetail;
-        }),
-      );
-      return images;
-    default:
-      return [];
-  }
-}
-
-export async function createMessageContent(
-  prompt: BasePromptTemplate,
-  nodes: BaseNode[],
-  extraParams: Record<string, string> = {},
-  metadataMode: MetadataMode = MetadataMode.NONE,
-): Promise<MessageContentDetail[]> {
-  const content: MessageContentDetail[] = [];
-  const nodeMap = splitNodesByType(nodes);
-  for (const type in nodeMap) {
-    // for each retrieved modality type, create message content
-    const nodes = nodeMap[type as ModalityType];
-    if (nodes) {
-      content.push(
-        ...(await createContentPerModality(
-          prompt,
-          type as ModalityType,
-          nodes,
-          extraParams,
-          metadataMode,
-        )),
-      );
-    }
-  }
-  return content;
-}

packages/core/src/workflow/events.ts

@@ -14,5 +14,5 @@ export type EventTypes<T extends Record<string, any> = any> = new (
   data: T,
 ) => WorkflowEvent<T>;
 
-export class StartEvent<T = string> extends WorkflowEvent<{ input: T }> {}
-export class StopEvent<T = string> extends WorkflowEvent<{ result: T }> {}
+export class StartEvent extends WorkflowEvent<{ input: string }> {}
+export class StopEvent extends WorkflowEvent<{ result: string }> {}

packages/core/src/workflow/workflow.ts

@@ -131,7 +131,7 @@ export class Workflow {
     }
   }
 
-  async run<T = string>(event: StartEvent<T> | string): Promise<StopEvent> {
+  async run(event: StartEvent | string): Promise<StopEvent> {
     // Validate the workflow before running if #validate is true
     if (this.#validate) {
       this.validate();

packages/core/tests/workflow.test.ts

@@ -140,30 +140,4 @@ describe("Workflow", () => {
     expect(result.data.result).toBe("Report generated");
     expect(collectedEvents).toHaveLength(1);
   });
-
-  test("run workflow with object-based StartEvent and StopEvent", async () => {
-    const objectFlow = new Workflow({ verbose: true });
-
-    type Person = { name: string; age: number };
-
-    const processObject = vi.fn(async (_context, ev: StartEvent<Person>) => {
-      const { name, age } = ev.data.input;
-      return new StopEvent({
-        result: { greeting: `Hello ${name}, you are ${age} years old!` },
-      });
-    });
-
-    objectFlow.addStep(StartEvent<Person>, processObject);
-
-    const result = await objectFlow.run(
-      new StartEvent<Person>({
-        input: { name: "Alice", age: 30 },
-      }),
-    );
-
-    expect(processObject).toHaveBeenCalledTimes(1);
-    expect(result.data.result).toEqual({
-      greeting: "Hello Alice, you are 30 years old!",
-    });
-  });
 });

packages/env/CHANGELOG.md

@@ -1,16 +1,5 @@
 # @llamaindex/env
 
-## 0.1.11
-
-### Patch Changes
-
-- ac07e3c: fix: replace instanceof check with `.type` check
-- 1a6137b: feat: experimental support for browser
-
-  If you see bundler issue in next.js edge runtime, please bump to `next@14` latest version.
-
-- ac07e3c: fix: add `console.warn` when import dual module
-
 ## 0.1.10
 
 ### Patch Changes

packages/env/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/env",
   "description": "environment wrapper, supports all JS environment including node, deno, bun, edge runtime, and cloudflare worker",
-  "version": "0.1.11",
+  "version": "0.1.10",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",
@@ -36,10 +36,6 @@
         "types": "./dist/type/index.edge-light.d.ts",
         "default": "./dist/index.edge-light.js"
       },
-      "browser": {
-        "types": "./dist/type/index.browser.d.ts",
-        "default": "./dist/index.browser.js"
-      },
       "import": {
         "types": "./dist/type/index.d.ts",
         "default": "./dist/index.js"

packages/env/src/global-check.ts

@@ -1,31 +0,0 @@
-const glo: any =
-  typeof globalThis !== "undefined"
-    ? globalThis
-    : // @ts-expect-error
-      typeof window !== "undefined"
-      ? // @ts-expect-error
-        window
-      : typeof global !== "undefined"
-        ? global
-        : {};
-
-const importIdentifier = "__ $@llamaindex/env$ __";
-
-if (glo[importIdentifier] === true) {
-  /**
-   * Dear reader of this message. Please take this seriously.
-   *
-   * If you see this message, make sure that you only import one version of llamaindex. In many cases,
-   * your package manager installs two versions of llamaindex that are used by different packages within your project.
-   * Another reason for this message is that some parts of your project use the CJS version of llamaindex
-   * and others use the ESM version of llamaindex.
-   *
-   * This often leads to issues that are hard to debug. We often need to perform constructor checks,
-   * e.g. `node instanceof TextNode`. If you imported different versions of llamaindex, it is impossible for us to
-   * do the constructor checks anymore - which might break the functionality of your application.
-   */
-  console.error(
-    "llamaindex was already imported. This breaks constructor checks and will lead to issues!",
-  );
-}
-glo[importIdentifier] = true;

packages/env/src/index.browser.ts

@@ -1,16 +0,0 @@
-/**
- * Web environment polyfill.
- *
- * @module
- */
-import "./global-check.js";
-export * from "./web-polyfill.js";
-
-export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js";
-
-// @ts-expect-error
-if (typeof window === "undefined") {
-  console.warn(
-    "You are not in a browser environment. This module is not supposed to be used in a non-browser environment.",
-  );
-}

packages/env/src/index.edge-light.ts

@@ -3,7 +3,6 @@
  *
  * @module
  */
-import "./global-check.js";
-export * from "./node-polyfill.js";
+export * from "./polyfill.js";
 
 export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js";

packages/env/src/index.ts

@@ -18,8 +18,7 @@ import path from "node:path";
 import { Readable } from "node:stream";
 import { fileURLToPath } from "node:url";
 import { createWriteStream, fs } from "./fs/node.js";
-import "./global-check.js";
-import type { SHA256 } from "./node-polyfill.js";
+import type { SHA256 } from "./polyfill.js";
 
 export function createSHA256(): SHA256 {
   const hash = createHash("sha256");
@@ -34,12 +33,7 @@ export function createSHA256(): SHA256 {
 }
 
 export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/node.js";
-export {
-  AsyncLocalStorage,
-  CustomEvent,
-  getEnv,
-  setEnvs,
-} from "./utils/index.js";
+export { AsyncLocalStorage, CustomEvent, getEnv, setEnvs } from "./utils.js";
 export {
   createWriteStream,
   EOL,

packages/env/src/index.workerd.ts

@@ -5,9 +5,9 @@
  *
  * @module
  */
-import { INTERNAL_ENV } from "./utils/index.js";
+import { INTERNAL_ENV } from "./utils.js";
 
-export * from "./node-polyfill.js";
+export * from "./polyfill.js";
 
 export function getEnv(name: string): string | undefined {
   return INTERNAL_ENV[name];

packages/env/src/polyfill.ts

@@ -46,9 +46,4 @@ export function randomUUID(): string {
   return crypto.randomUUID();
 }
 
-export {
-  AsyncLocalStorage,
-  CustomEvent,
-  getEnv,
-  setEnvs,
-} from "./utils/index.js";
+export { AsyncLocalStorage, CustomEvent, getEnv, setEnvs } from "./utils.js";

packages/env/src/utils/index.web.ts

@@ -1,45 +0,0 @@
-// DO NOT EXPOSE THIS VARIABLE TO PUBLIC, IT IS USED INTERNALLY FOR BROWSER ENVIRONMENT
-export const INTERNAL_ENV: Record<string, string> = {};
-
-export function setEnvs(envs: object): void {
-  Object.assign(INTERNAL_ENV, envs);
-}
-
-export function getEnv(name: string): string | undefined {
-  if (INTERNAL_ENV[name]) {
-    return INTERNAL_ENV[name];
-  }
-}
-
-// Web doesn't have AsyncLocalStorage and there's no alternative way to implement it
-// Wait for https://github.com/tc39/proposal-async-context
-export class AsyncLocalStorage<T> {
-  #store: T = null!;
-  static bind<Func extends (...args: any[]) => any>(fn: Func): Func {
-    return fn;
-  }
-
-  static snapshot(): <R, TArgs extends any[]>(
-    fn: (...args: TArgs) => R,
-    ...args: TArgs
-  ) => R {
-    return (cb: any, ...args: any[]) => cb(...args);
-  }
-
-  getStore() {
-    return this.#store;
-  }
-
-  run<R>(store: T, cb: () => R): R {
-    this.#store = store;
-    if (cb.constructor.name === "AsyncFunction") {
-      console.warn("AsyncLocalStorage is not supported in the web environment");
-      console.warn("Please note that some features may not work as expected");
-    }
-    return cb();
-  }
-}
-
-const defaultCustomEvent = (globalThis as any).CustomEvent;
-
-export { defaultCustomEvent as CustomEvent };

packages/env/src/web-polyfill.ts

@@ -1,54 +0,0 @@
-/**
- * Polyfill implementation for `@llamaindex/env`.
- *
- * The code should be compatible with any JS runtime.
- *
- * Sometimes you should overwrite the polyfill with a native implementation.
- *
- * @module
- */
-import { Sha256 } from "@aws-crypto/sha256-js";
-import pathe from "pathe";
-import { fs } from "./fs/memory.js";
-
-export { fs, pathe as path };
-
-export interface SHA256 {
-  update(data: string | Uint8Array): void;
-  // to base64
-  digest(): string;
-}
-
-export const EOL = "\n";
-
-export function ok(value: unknown, message?: string): asserts value {
-  if (!value) {
-    const error = Error(message);
-    error.name = "AssertionError";
-    error.message = message ?? "The expression evaluated to a falsy value.";
-    throw error;
-  }
-}
-
-export function createSHA256(): SHA256 {
-  const sha256 = new Sha256();
-  return {
-    update(data: string | Uint8Array): void {
-      sha256.update(data);
-    },
-    digest() {
-      return globalThis.btoa(sha256.digestSync().toString());
-    },
-  };
-}
-
-export function randomUUID(): string {
-  return crypto.randomUUID();
-}
-
-export {
-  AsyncLocalStorage,
-  CustomEvent,
-  getEnv,
-  setEnvs,
-} from "./utils/index.web.js";

packages/experimental/CHANGELOG.md

@@ -1,38 +1,5 @@
 # @llamaindex/experimental
 
-## 0.0.81
-
-### Patch Changes
-
-- 2cd1383: refactor: align `response-synthesizers` & `chat-engine` module
-
-  - builtin event system
-  - correct class extends
-  - aligin APIs, naming with llama-index python
-  - move stream out of first parameter to second parameter for the better tyep checking
-  - remove JSONQueryEngine in `@llamaindex/experimental`, as the code quality is not satisify and we will bring it back later
-
-- Updated dependencies [2cd1383]
-- Updated dependencies [5c4badb]
-  - llamaindex@0.6.3
-
-## 0.0.80
-
-### Patch Changes
-
-- Updated dependencies [749b43a]
-  - llamaindex@0.6.2
-
-## 0.0.79
-
-### Patch Changes
-
-- Updated dependencies [fbd5e01]
-- Updated dependencies [6b70c54]
-- Updated dependencies [1a6137b]
-- Updated dependencies [85c2e19]
-  - llamaindex@0.6.1
-
 ## 0.0.78
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.81",
+  "version": "0.0.78",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",

packages/experimental/src/engines/query/JSONQueryEngine.ts

@@ -0,0 +1,211 @@
+import jsonpath from "jsonpath";
+
+import { EngineResponse } from "llamaindex";
+
+import { serviceContextFromDefaults, type ServiceContext } from "llamaindex";
+
+import type {
+  QueryEngine,
+  QueryEngineParamsNonStreaming,
+  QueryEngineParamsStreaming,
+} from "llamaindex";
+
+import {
+  defaultJsonPathPrompt,
+  defaultResponseSynthesizePrompt,
+  type JSONPathPrompt,
+  type ResponseSynthesisPrompt,
+} from "./prompt.js";
+
+export type JSONSchemaType = Record<string, unknown>;
+
+function removeExtraQuotes(expr: string) {
+  let startIndex = 0;
+  let endIndex = expr.length;
+
+  // Trim the leading backticks and single quotes
+  while (
+    startIndex < endIndex &&
+    (expr[startIndex] === "`" || expr[startIndex] === "'")
+  ) {
+    startIndex++;
+  }
+
+  // Trim the trailing backticks and single quotes
+  while (
+    endIndex > startIndex &&
+    (expr[endIndex - 1] === "`" || expr[endIndex - 1] === "'")
+  ) {
+    endIndex--;
+  }
+
+  // Return the trimmed substring
+  return expr.substring(startIndex, endIndex);
+}
+
+export const defaultOutputProcessor = async ({
+  llmOutput,
+  jsonValue,
+}: {
+  llmOutput: string;
+  jsonValue: JSONSchemaType;
+}): Promise<Record<string, unknown>[]> => {
+  const expressions = llmOutput
+    .split(",")
+    .map((expr) => removeExtraQuotes(expr.trim()));
+
+  const results: Record<string, unknown>[] = [];
+
+  for (const expression of expressions) {
+    // get the key for example content from $.content
+    const key = expression.split(".").pop();
+
+    try {
+      const datums = jsonpath.query(jsonValue, expression);
+
+      if (!key) throw new Error(`Invalid JSON Path: ${expression}`);
+
+      for (const datum of datums) {
+        // in case there is a filter like [?(@.username=='simon')] without a key ie: $..comments[?(@.username=='simon').content]
+        if (key.includes("==")) {
+          results.push(datum);
+          continue;
+        }
+
+        results.push({
+          [key]: datum,
+        });
+      }
+    } catch (err) {
+      throw new Error(`Invalid JSON Path: ${expression}`);
+    }
+  }
+
+  return results;
+};
+
+type OutputProcessor = typeof defaultOutputProcessor;
+
+/**
+ * A JSON query engine that uses JSONPath to query a JSON object.
+ */
+export class JSONQueryEngine implements QueryEngine {
+  jsonValue: JSONSchemaType;
+  jsonSchema: JSONSchemaType;
+  serviceContext: ServiceContext;
+  outputProcessor: OutputProcessor;
+  verbose: boolean;
+  jsonPathPrompt: JSONPathPrompt;
+  synthesizeResponse: boolean;
+  responseSynthesisPrompt: ResponseSynthesisPrompt;
+
+  constructor(init: {
+    jsonValue: JSONSchemaType;
+    jsonSchema: JSONSchemaType;
+    serviceContext?: ServiceContext;
+    jsonPathPrompt?: JSONPathPrompt;
+    outputProcessor?: OutputProcessor;
+    synthesizeResponse?: boolean;
+    responseSynthesisPrompt?: ResponseSynthesisPrompt;
+    verbose?: boolean;
+  }) {
+    this.jsonValue = init.jsonValue;
+    this.jsonSchema = init.jsonSchema;
+    this.serviceContext = init.serviceContext ?? serviceContextFromDefaults({});
+    this.jsonPathPrompt = init.jsonPathPrompt ?? defaultJsonPathPrompt;
+    this.outputProcessor = init.outputProcessor ?? defaultOutputProcessor;
+    this.verbose = init.verbose ?? false;
+    this.synthesizeResponse = init.synthesizeResponse ?? true;
+    this.responseSynthesisPrompt =
+      init.responseSynthesisPrompt ?? defaultResponseSynthesizePrompt;
+  }
+
+  getPrompts(): Record<string, unknown> {
+    return {
+      jsonPathPrompt: this.jsonPathPrompt,
+      responseSynthesisPrompt: this.responseSynthesisPrompt,
+    };
+  }
+
+  updatePrompts(prompts: {
+    jsonPathPrompt?: JSONPathPrompt;
+    responseSynthesisPrompt?: ResponseSynthesisPrompt;
+  }): void {
+    if (prompts.jsonPathPrompt) {
+      this.jsonPathPrompt = prompts.jsonPathPrompt;
+    }
+    if (prompts.responseSynthesisPrompt) {
+      this.responseSynthesisPrompt = prompts.responseSynthesisPrompt;
+    }
+  }
+
+  getPromptModules(): Record<string, unknown> {
+    return {};
+  }
+
+  getSchemaContext(): string {
+    return JSON.stringify(this.jsonSchema);
+  }
+
+  query(
+    params: QueryEngineParamsStreaming,
+  ): Promise<AsyncIterable<EngineResponse>>;
+  query(params: QueryEngineParamsNonStreaming): Promise<EngineResponse>;
+  async query(
+    params: QueryEngineParamsStreaming | QueryEngineParamsNonStreaming,
+  ): Promise<EngineResponse | AsyncIterable<EngineResponse>> {
+    const { query, stream } = params;
+
+    if (stream) {
+      throw new Error("Streaming is not supported");
+    }
+
+    const schema = this.getSchemaContext();
+
+    const { text: jsonPathResponse } = await this.serviceContext.llm.complete({
+      prompt: this.jsonPathPrompt({ query, schema }),
+    });
+
+    if (this.verbose) {
+      console.log(
+        `> JSONPath Instructions:\n\`\`\`\n${jsonPathResponse}\n\`\`\`\n`,
+      );
+    }
+
+    const jsonPathOutput = await this.outputProcessor({
+      llmOutput: jsonPathResponse,
+      jsonValue: this.jsonValue,
+    });
+
+    if (this.verbose) {
+      console.log(`> JSONPath Output: ${jsonPathOutput}\n`);
+    }
+
+    let responseStr;
+
+    if (this.synthesizeResponse) {
+      responseStr = await this.serviceContext.llm.complete({
+        prompt: this.responseSynthesisPrompt({
+          query,
+          jsonSchema: schema,
+          jsonPath: jsonPathResponse,
+          jsonPathValue: JSON.stringify(jsonPathOutput),
+        }),
+      });
+
+      responseStr = responseStr.text;
+    } else {
+      responseStr = JSON.stringify(jsonPathOutput);
+    }
+
+    const responseMetadata = {
+      jsonPathResponse,
+    };
+
+    const response = EngineResponse.fromResponse(responseStr, false);
+
+    response.metadata = responseMetadata;
+
+    return response;
+  }
+}

packages/experimental/src/engines/query/index.ts

@@ -0,0 +1 @@
+export * from "./JSONQueryEngine.js";

packages/experimental/src/engines/query/prompt.ts

@@ -0,0 +1,36 @@
+export const defaultJsonPathPrompt = ({
+  query,
+  schema,
+}: {
+  query: string;
+  schema: string;
+}) => `
+We have provided a JSON schema below:
+${schema}
+Given a task, respond with a JSON Path query that can retrieve data from a JSON value that matches the schema.
+Task: ${query}
+JSONPath: 
+`;
+
+export type JSONPathPrompt = typeof defaultJsonPathPrompt;
+
+export const defaultResponseSynthesizePrompt = ({
+  query,
+  jsonSchema,
+  jsonPath,
+  jsonPathValue,
+}: {
+  query: string;
+  jsonSchema: string;
+  jsonPath: string;
+  jsonPathValue: string;
+}) => `
+Given a query, synthesize a response to satisfy the query using the JSON results. Only include details that are relevant to the query. If you don't know the answer, then say that.
+JSON Schema: ${jsonSchema}
+JSON Path: ${jsonPath}
+Value at path: ${jsonPathValue}
+Query: ${query}
+Response: 
+`;
+
+export type ResponseSynthesisPrompt = typeof defaultResponseSynthesizePrompt;

packages/experimental/src/index.ts

@@ -0,0 +1 @@
+export * from "./engines/query/index.js";

packages/extism-tools/README.md

@@ -0,0 +1,24 @@
+## Extism Tools
+
+### Prerequisites for Development
+
+- [Extism PDK](https://github.com/extism/js-pdk?tab=readme-ov-file#linux-macos)
+
+### Build WASM files
+
+```bash
+pnpm run build
+```
+
+### Run WASM files in Node.js using Extism SDK (https://github.com/extism/js-sdk)
+
+```bash
+cd examples
+pnpm run test:wiki
+```
+
+### Run WASM files in Python using Extism SDK (https://github.com/extism/python-sdk)
+
+```bash
+python examples/wasm/wiki.py
+```

packages/extism-tools/bin/compile.js

@@ -0,0 +1,31 @@
+// needed as extism-js doesn't support compiling multiple files
+import { execSync } from "child_process";
+import { mkdirSync, readdirSync } from "fs";
+
+const WASM_SRC_FOLDER = "wasm";
+const WASM_OUTPUT_FOLDER = "dist/wasm";
+
+// get list of tools from files (except index.d.ts)
+const tools = readdirSync(WASM_SRC_FOLDER)
+  .filter((file) => !file.includes("index.d.ts"))
+  .map((file) => file.split(".")[0]);
+
+// create dist/wasm folder if it doesn't exist using fs
+try {
+  mkdirSync(WASM_OUTPUT_FOLDER, { recursive: true });
+} catch (error) {
+  console.error("Error creating dist/wasm folder:", error.message);
+  process.exit(1);
+}
+
+// loop through each tool, compile it to wasm using extism-js
+tools.forEach((tool) => {
+  try {
+    execSync(
+      `extism-js ${WASM_SRC_FOLDER}/${tool}.js -i ${WASM_SRC_FOLDER}/index.d.ts -o ${WASM_OUTPUT_FOLDER}/${tool}.wasm`,
+    );
+  } catch (error) {
+    console.error(`Error compiling module ${tool}:`, error.message);
+    process.exit(1);
+  }
+});

packages/extism-tools/examples/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "@llamaindex/extism-tools-examples",
+  "private": true,
+  "version": "1.0.0",
+  "type": "module",
+  "scripts": {
+    "test:wiki": "tsx ./src/wiki.ts",
+    "test:todo": "tsx ./src/todo.ts"
+  },
+  "dependencies": {
+    "@llamaindex/extism-tools": "workspace:*",
+    "llamaindex": "workspace:*"
+  },
+  "devDependencies": {
+    "tsx": "^4.19.0"
+  }
+}

packages/extism-tools/examples/src/todo.ts

@@ -0,0 +1,13 @@
+import { ExtismToolFactory } from "@llamaindex/extism-tools";
+import { OpenAI, OpenAIAgent, Settings } from "llamaindex";
+
+async function main() {
+  const TodoTool = await ExtismToolFactory.createToolClass("todo");
+  const todoTool = new TodoTool();
+  Settings.llm = new OpenAI();
+  const agent = new OpenAIAgent({ tools: [todoTool] });
+  const result = await agent.chat({ message: "Get first todo" });
+  console.log(result.message);
+}
+
+void main();

packages/extism-tools/examples/src/wiki.py

@@ -0,0 +1,34 @@
+import extism
+import json
+from os.path import join, dirname
+
+
+def read_local_wasm(file_name):
+    path = join(dirname(__file__), file_name)  # Change this to your wasm file path
+    with open(path, "rb") as wasm_file:
+        return wasm_file.read()
+
+
+def _manifest(file_name):
+    wasm = read_local_wasm(file_name)
+    return {
+        "wasm": [{"data": wasm}],
+        "allowed_hosts": ["*.wikipedia.org"],
+    }
+
+
+manifest = _manifest("wiki.wasm")
+with extism.Plugin(manifest, wasi=True) as plugin:
+    metadata = plugin.call(
+        "getMetadata",
+        "",
+        parse=lambda output: json.loads(bytes(output).decode("utf-8")),
+    )
+    data = plugin.call(
+        "call",
+        json.dumps({"query": "Ho Chi Minh City"}),
+        parse=lambda output: json.loads(bytes(output).decode("utf-8")),
+    )
+
+print(metadata)
+print(data)

packages/extism-tools/examples/src/wiki.ts

@@ -0,0 +1,13 @@
+import { ExtismToolFactory } from "@llamaindex/extism-tools";
+import { OpenAI, OpenAIAgent, Settings } from "llamaindex";
+
+async function main() {
+  const WikiTool = await ExtismToolFactory.createToolClass("wiki");
+  const wikiTool = new WikiTool();
+  Settings.llm = new OpenAI();
+  const agent = new OpenAIAgent({ tools: [wikiTool] });
+  const result = await agent.chat({ message: "Ho Chi Minh City" });
+  console.log(result.message);
+}
+
+void main();

packages/extism-tools/examples/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "./lib",
+    "module": "node16",
+    "moduleResolution": "node16"
+  },
+  "include": ["./src"]
+}

packages/extism-tools/package.json

@@ -0,0 +1,62 @@
+{
+  "name": "@llamaindex/extism-tools",
+  "version": "0.0.1",
+  "license": "MIT",
+  "type": "module",
+  "dependencies": {
+    "@extism/extism": "^2.0.0-rc8",
+    "ajv": "^8.17.1",
+    "@llamaindex/core": "workspace:*",
+    "@llamaindex/env": "workspace:*",
+    "llamaindex": "workspace:*"
+  },
+  "devDependencies": {
+    "@swc/cli": "^0.4.0",
+    "@swc/core": "^1.7.22",
+    "typescript": "^5.5.4",
+    "@types/node": "^22.5.1"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "types": "./dist/index.d.ts",
+  "main": "./dist/cjs/index.js",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      },
+      "require": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/cjs/index.js"
+      }
+    },
+    "./*": {
+      "import": {
+        "types": "./dist/*.d.ts",
+        "default": "./dist/*.js"
+      },
+      "require": {
+        "types": "./dist/*.d.ts",
+        "default": "./dist/cjs/*.js"
+      }
+    }
+  },
+  "files": [
+    "dist",
+    "CHANGELOG.md"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/run-llama/LlamaIndexTS.git",
+    "directory": "packages/extism-tools"
+  },
+  "scripts": {
+    "build": "rm -rf ./dist && pnpm run build:wasm && pnpm run build:esm && pnpm run build:cjs && pnpm run build:type",
+    "build:esm": "swc src -d dist --strip-leading-paths --config-file ../../.swcrc",
+    "build:cjs": "swc src -d dist/cjs --strip-leading-paths --config-file ../../.cjs.swcrc",
+    "build:type": "tsc -p tsconfig.json",
+    "build:wasm": "node bin/compile.js"
+  }
+}

packages/extism-tools/src/ExtismToolFactory.ts

@@ -0,0 +1,82 @@
+import createPlugin, { type Plugin } from "@extism/extism";
+import type { JSONSchemaType } from "ajv";
+import type { BaseToolWithCall, ToolMetadata } from "llamaindex";
+import path from "path";
+import { fileURLToPath } from "url";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+export const WASM_DIRECTORY = path.join(__dirname, "..", "dist", "wasm");
+export const DEFAULT_MAX_HTTP_RESPONSE_BYTES = 100 * 1024 * 1024; // 100 MB
+
+export type ToolParams = Record<string, any>;
+
+export type ToolClassParams = {
+  metadata?: ToolMetadata<JSONSchemaType<ToolParams>>;
+};
+
+export type CreateToolClassParams = {
+  wasmFilename: string;
+  allowedHosts: string[];
+  maxHttpResponseBytes: number;
+  transformResponse: (response: any) => any;
+};
+
+export const createPluginInstance = async (
+  params: Omit<CreateToolClassParams, "transformResponse">,
+): Promise<Plugin> => {
+  const { wasmFilename, allowedHosts, maxHttpResponseBytes } = params;
+  const plugin = await createPlugin(`${WASM_DIRECTORY}/${wasmFilename}`, {
+    useWasi: true,
+    runInWorker: true,
+    allowedHosts,
+    memory: { maxHttpResponseBytes },
+  });
+  return plugin;
+};
+
+export const DEFAULT_TOOL_PARAMS: Omit<CreateToolClassParams, "wasmFilename"> =
+  {
+    allowedHosts: ["*"],
+    maxHttpResponseBytes: DEFAULT_MAX_HTTP_RESPONSE_BYTES,
+    transformResponse: (response: any) => response,
+  };
+
+export class ExtismToolFactory {
+  static async createToolClass(
+    toolName: string,
+    params: Omit<CreateToolClassParams, "wasmFilename"> = DEFAULT_TOOL_PARAMS,
+  ): Promise<new (params?: ToolClassParams) => BaseToolWithCall<ToolParams>> {
+    const config = { ...params, wasmFilename: `${toolName}.wasm` };
+    const plugin = await createPluginInstance(config);
+    try {
+      const wasmMetadata = await plugin.call("getMetadata");
+      if (!wasmMetadata) {
+        throw new Error("The WASM plugin did not return metadata.");
+      }
+      const defaultMetadata = wasmMetadata.json();
+
+      return class implements BaseToolWithCall<ToolParams> {
+        metadata: ToolMetadata<JSONSchemaType<ToolParams>>;
+
+        constructor(params?: ToolClassParams) {
+          this.metadata = params?.metadata || defaultMetadata;
+        }
+
+        async call(input: ToolParams): Promise<string> {
+          const pluginInstance = await createPluginInstance(config);
+          const data = await pluginInstance.call("call", JSON.stringify(input));
+          if (!data) return "No result";
+          const result = config.transformResponse(data.json());
+          await pluginInstance.close();
+          return result;
+        }
+      };
+    } catch (e) {
+      console.error(e);
+      throw new Error("Failed to create Tool instance.");
+    } finally {
+      await plugin.close();
+    }
+  }
+}

packages/extism-tools/src/index.ts

@@ -0,0 +1 @@
+export { ExtismToolFactory } from "./ExtismToolFactory.js";

packages/extism-tools/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "./src",
+    "outDir": "./dist",
+    "tsBuildInfoFile": "./dist/.tsbuildinfo",
+    "emitDeclarationOnly": true,
+    "module": "node16",
+    "moduleResolution": "node16",
+    "skipLibCheck": true,
+    "strict": true,
+    "types": ["node"]
+  },
+  "include": ["./src"],
+  "exclude": ["node_modules"],
+  "references": [
+    {
+      "path": "../env/tsconfig.json"
+    }
+  ]
+}

packages/extism-tools/wasm/index.d.ts

@@ -0,0 +1,4 @@
+declare module "main" {
+  export function getMetadata(): I32;
+  export function call(): I32;
+}

packages/extism-tools/wasm/todo.js

@@ -0,0 +1,34 @@
+function getMetadata() {
+  const metadata = {
+    name: "todo_tool",
+    description: "A tool helps search todo.",
+    parameters: {
+      type: "object",
+      properties: {
+        index: {
+          type: "number",
+          description: "The index of the todo to search for.",
+        },
+      },
+      required: ["index"],
+    },
+  };
+  Host.outputString(JSON.stringify(metadata));
+}
+
+function call() {
+  const params = JSON.parse(Host.inputString());
+  const index = params?.index;
+  if (!index) throw new Error("No index provided");
+  const request = {
+    method: "GET",
+    url: `https://jsonplaceholder.typicode.com/todos/${encodeURIComponent(index)}`,
+  };
+  const response = Http.request(request);
+  if (response.status != 200)
+    throw new Error(`Got non 200 response ${response.status}`);
+
+  Host.outputString(response.body);
+}
+
+module.exports = { getMetadata, call };

packages/extism-tools/wasm/wiki.js

@@ -0,0 +1,33 @@
+function getMetadata() {
+  const metadata = {
+    name: "wikipedia_tool",
+    description: "A tool that uses a query engine to search Wikipedia.",
+    parameters: {
+      type: "object",
+      properties: {
+        query: {
+          type: "string",
+          description: "The query to search for",
+        },
+      },
+      required: ["query"],
+    },
+  };
+  Host.outputString(JSON.stringify(metadata));
+}
+
+function call() {
+  const params = JSON.parse(Host.inputString());
+  const query = params?.query;
+  if (!query) throw new Error("No query provided");
+  const request = {
+    method: "GET",
+    url: `https://en.wikipedia.org/api/rest_v1/page/summary/${encodeURIComponent(query)}`,
+  };
+  const response = Http.request(request);
+  if (response.status != 200)
+    throw new Error(`Got non 200 response ${response.status}`);
+  Host.outputString(response.body);
+}
+
+module.exports = { getMetadata, call };

packages/llamaindex/CHANGELOG.md

@@ -1,65 +1,5 @@
 # llamaindex
 
-## 0.6.3
-
-### Patch Changes
-
-- 2cd1383: refactor: align `response-synthesizers` & `chat-engine` module
-
-  - builtin event system
-  - correct class extends
-  - aligin APIs, naming with llama-index python
-  - move stream out of first parameter to second parameter for the better tyep checking
-  - remove JSONQueryEngine in `@llamaindex/experimental`, as the code quality is not satisify and we will bring it back later
-
-- 5c4badb: Extend JinaAPIEmbedding parameters
-- Updated dependencies [fb36eff]
-- Updated dependencies [d24d3d1]
-- Updated dependencies [2cd1383]
-  - @llamaindex/cloud@0.2.7
-  - @llamaindex/core@0.2.3
-  - @llamaindex/openai@0.1.5
-  - @llamaindex/groq@0.0.4
-
-## 0.6.2
-
-### Patch Changes
-
-- 749b43a: fix: clip embedding transform function
-- Updated dependencies [b42adeb]
-- Updated dependencies [749b43a]
-  - @llamaindex/cloud@0.2.6
-  - @llamaindex/core@0.2.2
-  - @llamaindex/openai@0.1.4
-  - @llamaindex/groq@0.0.3
-
-## 0.6.1
-
-### Patch Changes
-
-- fbd5e01: refactor: move groq as llm package
-- 6b70c54: feat: update JinaAIEmbedding, support embedding v3
-- 1a6137b: feat: experimental support for browser
-
-  If you see bundler issue in next.js edge runtime, please bump to `next@14` latest version.
-
-- 85c2e19: feat: `@llamaindex/cloud` package update
-
-  - Bump to latest openapi schema
-  - Move LlamaParse class from llamaindex, this will allow you use llamaparse in more non-node.js environment
-
-- Updated dependencies [ac07e3c]
-- Updated dependencies [fbd5e01]
-- Updated dependencies [70ccb4a]
-- Updated dependencies [1a6137b]
-- Updated dependencies [85c2e19]
-- Updated dependencies [ac07e3c]
-  - @llamaindex/core@0.2.1
-  - @llamaindex/env@0.1.11
-  - @llamaindex/groq@0.0.2
-  - @llamaindex/cloud@0.2.5
-  - @llamaindex/openai@0.1.3
-
 ## 0.6.0
 
 ### Minor Changes

packages/llamaindex/e2e/.env.ci

@@ -1 +0,0 @@
-POSTGRES_USER=runner

packages/llamaindex/e2e/examples/cloudflare-worker-agent/CHANGELOG.md

@@ -1,30 +1,5 @@
 # @llamaindex/cloudflare-worker-agent-test
 
-## 0.0.65
-
-### Patch Changes
-
-- Updated dependencies [2cd1383]
-- Updated dependencies [5c4badb]
-  - llamaindex@0.6.3
-
-## 0.0.64
-
-### Patch Changes
-
-- Updated dependencies [749b43a]
-  - llamaindex@0.6.2
-
-## 0.0.63
-
-### Patch Changes
-
-- Updated dependencies [fbd5e01]
-- Updated dependencies [6b70c54]
-- Updated dependencies [1a6137b]
-- Updated dependencies [85c2e19]
-  - llamaindex@0.6.1
-
 ## 0.0.62
 
 ### Patch Changes

packages/llamaindex/e2e/examples/cloudflare-worker-agent/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/cloudflare-worker-agent-test",
-  "version": "0.0.65",
+  "version": "0.0.62",
   "type": "module",
   "private": true,
   "scripts": {

packages/llamaindex/e2e/examples/cloudflare-worker-agent/tsconfig.json

@@ -100,8 +100,7 @@
 
     /* Completeness */
     // "skipDefaultLibCheck": true,                      /* Skip type checking .d.ts files that are included with TypeScript. */
-    "skipLibCheck": true /* Skip type checking all .d.ts files. */,
-    "tsBuildInfoFile": "./dist/.tsbuildinfo"
+    "skipLibCheck": true /* Skip type checking all .d.ts files. */
   },
   "exclude": ["test"]
 }

packages/llamaindex/e2e/examples/llama-parse-browser/.gitignore

@@ -1,24 +0,0 @@
-# Logs
-logs
-*.log
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-pnpm-debug.log*
-lerna-debug.log*
-
-node_modules
-dist
-dist-ssr
-*.local
-
-# Editor directories and files
-.vscode/*
-!.vscode/extensions.json
-.idea
-.DS_Store
-*.suo
-*.ntvs*
-*.njsproj
-*.sln
-*.sw?

packages/llamaindex/e2e/examples/llama-parse-browser/CHANGELOG.md

@@ -1,23 +0,0 @@
-# @llamaindex/llama-parse-browser-test
-
-## 0.0.3
-
-### Patch Changes
-
-- Updated dependencies [fb36eff]
-- Updated dependencies [d24d3d1]
-  - @llamaindex/cloud@0.2.7
-
-## 0.0.2
-
-### Patch Changes
-
-- Updated dependencies [b42adeb]
-  - @llamaindex/cloud@0.2.6
-
-## 0.0.1
-
-### Patch Changes
-
-- Updated dependencies [85c2e19]
-  - @llamaindex/cloud@0.2.5

packages/llamaindex/e2e/examples/llama-parse-browser/index.html

@@ -1,13 +0,0 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8" />
-    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
-    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Vite + TS</title>
-  </head>
-  <body>
-    <div id="app"></div>
-    <script type="module" src="/src/main.ts"></script>
-  </body>
-</html>

packages/llamaindex/e2e/examples/llama-parse-browser/package.json

@@ -1,19 +0,0 @@
-{
-  "name": "@llamaindex/llama-parse-browser-test",
-  "private": true,
-  "version": "0.0.3",
-  "type": "module",
-  "scripts": {
-    "dev": "vite",
-    "build": "tsc && vite build",
-    "preview": "vite preview"
-  },
-  "devDependencies": {
-    "typescript": "^5.5.3",
-    "vite": "^5.4.1",
-    "vite-plugin-wasm": "^3.3.0"
-  },
-  "dependencies": {
-    "@llamaindex/cloud": "workspace:*"
-  }
-}

packages/llamaindex/e2e/examples/llama-parse-browser/src/main.ts

@@ -1,10 +0,0 @@
-import { LlamaParseReader } from "@llamaindex/cloud/reader";
-import "./style.css";
-
-new LlamaParseReader();
-
-document.querySelector<HTMLDivElement>("#app")!.innerHTML = `
-  <div>
-    Hello, world!
-  </div>
-`;

packages/llamaindex/e2e/examples/llama-parse-browser/src/style.css

@@ -1,96 +0,0 @@
-:root {
-  font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif;
-  line-height: 1.5;
-  font-weight: 400;
-
-  color-scheme: light dark;
-  color: rgba(255, 255, 255, 0.87);
-  background-color: #242424;
-
-  font-synthesis: none;
-  text-rendering: optimizeLegibility;
-  -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
-}
-
-a {
-  font-weight: 500;
-  color: #646cff;
-  text-decoration: inherit;
-}
-a:hover {
-  color: #535bf2;
-}
-
-body {
-  margin: 0;
-  display: flex;
-  place-items: center;
-  min-width: 320px;
-  min-height: 100vh;
-}
-
-h1 {
-  font-size: 3.2em;
-  line-height: 1.1;
-}
-
-#app {
-  max-width: 1280px;
-  margin: 0 auto;
-  padding: 2rem;
-  text-align: center;
-}
-
-.logo {
-  height: 6em;
-  padding: 1.5em;
-  will-change: filter;
-  transition: filter 300ms;
-}
-.logo:hover {
-  filter: drop-shadow(0 0 2em #646cffaa);
-}
-.logo.vanilla:hover {
-  filter: drop-shadow(0 0 2em #3178c6aa);
-}
-
-.card {
-  padding: 2em;
-}
-
-.read-the-docs {
-  color: #888;
-}
-
-button {
-  border-radius: 8px;
-  border: 1px solid transparent;
-  padding: 0.6em 1.2em;
-  font-size: 1em;
-  font-weight: 500;
-  font-family: inherit;
-  background-color: #1a1a1a;
-  cursor: pointer;
-  transition: border-color 0.25s;
-}
-button:hover {
-  border-color: #646cff;
-}
-button:focus,
-button:focus-visible {
-  outline: 4px auto -webkit-focus-ring-color;
-}
-
-@media (prefers-color-scheme: light) {
-  :root {
-    color: #213547;
-    background-color: #ffffff;
-  }
-  a:hover {
-    color: #747bff;
-  }
-  button {
-    background-color: #f9f9f9;
-  }
-}