test: add e2e test for nextjs/node with tokenizer

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.changeset/odd-insects-vanish.md

@@ -0,0 +1,5 @@
+---
+"@llamaindex/env": patch
+---
+
+Use tiktoken instead of tiktoken/lite and disable WASM tiktoken for non-Node environments

.changeset/pink-keys-kiss.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+include node relationships when converting jsonToDoc

.github/workflows/publish.yml

@@ -31,6 +31,6 @@ jobs:
 
       - name: Publish @llamaindex/core
         run: npx jsr publish --allow-slow-types
-        working-directory: packages/core
+        working-directory: packages/llamaindex
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/publish_github.yml

@@ -26,12 +26,12 @@ jobs:
       - name: Build tarball
         run: |
           pnpm pack
-        working-directory: packages/core
+        working-directory: packages/llamaindex
 
       - name: Create release
         uses: ncipollo/release-action@v1
         with:
-          artifacts: "packages/core/llamaindex-*.tgz"
+          artifacts: "packages/llamaindex/llamaindex-*.tgz"
           name: Release ${{ github.ref }}
-          bodyFile: "packages/core/CHANGELOG.md"
+          bodyFile: "packages/llamaindex/CHANGELOG.md"
           token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -71,7 +71,7 @@ jobs:
       - name: Build
         run: pnpm run build
       - name: Use Build For Examples
-        run: pnpm link ../packages/core/
+        run: pnpm link ../packages/llamaindex/
         working-directory: ./examples
       - name: Run Type Check
         run: pnpm run type-check
@@ -81,9 +81,9 @@ jobs:
         if: failure()
         with:
           name: typecheck-build-dist
-          path: ./packages/core/dist
+          path: ./packages/llamaindex/dist
           if-no-files-found: error
-  e2e-core-examples:
+  e2e-llamaindex-examples:
     strategy:
       fail-fast: false
       matrix:
@@ -91,9 +91,10 @@ jobs:
           - cloudflare-worker-agent
           - nextjs-agent
           - nextjs-edge-runtime
-          - waku-query-engine
+          - nextjs-node-runtime
+    #          - waku-query-engine
     runs-on: ubuntu-latest
-    name: Build Core Example (${{ matrix.packages }})
+    name: Build LlamaIndex Example (${{ matrix.packages }})
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v3
@@ -108,7 +109,7 @@ jobs:
         run: pnpm run build
       - name: Build ${{ matrix.packages }}
         run: pnpm run build
-        working-directory: packages/core/e2e/examples/${{ matrix.packages }}
+        working-directory: packages/llamaindex/e2e/examples/${{ matrix.packages }}
 
   typecheck-examples:
     runs-on: ubuntu-latest
@@ -132,7 +133,7 @@ jobs:
         working-directory: packages/env
       - name: Pack llamaindex
         run: pnpm pack --pack-destination ${{ runner.temp }}
-        working-directory: packages/core
+        working-directory: packages/llamaindex
       - name: Install
         run: npm add ${{ runner.temp }}/*.tgz
         working-directory: ${{ runner.temp }}/examples

CONTRIBUTING.md

@@ -6,7 +6,7 @@ This is a monorepo built with Turborepo
 
 Right now there are two packages of importance:
 
-packages/core which is the main NPM library llamaindex
+packages/llamaindex which is the main NPM library llamaindex
 
 examples is where the demo code lives
 
@@ -41,7 +41,7 @@ To run them, run
 pnpm run test
 ```
 
-To write new test cases write them in [packages/core/src/tests](/packages/core/src/tests)
+To write new test cases write them in [packages/llamaindex/tests](/packages/llamaindex/tests)
 
 We use Jest https://jestjs.io/ to write our test cases. Jest comes with a bunch of built in assertions using the expect function: https://jestjs.io/docs/expect
 
@@ -56,7 +56,7 @@ You can create new demo applications in the apps folder. Just run pnpm init in t
 To install packages for a specific package or demo application, run
 
 ```
-pnpm add [NPM Package] --filter [package or application i.e. core or docs]
+pnpm add [NPM Package] --filter [package or application i.e. llamaindex or docs]
 ```
 
 To install packages for every package or application run

README.md

@@ -194,19 +194,19 @@ Check out our NextJS playground at https://llama-playground.vercel.app/. The sou
 
 ## Core concepts for getting started:
 
-- [Document](/packages/core/src/Node.ts): A document represents a text file, PDF file or other contiguous piece of data.
+- [Document](/packages/llamaindex/src/Node.ts): A document represents a text file, PDF file or other contiguous piece of data.
 
-- [Node](/packages/core/src/Node.ts): The basic data building block. Most commonly, these are parts of the document split into manageable pieces that are small enough to be fed into an embedding model and LLM.
+- [Node](/packages/llamaindex/src/Node.ts): The basic data building block. Most commonly, these are parts of the document split into manageable pieces that are small enough to be fed into an embedding model and LLM.
 
-- [Embedding](/packages/core/src/embeddings/OpenAIEmbedding.ts): Embeddings are sets of floating point numbers which represent the data in a Node. By comparing the similarity of embeddings, we can derive an understanding of the similarity of two pieces of data. One use case is to compare the embedding of a question with the embeddings of our Nodes to see which Nodes may contain the data needed to answer that quesiton. Because the default service context is OpenAI, the default embedding is `OpenAIEmbedding`. If using different models, say through Ollama, use this [Embedding](/packages/core/src/embeddings/OllamaEmbedding.ts) (see all [here](/packages/core/src/embeddings)).
+- [Embedding](/packages/llamaindex/src/embeddings/OpenAIEmbedding.ts): Embeddings are sets of floating point numbers which represent the data in a Node. By comparing the similarity of embeddings, we can derive an understanding of the similarity of two pieces of data. One use case is to compare the embedding of a question with the embeddings of our Nodes to see which Nodes may contain the data needed to answer that quesiton. Because the default service context is OpenAI, the default embedding is `OpenAIEmbedding`. If using different models, say through Ollama, use this [Embedding](/packages/llamaindex/src/embeddings/OllamaEmbedding.ts) (see all [here](/packages/llamaindex/src/embeddings)).
 
-- [Indices](/packages/core/src/indices/): Indices store the Nodes and the embeddings of those nodes. QueryEngines retrieve Nodes from these Indices using embedding similarity.
+- [Indices](/packages/llamaindex/src/indices/): Indices store the Nodes and the embeddings of those nodes. QueryEngines retrieve Nodes from these Indices using embedding similarity.
 
-- [QueryEngine](/packages/core/src/engines/query/RetrieverQueryEngine.ts): Query engines are what generate the query you put in and give you back the result. Query engines generally combine a pre-built prompt with selected Nodes from your Index to give the LLM the context it needs to answer your query. To build a query engine from your Index (recommended), use the [`asQueryEngine`](/packages/core/src/indices/BaseIndex.ts) method on your Index. See all query engines [here](/packages/core/src/engines/query).
+- [QueryEngine](/packages/llamaindex/src/engines/query/RetrieverQueryEngine.ts): Query engines are what generate the query you put in and give you back the result. Query engines generally combine a pre-built prompt with selected Nodes from your Index to give the LLM the context it needs to answer your query. To build a query engine from your Index (recommended), use the [`asQueryEngine`](/packages/llamaindex/src/indices/BaseIndex.ts) method on your Index. See all query engines [here](/packages/llamaindex/src/engines/query).
 
-- [ChatEngine](/packages/core/src/engines/chat/SimpleChatEngine.ts): A ChatEngine helps you build a chatbot that will interact with your Indices. See all chat engines [here](/packages/core/src/engines/chat).
+- [ChatEngine](/packages/llamaindex/src/engines/chat/SimpleChatEngine.ts): A ChatEngine helps you build a chatbot that will interact with your Indices. See all chat engines [here](/packages/llamaindex/src/engines/chat).
 
-- [SimplePrompt](/packages/core/src/Prompt.ts): A simple standardized function call definition that takes in inputs and formats them in a template literal. SimplePrompts can be specialized using currying and combined using other SimplePrompt functions.
+- [SimplePrompt](/packages/llamaindex/src/Prompt.ts): A simple standardized function call definition that takes in inputs and formats them in a template literal. SimplePrompts can be specialized using currying and combined using other SimplePrompt functions.
 
 ## Tips when using in non-Node.js environments
 

apps/docs/CHANGELOG.md

@@ -1,5 +1,26 @@
 # docs
 
+## 0.0.27
+
+### Patch Changes
+
+- Updated dependencies [3c47910]
+- Updated dependencies [ed467a9]
+- Updated dependencies [cba5406]
+  - llamaindex@0.4.1
+
+## 0.0.26
+
+### Patch Changes
+
+- b1a4a74: docs: updated Bedrock Opus region and added a basic README
+- Updated dependencies [436bc41]
+- Updated dependencies [a44e54f]
+- Updated dependencies [a51ed8d]
+- Updated dependencies [d3b635b]
+  - llamaindex@0.4.0
+  - @llamaindex/examples@0.0.5
+
 ## 0.0.25
 
 ### Patch Changes

apps/docs/docs/getting_started/concepts.md

@@ -32,7 +32,7 @@ LlamaIndex.TS help you prepare the knowledge base with a suite of data connector
 
 ![](../_static/concepts/indexing.jpg)
 
-[**Data Loaders**](../modules/data_loader.md):
+[**Data Loaders**](../modules/data_loaders/index.mdx):
 A data connector (i.e. `Reader`) ingest data from different data sources and data formats into a simple `Document` representation (text and simple metadata).
 
 [**Documents / Nodes**](../modules/documents_and_nodes/index.md): A `Document` is a generic container around any data source - for instance, a PDF, an API output, or retrieved data from a database. A `Node` is the atomic unit of data in LlamaIndex and represents a "chunk" of a source `Document`. It's a rich representation that includes metadata and relationships (to other nodes) to enable accurate and expressive retrieval operations.

apps/docs/docs/getting_started/starter_tutorial/retrieval_augmented_generation.mdx

@@ -21,7 +21,7 @@ npm install -D typescript @types/node
 
 Then, check out the [installation](../installation) steps to install LlamaIndex.TS and prepare an OpenAI key.
 
-You can use [other LLMs](../examples/other_llms) via their APIs; if you would prefer to use local models check out our [local LLM example](../../examples/local_llm).
+You can use [other LLMs](../../examples/other_llms) via their APIs; if you would prefer to use local models check out our [local LLM example](../../examples/local_llm).
 
 ## Run queries
 

apps/docs/docs/getting_started/starter_tutorial/structured_data_extraction.mdx

@@ -7,9 +7,9 @@ import CodeSource from "!raw-loader!../../../../../examples/jsonExtract";
 
 # Structured data extraction tutorial
 
-Make sure you have installed LlamaIndex.TS and have an OpenAI key. If you haven't, check out the [installation](installation) guide.
+Make sure you have installed LlamaIndex.TS and have an OpenAI key. If you haven't, check out the [installation](../installation) guide.
 
-You can use [other LLMs](../examples/other_llms) via their APIs; if you would prefer to use local models check out our [local LLM example](../../examples/local_llm).
+You can use [other LLMs](../../examples/other_llms) via their APIs; if you would prefer to use local models check out our [local LLM example](../../examples/local_llm).
 
 ## Set up
 

apps/docs/docs/introduction.md

@@ -35,7 +35,7 @@ For more complex applications, our lower-level APIs allow advanced users to cust
 
 `npm install llamaindex`
 
-Our documentation includes [Installation Instructions](./getting_started/installation.mdx) and a [Starter Tutorial](./getting_started/starter.mdx) to build your first application.
+Our documentation includes [Installation Instructions](./getting_started/installation.mdx) and a [Starter Tutorial](./getting_started/starter_tutorial/retrieval_augmented_generation.mdx) to build your first application.
 
 Once you're up and running, [High-Level Concepts](./getting_started/concepts.md) has an overview of LlamaIndex's modular architecture. For more hands-on practical examples, look through our Examples section on the sidebar.
 

apps/docs/docs/modules/agent/index.md

@@ -18,3 +18,9 @@ LlamaIndex.TS comes with a few built-in agents, but you can also create your own
 ## Examples
 
 - [OpenAI Agent](../../examples/agent.mdx)
+
+## Api References
+
+- [OpenAIAgent](../../api/classes/OpenAIAgent.md)
+- [AnthropicAgent](../../api/classes/AnthropicAgent.md)
+- [ReActAgent](../../api/classes/ReActAgent.md)

apps/docs/docs/modules/data_loader.mdx

@@ -1,68 +0,0 @@
----
-sidebar_position: 4
----
-
-import CodeBlock from "@theme/CodeBlock";
-import CodeSource from "!raw-loader!../../../../examples/readers/src/simple-directory-reader";
-import CodeSource2 from "!raw-loader!../../../../examples/readers/src/custom-simple-directory-reader";
-import CodeSource3 from "!raw-loader!../../../../examples/readers/src/llamaparse";
-import CodeSource4 from "!raw-loader!../../../../examples/readers/src/simple-directory-reader-with-llamaparse.ts";
-
-# Loader
-
-Before you can start indexing your documents, you need to load them into memory.
-
-### SimpleDirectoryReader
-
-[![Open in StackBlitz](https://developer.stackblitz.com/img/open_in_stackblitz.svg)](https://stackblitz.com/github/run-llama/LlamaIndexTS/tree/main/examples/readers?file=src/simple-directory-reader.ts&title=Simple%20Directory%20Reader)
-
-LlamaIndex.TS supports easy loading of files from folders using the `SimpleDirectoryReader` class.
-
-It is a simple reader that reads all files from a directory and its subdirectories.
-
-<CodeBlock language="ts">{CodeSource}</CodeBlock>
-
-Currently, it supports reading `.txt`, `.pdf`, `.csv`, `.md`, `.docx`, `.htm`, `.html`, `.jpg`, `.jpeg`, `.png` and `.gif` files, but support for other file types is planned.
-
-You can override the default reader for all file types, inlcuding unsupported ones, with the `overrideReader` option.
-Additionally, you can override the default reader for specific file types or add support for additional file types with the `fileExtToReader` option.
-Also, you can provide a `defaultReader` as a fallback for files with unsupported extensions. By default it is `TextFileReader`.
-
-SimpleDirectoryReader supports up to 9 concurrent requests. Use the `numWorkers` option to set the number of concurrent requests. By default it runs in sequential mode, i.e. set to 1.
-
-<CodeBlock language="ts" showLineNumbers metastring="{8-12,17-21}">
-  {CodeSource2}
-</CodeBlock>
-
-### LlamaParse
-
-LlamaParse is an API created by LlamaIndex to efficiently parse files, e.g. it's great at converting PDF tables into markdown.
-
-To use it, first login and get an API key from https://cloud.llamaindex.ai. Make sure to store the key as `apiKey` parameter or in the environment variable `LLAMA_CLOUD_API_KEY`.
-
-Then, you can use the `LlamaParseReader` class to local files and convert them into a parsed document that can be used by LlamaIndex.
-See [LlamaParseReader.ts](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/readers/LlamaParseReader.ts#L6) for a list of supported file types:
-
-<CodeBlock language="ts">{CodeSource3}</CodeBlock>
-
-Additional options can be set with the `LlamaParseReader` constructor:
-
-- `resultType` can be set to `markdown`, `text` or `.json`. Defaults to `text`
-- `language` primarly helps with OCR recognition. Defaults to `en`. See [../readers/type.ts](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/readers/type.ts#L20) for a list of supported languages.
-- `parsingInstructions` can help with complicated document structures. See this [LlamaIndex Blog Post](https://www.llamaindex.ai/blog/launching-the-first-genai-native-document-parsing-platform) for an example.
-- `skipDiagonalText` set to true to ignore diagonal text.
-- `invalidateCache` set to true to ignore the LlamaCloud cache. All document are kept in cache for 48hours after the job was completed to avoid processing the same document twice. Can be useful for testing when trying to re-parse the same document with, e.g. different `parsingInstructions`.
-- `gpt4oMode` set to true to use GPT-4o to extract content.
-- `gpt4oApiKey` set the GPT-4o API key. Optional. Lowers the cost of parsing by using your own API key. Your OpenAI account will be charged. Can also be set in the environment variable `LLAMA_CLOUD_GPT4O_API_KEY`.
-- `numWorkers` as in the python version, is set in `SimpleDirectoryReader`. Default is 1.
-
-## LlamaParse with SimpleDirectoryReader
-
-Below a full example of `LlamaParse` integrated in `SimpleDirectoryReader` with additional options.
-
-<CodeBlock language="ts">{CodeSource4}</CodeBlock>
-
-## API Reference
-
-- [SimpleDirectoryReader](../api/classes/SimpleDirectoryReader.md)
-- [LlamaParseReader](../api/classes/LlamaParseReader.md)

apps/docs/docs/modules/data_loaders/_category_.yml

@@ -0,0 +1,2 @@
+label: "Loaders"
+position: 1

apps/docs/docs/modules/data_loaders/index.mdx

@@ -0,0 +1,37 @@
+import CodeBlock from "@theme/CodeBlock";
+import CodeSource from "!raw-loader!../../../../../examples/readers/src/simple-directory-reader";
+import CodeSource2 from "!raw-loader!../../../../../examples/readers/src/custom-simple-directory-reader";
+
+# Loader
+
+Before you can start indexing your documents, you need to load them into memory.
+
+## SimpleDirectoryReader
+
+[![Open in StackBlitz](https://developer.stackblitz.com/img/open_in_stackblitz.svg)](https://stackblitz.com/github/run-llama/LlamaIndexTS/tree/main/examples/readers?file=src/simple-directory-reader.ts&title=Simple%20Directory%20Reader)
+
+LlamaIndex.TS supports easy loading of files from folders using the `SimpleDirectoryReader` class.
+
+It is a simple reader that reads all files from a directory and its subdirectories.
+
+<CodeBlock language="ts">{CodeSource}</CodeBlock>
+
+Currently, it supports reading `.txt`, `.pdf`, `.csv`, `.md`, `.docx`, `.htm`, `.html`, `.jpg`, `.jpeg`, `.png` and `.gif` files, but support for other file types is planned.
+
+You can modify the reader three different ways:
+
+- `overrideReader` overrides the reader for all file types, including unsupported ones.
+- `fileExtToReader` maps a reader to a specific file type. Can override reader for existing file types or add support for new file types.
+- `defaultReader` sets a fallback reader for files with unsupported extensions. By default it is `TextFileReader`.
+
+SimpleDirectoryReader supports up to 9 concurrent requests. Use the `numWorkers` option to set the number of concurrent requests. By default it runs in sequential mode, i.e. set to 1.
+
+### Example
+
+<CodeBlock language="ts" showLineNumbers metastring="{8-12,17-21}">
+  {CodeSource2}
+</CodeBlock>
+
+## API Reference
+
+- [SimpleDirectoryReader](../../api/classes/SimpleDirectoryReader.md)

apps/docs/docs/modules/data_loaders/llama_parse/_category_.yml

@@ -0,0 +1 @@
+label: "LlamaParse"

apps/docs/docs/modules/data_loaders/llama_parse/images.mdx

@@ -0,0 +1,117 @@
+---
+sidebar_position: 2
+---
+
+# Image Retrieval
+
+LlamaParse `json` mode supports extracting any images found in a page object by using the `getImages` function. They are downloaded to a local folder and can then be sent to a multimodal LLM for further processing.
+
+## Usage
+
+We use the `getImages` method to input our array of JSON objects, download the images to a specified folder and get a list of ImageNodes.
+
+```ts
+const reader = new LlamaParseReader();
+const jsonObjs = await reader.loadJson("../data/uber_10q_march_2022.pdf");
+const imageDicts = await reader.getImages(jsonObjs, "images");
+```
+
+### Multimodal Indexing
+
+You can create an index across both text and image nodes by requesting alternative text for the image from a multimodal LLM.
+
+```ts
+import {
+  Document,
+  ImageNode,
+  LlamaParseReader,
+  OpenAI,
+  VectorStoreIndex,
+} from "llamaindex";
+import { createMessageContent } from "llamaindex/synthesizers/utils";
+
+const reader = new LlamaParseReader();
+async function main() {
+  // Load PDF using LlamaParse JSON mode and return an array of json objects
+  const jsonObjs = await reader.loadJson("../data/uber_10q_march_2022.pdf");
+  // Access the first "pages" (=a single parsed file) object in the array
+  const jsonList = jsonObjs[0]["pages"];
+
+  const textDocs = getTextDocs(jsonList);
+  const imageTextDocs = await getImageTextDocs(jsonObjs);
+  const documents = [...textDocs, ...imageTextDocs];
+  // Split text, create embeddings and query the index
+  const index = await VectorStoreIndex.fromDocuments(documents);
+  const queryEngine = index.asQueryEngine();
+  const response = await queryEngine.query({
+    query:
+      "What does the bar graph titled 'Monthly Active Platform Consumers' show?",
+  });
+
+  console.log(response.toString());
+}
+
+main().catch(console.error);
+```
+
+We use two helper functions to create documents from the text and image nodes provided.
+
+#### Text Documents
+
+To create documents from the text nodes of the json object, we just map the needed values to a new `Document` object. In this case we assign the text as text and the page number as metadata.
+
+```ts
+function getTextDocs(jsonList: { text: string; page: number }[]): Document[] {
+  return jsonList.map(
+    (page) => new Document({ text: page.text, metadata: { page: page.page } }),
+  );
+}
+```
+
+#### Image Documents
+
+To create documents from the images, we need to use a multimodal LLM to generate alt text.
+
+For this we create `ImageNodes` and add them as part of our message.
+
+We can use the `createMessageContent` function to simplify this.
+
+```ts
+async function getImageTextDocs(
+  jsonObjs: Record<string, any>[],
+): Promise<Document[]> {
+  const llm = new OpenAI({
+    model: "gpt-4o",
+    temperature: 0.2,
+    maxTokens: 1000,
+  });
+  const imageDicts = await reader.getImages(jsonObjs, "images");
+  const imageDocs = [];
+
+  for (const imageDict of imageDicts) {
+    const imageDoc = new ImageNode({ image: imageDict.path });
+    const prompt = () => `Describe the image as alt text`;
+    const message = await createMessageContent(prompt, [imageDoc]);
+
+    const response = await llm.complete({
+      prompt: message,
+    });
+
+    const doc = new Document({
+      text: response.text,
+      metadata: { path: imageDict.path },
+    });
+    imageDocs.push(doc);
+  }
+
+  return imageDocs;
+}
+```
+
+The returned `imageDocs` have the alt text assigned as text and the image path as metadata.
+
+You can see the full example file [here](https://github.com/run-llama/LlamaIndexTS/blob/main/examples/readers/src/llamaparse-json.ts).
+
+## API Reference
+
+- [LlamaParseReader](../../../api/classes/LlamaParseReader.md)

apps/docs/docs/modules/data_loaders/llama_parse/index.mdx

@@ -0,0 +1,58 @@
+import CodeBlock from "@theme/CodeBlock";
+import CodeSource from "!raw-loader!../../../../../../examples/readers/src/llamaparse";
+import CodeSource2 from "!raw-loader!../../../../../../examples/readers/src/simple-directory-reader-with-llamaparse.ts";
+
+# LlamaParse
+
+LlamaParse is an API created by LlamaIndex to efficiently parse files, e.g. it's great at converting PDF tables into markdown.
+
+To use it, first login and get an API key from https://cloud.llamaindex.ai. Make sure to store the key as `apiKey` parameter or in the environment variable `LLAMA_CLOUD_API_KEY`.
+
+Official documentation for LlamaParse can be found [here](https://docs.cloud.llamaindex.ai/).
+
+## Usage
+
+You can then use the `LlamaParseReader` class to load local files and convert them into a parsed document that can be used by LlamaIndex.
+See [LlamaParseReader.ts](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/llamaindex/src/readers/LlamaParseReader.ts) for a list of supported file types:
+
+<CodeBlock language="ts">{CodeSource}</CodeBlock>
+
+### Params
+
+All options can be set with the `LlamaParseReader` constructor.
+
+They can be divided into two groups.
+
+#### General params:
+
+- `apiKey` is required. Can be set as an environment variable `LLAMA_CLOUD_API_KEY`
+- `checkInterval` is the interval in seconds to check if the parsing is done. Default is `1`.
+- `maxTimeout` is the maximum timout to wait for parsing to finish. Default is `2000`
+- `verbose` shows progress of the parsing. Default is `true`
+- `ignoreErrors` set to false to get errors while parsing. Default is `true` and returns an empty array on error.
+
+#### Advanced params:
+
+- `resultType` can be set to `markdown`, `text` or `json`. Defaults to `text`. More information about `json` mode on the next pages.
+- `language` primarly helps with OCR recognition. Defaults to `en`. Click [here](../../../api/type-aliases/Language.md) for a list of supported languages.
+- `parsingInstructions?` Optional. Can help with complicated document structures. See this [LlamaIndex Blog Post](https://www.llamaindex.ai/blog/launching-the-first-genai-native-document-parsing-platform) for an example.
+- `skipDiagonalText?` Optional. Set to true to ignore diagonal text. (Text that is not rotated 0, 90, 180 or 270 degrees)
+- `invalidateCache?` Optional. Set to true to ignore the LlamaCloud cache. All document are kept in cache for 48hours after the job was completed to avoid processing the same document twice. Can be useful for testing when trying to re-parse the same document with, e.g. different `parsingInstructions`.
+- `doNotCache?` Optional. Set to true to not cache the document.
+- `fastMode?` Optional. Set to true to use the fast mode. This mode will skip OCR of images, and table/heading reconstruction. Note: Non-compatible with `gpt4oMode`.
+- `doNotUnrollColumns?` Optional. Set to true to keep the text according to document layout. Reduce reconstruction accuracy, and LLM's/embedings performances in most cases.
+- `pageSeperator?` Optional. The page seperator to use. Defaults is `\\n---\\n`.
+- `gpt4oMode` set to true to use GPT-4o to extract content. Default is `false`.
+- `gpt4oApiKey?` Optional. Set the GPT-4o API key. Lowers the cost of parsing by using your own API key. Your OpenAI account will be charged. Can also be set in the environment variable `LLAMA_CLOUD_GPT4O_API_KEY`.
+- `numWorkers` as in the python version, is set in `SimpleDirectoryReader`. Default is 1.
+
+### LlamaParse with SimpleDirectoryReader
+
+Below a full example of `LlamaParse` integrated in `SimpleDirectoryReader` with additional options.
+
+<CodeBlock language="ts">{CodeSource2}</CodeBlock>
+
+## API Reference
+
+- [SimpleDirectoryReader](../../../api/classes/SimpleDirectoryReader.md)
+- [LlamaParseReader](../../../api/classes/LlamaParseReader.md)

apps/docs/docs/modules/data_loaders/llama_parse/json_mode.mdx

@@ -0,0 +1,59 @@
+---
+sidebar_position: 1
+---
+
+# JSON Mode
+
+In JSON mode, LlamaParse will return a data structure representing the parsed object.
+
+## Usage
+
+For Json mode, you need to use `loadJson`. The `resultType` is automatically set with this method. Currently it can't be used with `SimpleDirectoryReader`.
+More information about indexing the results on the next page.
+
+```ts
+const reader = new LlamaParseReader();
+async function main() {
+  // Load the file and return an array of json objects
+  const jsonObjs = await reader.loadJson("../data/uber_10q_march_2022.pdf");
+  // Access the first "pages" (=a single parsed file) object in the array
+  const jsonList = jsonObjs[0]["pages"];
+  // Further process the jsonList object as needed.
+}
+```
+
+### Output
+
+The result format of the response, written to `jsonObjs` in the example, follows this structure:
+
+```json
+{
+    "pages": [
+        ..page objects..
+    ],
+    "job_metadata": {
+        "credits_used": int,
+        "credits_max": int,
+        "job_credits_usage": int,
+        "job_pages": int,
+        "job_is_cache_hit": boolean
+    },
+    "job_id": string ,
+    "file_path": string,
+    }
+}
+```
+
+#### Page objects
+
+Within page objects, the following keys may be present depending on your document.
+
+- `page`: The page number of the document.
+- `text`: The text extracted from the page.
+- `md`: The markdown version of the extracted text.
+- `images`: Any images extracted from the page.
+- `items`: An array of heading, text and table objects in the order they appear on the page.
+
+## API Reference
+
+- [LlamaParseReader](../../../api/classes/LlamaParseReader.md)

apps/docs/docs/modules/documents_and_nodes/index.md

@@ -14,5 +14,5 @@ document = new Document({ text: "text", metadata: { key: "val" } });
 
 ## API Reference
 
-- [Document](../api/classes/Document.md)
-- [TextNode](../api/classes/TextNode.md)
+- [Document](../../api/classes/Document.md)
+- [TextNode](../../api/classes/TextNode.md)

apps/docs/docs/modules/documents_and_nodes/metadata_extraction.md

@@ -43,3 +43,10 @@ async function main() {
 
 main().then(() => console.log("done"));
 ```
+
+## API Reference
+
+- [SummaryExtractor](../../api/classes/SummaryExtractor.md)
+- [QuestionsAnsweredExtractor](../../api/classes/QuestionsAnsweredExtractor.md)
+- [TitleExtractor](../../api/classes/TitleExtractor.md)
+- [KeywordExtractor](../../api/classes/KeywordExtractor.md)

apps/docs/docs/modules/embeddings/available_embeddings/deepinfra.md

@@ -77,3 +77,7 @@ main();
 ```
 
 For questions or feedback, please contact us at [feedback@deepinfra.com](mailto:feedback@deepinfra.com)
+
+## API Reference
+
+- [DeepInfraEmbedding](../../../api/classes/DeepInfraEmbedding.md)

apps/docs/docs/modules/embeddings/available_embeddings/gemini.md

@@ -31,3 +31,7 @@ Settings.embedModel = new GeminiEmbedding({
   model: GEMINI_MODEL.GEMINI_PRO_LATEST,
 });
 ```
+
+## API Reference
+
+- [GeminiEmbedding](../../../api/classes/GeminiEmbedding.md)

apps/docs/docs/modules/embeddings/available_embeddings/huggingface.md

@@ -32,3 +32,7 @@ Settings.embedModel = new HuggingFaceEmbedding({
   quantized: false,
 });
 ```
+
+## API Reference
+
+- [HuggingFaceEmbedding](../../../api/classes/HuggingFaceEmbedding.md)

apps/docs/docs/modules/embeddings/available_embeddings/jinaai.md

@@ -19,3 +19,7 @@ const results = await queryEngine.query({
   query,
 });
 ```
+
+## API Reference
+
+- [JinaAIEmbedding](../../../api/classes/JinaAIEmbedding.md)

apps/docs/docs/modules/embeddings/available_embeddings/mistral.md

@@ -22,3 +22,7 @@ const results = await queryEngine.query({
   query,
 });
 ```
+
+## API Reference
+
+- [MistralAIEmbedding](../../../api/classes/MistralAIEmbedding.md)

apps/docs/docs/modules/embeddings/available_embeddings/ollama.md

@@ -27,3 +27,7 @@ const results = await queryEngine.query({
   query,
 });
 ```
+
+## API Reference
+
+- [OllamaEmbedding](../../../api/classes/OllamaEmbedding.md)

apps/docs/docs/modules/embeddings/available_embeddings/openai.md

@@ -19,3 +19,7 @@ const results = await queryEngine.query({
   query,
 });
 ```
+
+## API Reference
+
+- [OpenAIEmbedding](../../../api/classes/OpenAIEmbedding.md)

apps/docs/docs/modules/embeddings/available_embeddings/together.md

@@ -21,3 +21,7 @@ const results = await queryEngine.query({
   query,
 });
 ```
+
+## API Reference
+
+- [TogetherEmbedding](../../../api/classes/TogetherEmbedding.md)

apps/docs/docs/modules/evaluation/modules/correctness.md

@@ -56,3 +56,7 @@ console.log(
 ```bash
 the response is not correct with a score of 2.5
 ```
+
+## API Reference
+
+- [CorrectnessEvaluator](../../../api/classes/CorrectnessEvaluator.md)

apps/docs/docs/modules/evaluation/modules/faithfulness.md

@@ -76,3 +76,7 @@ console.log(`the response is ${result.passing ? "faithful" : "not faithful"}`);
 ```bash
 the response is faithful
 ```
+
+## API Reference
+
+- [FaithfulnessEvaluator](../../../api/classes/FaithfulnessEvaluator.md)

apps/docs/docs/modules/evaluation/modules/relevancy.md

@@ -70,3 +70,7 @@ console.log(`the response is ${result.passing ? "relevant" : "not relevant"}`);
 ```bash
 the response is relevant
 ```
+
+## API Reference
+
+- [RelevancyEvaluator](../../../api/classes/RelevancyEvaluator.md)

apps/docs/docs/modules/ingestion_pipeline/index.md

@@ -97,3 +97,7 @@ async function main() {
 
 main().catch(console.error);
 ```
+
+## API Reference
+
+- [IngestionPipeline](../../api/classes/IngestionPipeline.md)

apps/docs/docs/modules/ingestion_pipeline/transformations.md

@@ -4,9 +4,9 @@ A transformation is something that takes a list of nodes as an input, and return
 
 Currently, the following components are Transformation objects:
 
-- [SimpleNodeParser](../api/classes/SimpleNodeParser.md)
+- [SimpleNodeParser](../../api/classes/SimpleNodeParser.md)
 - [MetadataExtractor](../documents_and_nodes/metadata_extraction.md)
-- Embeddings
+- [Embeddings](../embeddings/index.md)
 
 ## Usage Pattern
 

apps/docs/docs/modules/llms/available_llms/anthropic.md

@@ -63,3 +63,7 @@ async function main() {
   console.log(response.response);
 }
 ```
+
+## API Reference
+
+- [Anthropic](../../../api/classes/Anthropic.md)

apps/docs/docs/modules/llms/available_llms/azure.md

@@ -74,3 +74,7 @@ async function main() {
   console.log(response.response);
 }
 ```
+
+## API Reference
+
+- [OpenAI](../../../api/classes/OpenAI.md)

apps/docs/docs/modules/llms/available_llms/bedrock.md

@@ -23,7 +23,8 @@ ANTHROPIC_CLAUDE_2 = "anthropic.claude-v2";
 ANTHROPIC_CLAUDE_2_1 = "anthropic.claude-v2:1";
 ANTHROPIC_CLAUDE_3_SONNET = "anthropic.claude-3-sonnet-20240229-v1:0";
 ANTHROPIC_CLAUDE_3_HAIKU = "anthropic.claude-3-haiku-20240307-v1:0";
-ANTHROPIC_CLAUDE_3_OPUS = "anthropic.claude-3-opus-20240229-v1:0"; // Not currently on Bedrock
+ANTHROPIC_CLAUDE_3_OPUS = "anthropic.claude-3-opus-20240229-v1:0"; // available on us-west-2
+ANTHROPIC_CLAUDE_3_5_SONNET = "anthropic.claude-3-5-sonnet-20240620-v1:0";
 ```
 
 Sonnet, Haiku and Opus are multimodal, image_url only supports base64 data url format, e.g. `data:image/jpeg;base64,SGVsbG8sIFdvcmxkIQ==`

apps/docs/docs/modules/llms/available_llms/deepinfra.md

@@ -81,3 +81,7 @@ async function main() {
 ## Feedback
 
 If you have any feedback, please reach out to us at [feedback@deepinfra.com](mailto:feedback@deepinfra.com)
+
+## API Reference
+
+- [DeepInfra](../../../api/classes/DeepInfra)

apps/docs/docs/modules/llms/available_llms/fireworks.md

@@ -59,3 +59,7 @@ async function main() {
 
 main().catch(console.error);
 ```
+
+## API Reference
+
+- [FireworksLLM](../../../api/classes/FireworksLLM.md)

apps/docs/docs/modules/llms/available_llms/gemini.md

@@ -99,3 +99,7 @@ async function main() {
   console.log(response.response);
 }
 ```
+
+## API Reference
+
+- [Gemini](../../../api/classes/Gemini.md)

apps/docs/docs/modules/llms/available_llms/groq.mdx

@@ -50,3 +50,7 @@ const results = await queryEngine.query({
 <CodeBlock language="ts" showLineNumbers>
   {CodeSource}
 </CodeBlock>
+
+## API Reference
+
+- [Groq](../../../api/classes/Groq.md)

apps/docs/docs/modules/llms/available_llms/llama2.md

@@ -89,3 +89,8 @@ async function main() {
   console.log(response.response);
 }
 ```
+
+## API Reference
+
+- [LlamaDeuce](../../../api/variables/LlamaDeuce.md)
+- [DeuceChatStrategy](../../../api/variables/DeuceChatStrategy.md)

apps/docs/docs/modules/llms/available_llms/mistral.md

@@ -66,3 +66,7 @@ async function main() {
   console.log(response.response);
 }
 ```
+
+## API Reference
+
+- [MistralAI](../../../api/classes/MistralAI.md)

apps/docs/docs/modules/llms/available_llms/ollama.md

@@ -71,3 +71,7 @@ async function main() {
   console.log(response.response);
 }
 ```
+
+## API Reference
+
+- [Ollama](../../../api/classes/Ollama.md)

apps/docs/docs/modules/llms/available_llms/openai.md

@@ -67,3 +67,7 @@ async function main() {
   console.log(response.response);
 }
 ```
+
+## API Reference
+
+- [OpenAI](../../../api/classes/OpenAI.md)

apps/docs/docs/modules/llms/available_llms/portkey.md

@@ -68,3 +68,7 @@ async function main() {
   console.log(response.response);
 }
 ```
+
+## API Reference
+
+- [Portkey](../../../api/classes/Portkey.md)

apps/docs/docs/modules/llms/available_llms/together.md

@@ -66,3 +66,7 @@ async function main() {
   console.log(response.response);
 }
 ```
+
+## API Reference
+
+- [TogetherLLM](../../../api/classes/TogetherLLM.md)

apps/docs/docs/modules/llms/index.md

@@ -32,4 +32,4 @@ For local LLMs, currently we recommend the use of [Ollama](./available_llms/olla
 
 ## API Reference
 
-- [OpenAI](../api/classes/OpenAI.md)
+- [OpenAI](../../api/classes/OpenAI.md)

apps/docs/docs/modules/node_parser.md

@@ -95,3 +95,4 @@ The output metadata will be something like:
 
 - [SimpleNodeParser](../api/classes/SimpleNodeParser.md)
 - [SentenceSplitter](../api/classes/SentenceSplitter.md)
+- [MarkdownNodeParser](../api/classes/MarkdownNodeParser.md)

apps/docs/docs/modules/node_postprocessors/cohere_reranker.md

@@ -65,3 +65,7 @@ const queryEngine = index.asQueryEngine({
 // log the response
 const response = await queryEngine.query("Where did the author grown up?");
 ```
+
+## API Reference
+
+- [CohereRerank](../../api/classes/CohereRerank.md)

apps/docs/docs/modules/node_postprocessors/index.md

@@ -103,3 +103,7 @@ const processor = new SimilarityPostprocessor({
 
 const filteredNodes = processor.postprocessNodes(nodes);
 ```
+
+## API Reference
+
+- [SimilarityPostprocessor](../../api/classes/SimilarityPostprocessor.md)

apps/docs/docs/modules/node_postprocessors/jinaai_reranker.md

@@ -69,3 +69,7 @@ const queryEngine = index.asQueryEngine({
 // log the response
 const response = await queryEngine.query("Where did the author grown up?");
 ```
+
+## API Reference
+
+- [JinaAIReranker](../../api/classes/JinaAIReranker.md)

apps/docs/docs/modules/prompt/index.md

@@ -70,3 +70,9 @@ const response = await queryEngine.query({
   query: "What did the author do in college?",
 });
 ```
+
+## API Reference
+
+- [TextQaPrompt](../../api/type-aliases/TextQaPrompt.md)
+- [ResponseSynthesizer](../../api/classes/ResponseSynthesizer.md)
+- [CompactAndRefine](../../api/classes/CompactAndRefine.md)

apps/docs/docs/modules/query_engines/index.md

@@ -38,4 +38,4 @@ You can learn more about Tools by taking a look at the LlamaIndex Python documen
 
 - [RetrieverQueryEngine](../../api/classes/RetrieverQueryEngine.md)
 - [SubQuestionQueryEngine](../../api/classes/SubQuestionQueryEngine.md)
-- [QueryEngineTool](../../api/interfaces/QueryEngineTool.md)
+- [QueryEngineTool](../../api/classes/QueryEngineTool.md)

apps/docs/docs/modules/query_engines/metadata_filtering.md

@@ -151,3 +151,8 @@ async function main() {
 
 main();
 ```
+
+## API Reference
+
+- [VectorStoreIndex](../../api/classes/VectorStoreIndex.md)
+- [ChromaVectorStore](../../api/classes/ChromaVectorStore.md)

apps/docs/docs/modules/query_engines/router_query_engine.md

@@ -165,3 +165,7 @@ async function main() {
 
 main().then(() => console.log("Done"));
 ```
+
+## API Reference
+
+- [RouterQueryEngine](../../api/classes/RouterQueryEngine.md)

apps/docs/docs/modules/storage.md

@@ -23,4 +23,4 @@ const index = await VectorStoreIndex.fromDocuments([document], {
 
 ## API Reference
 
-- [StorageContext](../api/interfaces//StorageContext.md)
+- [StorageContext](../api/interfaces/StorageContext.md)

apps/docs/docs/modules/vector_stores/qdrant.md

@@ -84,3 +84,7 @@ async function main() {
 
 main().catch(console.error);
 ```
+
+## API Reference
+
+- [QdrantVectorStore](../../api/classes/QdrantVectorStore.md)

apps/docs/docusaurus.config.js

@@ -167,7 +167,7 @@ const config = {
     [
       "docusaurus-plugin-typedoc",
       {
-        entryPoints: ["../../packages/core/src/index.ts"],
+        entryPoints: ["../../packages/llamaindex/src/index.ts"],
         tsconfig: "../../tsconfig.json",
         readme: "none",
         sourceLinkTemplate:

apps/docs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docs",
-  "version": "0.0.25",
+  "version": "0.0.27",
   "private": true,
   "scripts": {
     "docusaurus": "docusaurus",
@@ -37,7 +37,7 @@
     "docusaurus-plugin-typedoc": "^1.0.1",
     "typedoc": "^0.25.13",
     "typedoc-plugin-markdown": "^4.0.1",
-    "typescript": "^5.4.5"
+    "typescript": "^5.5.2"
   },
   "browserslist": {
     "production": [

examples/CHANGELOG.md

@@ -1,5 +1,15 @@
 # examples
 
+## 0.0.5
+
+### Patch Changes
+
+- Updated dependencies [436bc41]
+- Updated dependencies [a44e54f]
+- Updated dependencies [a51ed8d]
+- Updated dependencies [d3b635b]
+  - llamaindex@0.4.0
+
 ## 0.0.4
 
 ### Patch Changes

examples/agent/openai.ts

@@ -53,7 +53,7 @@ async function main() {
     message: "How much is 5 + 5? then divide by 2",
   });
 
-  console.log(response.response.message);
+  console.log(response.message);
 }
 
 void main().then(() => {

examples/agent/react_agent.ts

@@ -68,7 +68,7 @@ async function main() {
   });
 
   // Chat with the agent
-  const { response } = await agent.chat({
+  const response = await agent.chat({
     message: "Divide 16 by 2 then add 20",
   });
 

examples/agent/step_wise_query_tool.ts

@@ -31,7 +31,7 @@ async function main() {
     tools: [queryEngineTool],
   });
 
-  const { response } = await agent.chat({
+  const response = await agent.chat({
     message: "What was his salary?",
   });
 

examples/agent/stream_openai_agent.ts

@@ -68,9 +68,7 @@ async function main() {
 
   console.log("Response:");
 
-  for await (const {
-    response: { delta },
-  } of stream) {
+  for await (const { delta } of stream) {
     process.stdout.write(delta);
   }
 }

examples/agent/wiki.ts

@@ -16,9 +16,7 @@ async function main() {
     stream: true,
   });
 
-  for await (const {
-    response: { delta },
-  } of response) {
+  for await (const { delta } of response) {
     process.stdout.write(delta);
   }
 }

examples/anthropic/agent.ts

@@ -1,11 +1,17 @@
-import { FunctionTool, Settings, WikipediaTool } from "llamaindex";
+import { Anthropic, FunctionTool, Settings, WikipediaTool } from "llamaindex";
 import { AnthropicAgent } from "llamaindex/agent/anthropic";
 
 Settings.callbackManager.on("llm-tool-call", (event) => {
   console.log("llm-tool-call", event.detail.payload.toolCall);
 });
 
+const anthropic = new Anthropic({
+  apiKey: process.env.ANTHROPIC_API_KEY,
+  model: "claude-3-5-sonnet",
+});
+
 const agent = new AnthropicAgent({
+  llm: anthropic,
   tools: [
     FunctionTool.from<{ location: string }>(
       (query) => {
@@ -31,7 +37,6 @@ const agent = new AnthropicAgent({
 });
 
 async function main() {
-  // https://docs.anthropic.com/claude/docs/tool-use#tool-use-best-practices-and-limitations
   const { response } = await agent.chat({
     message:
       "What is the weather in New York? What's the history of New York from Wikipedia in 3 sentences?",

examples/anthropic/stream.ts

@@ -3,7 +3,7 @@ import { Anthropic } from "llamaindex";
 (async () => {
   const anthropic = new Anthropic({
     apiKey: process.env.ANTHROPIC_API_KEY,
-    model: "claude-instant-1.2",
+    model: "claude-3-5-sonnet",
   });
   const stream = await anthropic.chat({
     messages: [

examples/groq.ts

@@ -35,4 +35,4 @@ async function main() {
   console.log(response.response);
 }
 
-await main();
+main().catch(console.error);

examples/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/examples",
   "private": true,
-  "version": "0.0.4",
+  "version": "0.0.5",
   "dependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
     "@datastax/astra-db-ts": "^1.2.1",
@@ -12,15 +12,15 @@
     "commander": "^12.1.0",
     "dotenv": "^16.4.5",
     "js-tiktoken": "^1.0.12",
-    "llamaindex": "^0.3.14",
+    "llamaindex": "^0.4.0",
     "mongodb": "^6.7.0",
     "pathe": "^1.1.2"
   },
   "devDependencies": {
     "@types/node": "^20.14.1",
     "ts-node": "^10.9.2",
-    "tsx": "^4.11.2",
-    "typescript": "^5.4.5"
+    "tsx": "^4.15.6",
+    "typescript": "^5.5.2"
   },
   "scripts": {
     "lint": "eslint ."

examples/readers/package.json

@@ -11,14 +11,15 @@
     "start:pdf": "node --import tsx ./src/pdf.ts",
     "start:llamaparse": "node --import tsx ./src/llamaparse.ts",
     "start:notion": "node --import tsx ./src/notion.ts",
-    "start:llamaparse-dir": "node --import tsx ./src/simple-directory-reader-with-llamaparse.ts"
+    "start:llamaparse-dir": "node --import tsx ./src/simple-directory-reader-with-llamaparse.ts",
+    "start:llamaparse-json": "node --import tsx ./src/llamaparse-json.ts"
   },
   "dependencies": {
     "llamaindex": "*"
   },
   "devDependencies": {
     "@types/node": "^20.12.11",
-    "tsx": "^4.9.3",
-    "typescript": "^5.4.5"
+    "tsx": "^4.15.6",
+    "typescript": "^5.5.2"
   }
 }

examples/readers/src/llamaparse-json.ts

@@ -0,0 +1,68 @@
+import {
+  Document,
+  ImageNode,
+  LlamaParseReader,
+  OpenAI,
+  VectorStoreIndex,
+} from "llamaindex";
+import { createMessageContent } from "llamaindex/synthesizers/utils";
+
+const reader = new LlamaParseReader();
+async function main() {
+  // Load PDF using LlamaParse JSON mode and return an array of json objects
+  const jsonObjs = await reader.loadJson("../data/uber_10q_march_2022.pdf");
+  // Access the first "pages" (=a single parsed file) object in the array
+  const jsonList = jsonObjs[0]["pages"];
+
+  const textDocs = getTextDocs(jsonList);
+  const imageTextDocs = await getImageTextDocs(jsonObjs);
+  const documents = [...textDocs, ...imageTextDocs];
+  // Split text, create embeddings and query the index
+  const index = await VectorStoreIndex.fromDocuments(documents);
+  const queryEngine = index.asQueryEngine();
+  const response = await queryEngine.query({
+    query:
+      "What does the bar graph titled 'Monthly Active Platform Consumers' show?",
+  });
+
+  console.log(response.toString());
+}
+
+main().catch(console.error);
+
+// Extract and assign text and page number from jsonList, return an array of Document objects
+function getTextDocs(jsonList: { text: string; page: number }[]): Document[] {
+  return jsonList.map(
+    (page) => new Document({ text: page.text, metadata: { page: page.page } }),
+  );
+}
+// Download all images from jsonObjs, send them to OpenAI API to get alt text, return an array of Document objects
+async function getImageTextDocs(
+  jsonObjs: Record<string, any>[],
+): Promise<Document[]> {
+  const llm = new OpenAI({
+    model: "gpt-4o",
+    temperature: 0.2,
+    maxTokens: 1000,
+  });
+  const imageDicts = await reader.getImages(jsonObjs, "images");
+  const imageDocs = [];
+
+  for (const imageDict of imageDicts) {
+    const imageDoc = new ImageNode({ image: imageDict.path });
+    const prompt = () => `Describe the image as alt text`;
+    const message = await createMessageContent(prompt, [imageDoc]);
+
+    const response = await llm.complete({
+      prompt: message,
+    });
+
+    const doc = new Document({
+      text: response.text,
+      metadata: { path: imageDict.path },
+    });
+    imageDocs.push(doc);
+  }
+
+  return imageDocs;
+}

package.json

@@ -14,25 +14,25 @@
     "type-check": "tsc -b --diagnostics",
     "release": "pnpm run check-minor-version && pnpm run build:release && changeset publish",
     "release-snapshot": "pnpm run check-minor-version && pnpm run build:release && changeset publish --tag snapshot",
-    "check-minor-version": "node ./scripts/check-minor-version",
-    "new-version": "changeset version && pnpm run check-minor-version && pnpm format:write && pnpm run build:release",
+    "check-minor-version": "node ./scripts/check-minor-version.mjs",
+    "new-version": "changeset version && pnpm run check-minor-version && pnpm format:write && pnpm install && pnpm run build:release",
     "new-snapshot": "pnpm run build:release && changeset version --snapshot"
   },
   "devDependencies": {
     "@changesets/cli": "^2.27.5",
-    "@typescript-eslint/eslint-plugin": "^7.11.0",
+    "@typescript-eslint/eslint-plugin": "^7.13.1",
     "eslint": "^8.57.0",
-    "eslint-config-next": "^14.2.3",
+    "eslint-config-next": "^14.2.4",
     "eslint-config-prettier": "^9.1.0",
-    "eslint-config-turbo": "^1.13.3",
+    "eslint-config-turbo": "^1.13.4",
     "eslint-plugin-react": "7.34.1",
     "husky": "^9.0.11",
-    "lint-staged": "^15.2.5",
+    "lint-staged": "^15.2.7",
     "madge": "^7.0.0",
-    "prettier": "^3.2.5",
+    "prettier": "^3.3.2",
     "prettier-plugin-organize-imports": "^3.2.4",
-    "turbo": "^1.13.3",
-    "typescript": "^5.4.5"
+    "turbo": "^1.13.4",
+    "typescript": "^5.5.2"
   },
   "packageManager": "pnpm@9.0.5",
   "pnpm": {

packages/autotool/CHANGELOG.md

@@ -0,0 +1,11 @@
+# @llamaindex/autotool
+
+## 1.0.0
+
+### Patch Changes
+
+- Updated dependencies [436bc41]
+- Updated dependencies [a44e54f]
+- Updated dependencies [a51ed8d]
+- Updated dependencies [d3b635b]
+  - llamaindex@0.4.0

packages/autotool/examples/01_node/package.json

@@ -5,10 +5,10 @@
   "dependencies": {
     "@llamaindex/autotool": "workspace:*",
     "llamaindex": "workspace:*",
-    "openai": "^4.43.0"
+    "openai": "^4.52.0"
   },
   "devDependencies": {
-    "tsx": "^4.9.3"
+    "tsx": "^4.15.6"
   },
   "scripts": {
     "start": "node --import tsx --import @llamaindex/autotool/node ./src/index.ts"

packages/autotool/examples/02_nextjs/CHANGELOG.md

@@ -1,5 +1,26 @@
 # @llamaindex/autotool-02-next-example
 
+## 0.1.11
+
+### Patch Changes
+
+- Updated dependencies [3c47910]
+- Updated dependencies [ed467a9]
+- Updated dependencies [cba5406]
+  - llamaindex@0.4.1
+  - @llamaindex/autotool@1.0.0
+
+## 0.1.10
+
+### Patch Changes
+
+- Updated dependencies [436bc41]
+- Updated dependencies [a44e54f]
+- Updated dependencies [a51ed8d]
+- Updated dependencies [d3b635b]
+  - llamaindex@0.4.0
+  - @llamaindex/autotool@1.0.0
+
 ## 0.1.9
 
 ### Patch Changes

packages/autotool/examples/02_nextjs/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool-02-next-example",
   "private": true,
-  "version": "0.1.9",
+  "version": "0.1.11",
   "scripts": {
     "dev": "next dev",
     "build": "next build",
@@ -9,8 +9,8 @@
   },
   "dependencies": {
     "@llamaindex/autotool": "workspace:*",
-    "@radix-ui/react-slot": "^1.0.2",
-    "ai": "^3.1.3",
+    "@radix-ui/react-slot": "^1.1.0",
+    "ai": "^3.2.1",
     "class-variance-authority": "^0.7.0",
     "dotenv": "^16.3.1",
     "llamaindex": "workspace:*",
@@ -20,18 +20,18 @@
     "react-dom": "^18.3.1",
     "react-markdown": "^9.0.1",
     "react-syntax-highlighter": "^15.5.0",
-    "sonner": "^1.4.41",
+    "sonner": "^1.5.0",
     "tailwind-merge": "^2.1.0"
   },
   "devDependencies": {
     "@types/node": "^20.12.11",
-    "@types/react": "^18.3.1",
+    "@types/react": "^18.3.3",
     "@types/react-dom": "^18.3.0",
     "@types/react-syntax-highlighter": "^15.5.11",
     "autoprefixer": "^10.4.16",
     "cross-env": "^7.0.3",
     "postcss": "^8.4.32",
-    "tailwindcss": "^3.3.6",
-    "typescript": "^5.4.5"
+    "tailwindcss": "^3.4.4",
+    "typescript": "^5.5.2"
   }
 }

packages/autotool/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/autotool",
   "type": "module",
-  "version": "0.0.1",
+  "version": "1.0.0",
   "description": "auto transpile your JS function to LLM Agent compatible",
   "files": [
     "dist",
@@ -45,13 +45,13 @@
     "dev": "bunchee --watch"
   },
   "dependencies": {
-    "@swc/core": "^1.5.5",
-    "jotai": "^2.8.0",
+    "@swc/core": "^1.6.3",
+    "jotai": "^2.8.3",
     "typedoc": "^0.25.13",
     "unplugin": "^1.10.1"
   },
   "peerDependencies": {
-    "llamaindex": "^0.3.17",
+    "llamaindex": "^0.4.1",
     "openai": "^4",
     "typescript": "^4"
   },
@@ -67,16 +67,16 @@
     }
   },
   "devDependencies": {
-    "@swc/types": "^0.1.6",
+    "@swc/types": "^0.1.8",
     "@types/json-schema": "^7.0.15",
     "@types/node": "^20.12.11",
-    "bunchee": "^5.1.5",
+    "bunchee": "^5.2.1",
     "llamaindex": "workspace:*",
     "next": "14.2.3",
-    "rollup": "^4.17.2",
-    "tsx": "^4.9.3",
-    "typescript": "^5.4.5",
+    "rollup": "^4.18.0",
+    "tsx": "^4.15.6",
+    "typescript": "^5.5.2",
     "vitest": "^1.6.0",
-    "webpack": "^5.91.0"
+    "webpack": "^5.92.1"
   }
 }

packages/autotool/tsconfig.json

@@ -10,7 +10,7 @@
   "include": ["./src"],
   "references": [
     {
-      "path": "../core/tsconfig.json"
+      "path": "../llamaindex/tsconfig.json"
     },
     {
       "path": "../env/tsconfig.json"

packages/community/CHANGELOG.md

@@ -1,5 +1,26 @@
 # @llamaindex/community
 
+## 0.0.5
+
+### Patch Changes
+
+- ed467a9: Add model ids for Anthropic Claude 3.5 Sonnet model on Anthropic and Bedrock
+- Updated dependencies [3c47910]
+- Updated dependencies [ed467a9]
+- Updated dependencies [cba5406]
+  - llamaindex@0.4.1
+
+## 0.0.4
+
+### Patch Changes
+
+- b1a4a74: docs: updated Bedrock Opus region and added a basic README
+- Updated dependencies [436bc41]
+- Updated dependencies [a44e54f]
+- Updated dependencies [a51ed8d]
+- Updated dependencies [d3b635b]
+  - llamaindex@0.4.0
+
 ## 0.0.3
 
 ### Patch Changes

packages/community/README.md

@@ -0,0 +1,11 @@
+# @llamaindex/community
+
+> Tools written by the community for llamaindex
+
+## Current Features:
+
+- Bedrock support for the Anthropic Claude Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
+
+## LICENSE
+
+MIT

packages/community/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/community",
   "description": "Community package for LlamaIndexTS",
-  "version": "0.0.3",
+  "version": "0.0.5",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",
@@ -45,12 +45,12 @@
   },
   "devDependencies": {
     "@swc/cli": "^0.3.12",
-    "@swc/core": "^1.5.5",
+    "@swc/core": "^1.6.3",
     "concurrently": "^8.2.2",
     "tsup": "^8.1.0"
   },
   "dependencies": {
-    "@aws-sdk/client-bedrock-runtime": "^3.582.0",
+    "@aws-sdk/client-bedrock-runtime": "^3.600.0",
     "@types/node": "^20.14.2",
     "llamaindex": "workspace:*"
   }

packages/community/src/llm/bedrock/base.ts

@@ -67,6 +67,7 @@ export enum BEDROCK_MODELS {
   ANTHROPIC_CLAUDE_3_SONNET = "anthropic.claude-3-sonnet-20240229-v1:0",
   ANTHROPIC_CLAUDE_3_HAIKU = "anthropic.claude-3-haiku-20240307-v1:0",
   ANTHROPIC_CLAUDE_3_OPUS = "anthropic.claude-3-opus-20240229-v1:0",
+  ANTHROPIC_CLAUDE_3_5_SONNET = "anthropic.claude-3-5-sonnet-20240620-v1:0",
   META_LLAMA2_13B_CHAT = "meta.llama2-13b-chat-v1",
   META_LLAMA2_70B_CHAT = "meta.llama2-70b-chat-v1",
   META_LLAMA3_8B_INSTRUCT = "meta.llama3-8b-instruct-v1:0",
@@ -100,6 +101,7 @@ const CHAT_ONLY_MODELS = {
   [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_SONNET]: 200000,
   [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU]: 200000,
   [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_OPUS]: 200000,
+  [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET]: 200000,
   [BEDROCK_MODELS.META_LLAMA2_13B_CHAT]: 2048,
   [BEDROCK_MODELS.META_LLAMA2_70B_CHAT]: 4096,
   [BEDROCK_MODELS.META_LLAMA3_8B_INSTRUCT]: 8192,
@@ -126,6 +128,7 @@ export const STREAMING_MODELS = new Set([
   BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_SONNET,
   BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU,
   BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_OPUS,
+  BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET,
   BEDROCK_MODELS.META_LLAMA2_13B_CHAT,
   BEDROCK_MODELS.META_LLAMA2_70B_CHAT,
   BEDROCK_MODELS.META_LLAMA3_8B_INSTRUCT,

packages/community/tsconfig.json

@@ -13,7 +13,7 @@
   "exclude": ["node_modules"],
   "references": [
     {
-      "path": "../core/tsconfig.json"
+      "path": "../llamaindex/tsconfig.json"
     }
   ]
 }

packages/core/src/GlobalsHelper.ts

@@ -1,49 +0,0 @@
-import { encodingForModel } from "js-tiktoken";
-
-export enum Tokenizers {
-  CL100K_BASE = "cl100k_base",
-}
-
-/**
- * @internal Helper class singleton
- */
-class GlobalsHelper {
-  defaultTokenizer: {
-    encode: (text: string) => Uint32Array;
-    decode: (tokens: Uint32Array) => string;
-  };
-
-  constructor() {
-    const encoding = encodingForModel("text-embedding-ada-002"); // cl100k_base
-
-    this.defaultTokenizer = {
-      encode: (text: string) => {
-        return new Uint32Array(encoding.encode(text));
-      },
-      decode: (tokens: Uint32Array) => {
-        const numberArray = Array.from(tokens);
-        const text = encoding.decode(numberArray);
-        const uint8Array = new TextEncoder().encode(text);
-        return new TextDecoder().decode(uint8Array);
-      },
-    };
-  }
-
-  tokenizer(encoding?: Tokenizers) {
-    if (encoding && encoding !== Tokenizers.CL100K_BASE) {
-      throw new Error(`Tokenizer encoding ${encoding} not yet supported`);
-    }
-
-    return this.defaultTokenizer!.encode.bind(this.defaultTokenizer);
-  }
-
-  tokenizerDecoder(encoding?: Tokenizers) {
-    if (encoding && encoding !== Tokenizers.CL100K_BASE) {
-      throw new Error(`Tokenizer encoding ${encoding} not yet supported`);
-    }
-
-    return this.defaultTokenizer!.decode.bind(this.defaultTokenizer);
-  }
-}
-
-export const globalsHelper = new GlobalsHelper();

packages/core/src/Response.ts

@@ -1,23 +0,0 @@
-import type { NodeWithScore } from "./Node.js";
-
-/**
- * Response is the output of a LLM
- */
-export class Response {
-  response: string;
-  sourceNodes?: NodeWithScore[];
-  metadata: Record<string, unknown> = {};
-
-  constructor(response: string, sourceNodes?: NodeWithScore[]) {
-    this.response = response;
-    this.sourceNodes = sourceNodes || [];
-  }
-
-  protected _getFormattedSources() {
-    throw new Error("Not implemented yet");
-  }
-
-  toString() {
-    return this.response ?? "";
-  }
-}

packages/env/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @llamaindex/env
 
+## 0.1.4
+
+### Patch Changes
+
+- 56fabbb: Release env changes to tokenizer
+
 ## 0.1.3
 
 ### Patch Changes

packages/env/jsr.json

@@ -1,6 +1,6 @@
 {
   "name": "@llamaindex/env",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "exports": {
     ".": "./src/index.ts"
   },

packages/env/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/env",
   "description": "environment wrapper, supports all JS environment including node, deno, bun, edge runtime, and cloudflare worker",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",
@@ -69,18 +69,20 @@
   "devDependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
     "@swc/cli": "^0.3.12",
-    "@swc/core": "^1.5.5",
+    "@swc/core": "^1.6.3",
     "concurrently": "^8.2.2",
     "pathe": "^1.1.2",
     "vitest": "^1.6.0"
   },
   "dependencies": {
-    "@types/lodash": "^4.17.1",
+    "@types/lodash": "^4.17.5",
     "@types/node": "^20.12.11"
   },
   "peerDependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
-    "pathe": "^1.1.2"
+    "js-tiktoken": "^1.0.12",
+    "pathe": "^1.1.2",
+    "tiktoken": "^1.0.15"
   },
   "peerDependenciesMeta": {
     "@aws-crypto/sha256-js": {

packages/env/src/index.edge-light.ts

@@ -4,3 +4,5 @@
  * @module
  */
 export * from "./polyfill.js";
+
+export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js";

packages/env/src/index.ts

@@ -35,14 +35,15 @@ export function createSHA256(): SHA256 {
   };
 }
 
+export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/node.js";
 export { AsyncLocalStorage, CustomEvent, getEnv, setEnvs } from "./utils.js";
 export {
   EOL,
-  ReadableStream,
-  TransformStream,
-  WritableStream,
   fs,
   ok,
   path,
   randomUUID,
+  ReadableStream,
+  TransformStream,
+  WritableStream,
 };

packages/env/src/index.workerd.ts

@@ -12,3 +12,5 @@ export * from "./polyfill.js";
 export function getEnv(name: string): string | undefined {
   return INTERNAL_ENV[name];
 }
+
+export { Tokenizers, tokenizers, type Tokenizer } from "./tokenizers/js.js";

packages/env/src/tokenizers/js.ts

@@ -0,0 +1,35 @@
+// Note: js-tiktoken it's 60x slower than the WASM implementation - use it only for unsupported environments
+import { getEncoding } from "js-tiktoken";
+import type { Tokenizer } from "./types.js";
+import { Tokenizers } from "./types.js";
+
+class TokenizerSingleton {
+  private defaultTokenizer: Tokenizer;
+
+  constructor() {
+    const encoding = getEncoding("cl100k_base");
+
+    this.defaultTokenizer = {
+      encode: (text: string) => {
+        return new Uint32Array(encoding.encode(text));
+      },
+      decode: (tokens: Uint32Array) => {
+        const numberArray = Array.from(tokens);
+        const text = encoding.decode(numberArray);
+        const uint8Array = new TextEncoder().encode(text);
+        return new TextDecoder().decode(uint8Array);
+      },
+    };
+  }
+
+  tokenizer(encoding?: Tokenizers) {
+    if (encoding && encoding !== Tokenizers.CL100K_BASE) {
+      throw new Error(`Tokenizer encoding ${encoding} not yet supported`);
+    }
+
+    return this.defaultTokenizer;
+  }
+}
+
+export const tokenizers = new TokenizerSingleton();
+export { Tokenizers, type Tokenizer };

packages/env/src/tokenizers/node.ts

@@ -0,0 +1,34 @@
+// Note: This is using th WASM implementation of tiktoken which is 60x faster
+import type { Tokenizer } from "./types.js";
+import { Tokenizers } from "./types.js";
+
+import { get_encoding } from "tiktoken";
+
+class TokenizerSingleton {
+  private defaultTokenizer: Tokenizer;
+
+  constructor() {
+    const encoding = get_encoding("cl100k_base");
+
+    this.defaultTokenizer = {
+      encode: (text: string) => {
+        return encoding.encode(text);
+      },
+      decode: (tokens: Uint32Array) => {
+        const text = encoding.decode(tokens);
+        return new TextDecoder().decode(text);
+      },
+    };
+  }
+
+  tokenizer(encoding?: Tokenizers): Tokenizer {
+    if (encoding && encoding !== Tokenizers.CL100K_BASE) {
+      throw new Error(`Tokenizer encoding ${encoding} not yet supported`);
+    }
+
+    return this.defaultTokenizer;
+  }
+}
+
+export const tokenizers: TokenizerSingleton = new TokenizerSingleton();
+export { Tokenizers, type Tokenizer };

packages/env/src/tokenizers/types.ts

@@ -0,0 +1,8 @@
+export enum Tokenizers {
+  CL100K_BASE = "cl100k_base",
+}
+
+export interface Tokenizer {
+  encode: (text: string) => Uint32Array;
+  decode: (tokens: Uint32Array) => string;
+}

packages/env/tsconfig.json

@@ -5,9 +5,10 @@
     "outDir": "./dist/type",
     "tsBuildInfoFile": "./dist/.tsbuildinfo",
     "emitDeclarationOnly": true,
-    "module": "node16",
+    "module": "nodenext",
     "moduleResolution": "node16",
-    "types": ["node"]
+    "types": ["node"],
+    "resolveJsonModule": true
   },
   "include": ["./src"],
   "exclude": ["node_modules"]

packages/experimental/CHANGELOG.md

@@ -1,5 +1,24 @@
 # @llamaindex/experimental
 
+## 0.0.36
+
+### Patch Changes
+
+- Updated dependencies [3c47910]
+- Updated dependencies [ed467a9]
+- Updated dependencies [cba5406]
+  - llamaindex@0.4.1
+
+## 0.0.35
+
+### Patch Changes
+
+- Updated dependencies [436bc41]
+- Updated dependencies [a44e54f]
+- Updated dependencies [a51ed8d]
+- Updated dependencies [d3b635b]
+  - llamaindex@0.4.0
+
 ## 0.0.34
 
 ### Patch Changes

packages/experimental/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@llamaindex/experimental",
   "description": "Experimental package for LlamaIndexTS",
-  "version": "0.0.34",
+  "version": "0.0.36",
   "type": "module",
   "types": "dist/type/index.d.ts",
   "main": "dist/cjs/index.js",
@@ -57,13 +57,13 @@
   "devDependencies": {
     "@aws-crypto/sha256-js": "^5.2.0",
     "@swc/cli": "^0.3.12",
-    "@swc/core": "^1.5.5",
+    "@swc/core": "^1.6.3",
     "@types/jsonpath": "^0.2.4",
     "concurrently": "^8.2.2",
     "pathe": "^1.1.2"
   },
   "dependencies": {
-    "@types/lodash": "^4.17.1",
+    "@types/lodash": "^4.17.5",
     "@types/node": "^20.12.11",
     "jsonpath": "^1.1.1",
     "llamaindex": "workspace:*",