run-llama/LlamaIndexTS
1
0
Fork 0
mirror of https://github.com/run-llama/LlamaIndexTS.git synced 2026-07-20 22:41:23 -04:00
Code Issues 138 Packages Projects Releases 10 Wiki Activity

feat(readers): add XMLReader for parsing XML files (#1846) (#2031)

Browse Source 
Co-authored-by: Marcus Schiesser <marcus.schiesser@googlemail.com>
This commit is contained in:
jerinthomascarmel
2025-06-24 09:16:32 +05:30
committed by GitHub
parent 97bbce6e13
commit c5846bd7dc
9 changed files with 258 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/smooth-points-move.md
+6
View File
@@ -0,0 +1,6 @@
---
"@llamaindex/readers": patch
"llamaindex-loader-example": patch
---
feat(readers): add XMLReader for parsing XML files (#1846)
examples/data/company.xml
+72
View File
@@ -0,0 +1,72 @@
<?xml version="1.0" encoding="UTF-8"?>
<company name="MidSizeCorp" founded="2008">
<division name="Engineering" head="Dana White">
<department name="Frontend" lead="Alex Kim">
<team name="Web">
<employee id="E01">
<name>Jordan Lee</name>
<role>Lead Developer</role>
<projects>
<project code="PRJ101" status="active">
<title>User Portal</title>
<deadline>2025-08-01</deadline>
<tasks>
<task id="T1011">
<description>Implement login page</description>
<due>2025-05-10</due>
</task>
<task id="T1012">
<description>Design dashboard</description>
<due>2025-05-20</due>
</task>
</tasks>
</project>
</projects>
</employee>
<employee id="E02">
<name>Riley Chen</name>
<role>UI Designer</role>
</employee>
</team>
<team name="Mobile">
<employee id="E03">
<name>Sam Patel</name>
<role>iOS Developer</role>
</employee>
</team>
</department>
<department name="Backend" lead="Morgan Reed">
<team name="API">
<employee id="E04">
<name>Taylor Jones</name>
<role>API Engineer</role>
</employee>
</team>
<team name="Database">
<employee id="E05">
<name>Casey Nguyen</name>
<role>DB Administrator</role>
</employee>
</team>
</department>
</division>
<division name="Marketing" head="Pat Morgan">
<department name="Digital" lead="Alex Rivera">
<team name="Content">
<employee id="M01">
<name>Charlie Brooks</name>
<role>Content Strategist</role>
</employee>
</team>
</department>
</division>
<headquarters location="Chicago, USA">
<address>
<street>789 Lake Shore Drive</street>
<city>Chicago</city>
<zip>60601</zip>
</address>
</headquarters>
</company>
examples/readers/package.json
+2 -1
View File
@@ -15,7 +15,8 @@
"start:llamaparse-json": "node --import tsx ./src/llamaparse-json.ts",
"start:discord": "node --import tsx ./src/discord.ts",
"start:json": "node --import tsx ./src/json.ts",
"start:obsidian": "node --import tsx ./src/obsidian.ts"
"start:obsidian": "node --import tsx ./src/obsidian.ts",
"start:xml": "node --import tsx ./src/xml.ts"
},
"dependencies": {
"@llamaindex/cloud": "workspace:* || ^2.0.24",
examples/readers/src/xml.ts
+16
View File
@@ -0,0 +1,16 @@
import { XMLReader } from "@llamaindex/readers/xml";
async function main() {
// Load PDF
const reader = new XMLReader({
splitLevel: 2,
});
const documents = await reader.loadData("../data/company.xml");
for (const doc of documents) {
console.log(doc.text);
console.log("----");
}
}
main().catch(console.error);
packages/readers/package.json
+13 -1
View File
@@ -129,6 +129,16 @@
"types": "./text/dist/index.d.ts",
"default": "./text/dist/index.js"
}
},
"./xml": {
"require": {
"types": "./xml/dist/index.d.cts",
"default": "./xml/dist/index.cjs"
},
"import": {
"types": "./xml/dist/index.d.ts",
"default": "./xml/dist/index.js"
}
}
},
"files": [
@@ -142,7 +152,8 @@
"obsidian",
"pdf",
"text",
"node"
"node",
"xml"
],
"repository": {
"type": "git",
@@ -166,6 +177,7 @@
},
"dependencies": {
"@discoveryjs/json-ext": "^0.6.1",
"@xmldom/xmldom": "^0.9.8",
"csv-parse": "^5.5.6",
"mammoth": "^1.7.2",
"unpdf": "^0.12.1"
packages/readers/src/directory/index.ts
+3
View File
@@ -7,6 +7,8 @@ import { ImageReader } from "../image";
import { MarkdownReader } from "../markdown";
import { PDFReader } from "../pdf";
import { TextFileReader } from "../text";
import { XMLReader } from "../xml";
import {
AbstractSimpleDirectoryReader,
type SimpleDirectoryReaderLoadDataParams,
@@ -24,6 +26,7 @@ export const FILE_EXT_TO_READER: Record<string, FileReader> = {
jpeg: new ImageReader(),
png: new ImageReader(),
gif: new ImageReader(),
xml: new XMLReader(),
};
/**
packages/readers/src/xml.ts
+70
View File
@@ -0,0 +1,70 @@
import { Document, FileReader } from "@llamaindex/core/schema";
import {
DOMParser,
XMLSerializer,
Document as XmlDocument,
Element as XmlElement,
} from "@xmldom/xmldom";
export class XMLReader extends FileReader<Document> {
private splitLevel: number;
/**
* @param splitLevel how deep to split the XML tree
*/
constructor({ splitLevel }: { splitLevel?: number } = {}) {
super();
this.splitLevel = splitLevel ?? 0;
}
/** XMLParser */
async loadDataAsContent(fileContent: Uint8Array): Promise<Document[]> {
const xmlStr = new TextDecoder().decode(fileContent);
const doc: XmlDocument = new DOMParser().parseFromString(
xmlStr,
"application/xml",
);
if (!doc || !doc.documentElement) {
throw new Error("Invalid XML: unable to parse document");
}
const root: XmlElement = doc.documentElement!;
return this._parseElementToDocuments(root);
}
/**
* Internal: split tree into leaf or level-matched nodes
*/
private _getLeafNodesUpToLevel(
root: XmlElement,
level: number,
): XmlElement[] {
const result: XmlElement[] = [];
const traverse = (node: XmlElement, currLevel: number) => {
const children = Array.from(node.childNodes).filter(
(n) => n.nodeType === n.ELEMENT_NODE,
) as XmlElement[];
if (children.length === 0 || currLevel === level) {
result.push(node);
} else {
for (const child of children) {
traverse(child, currLevel + 1);
}
}
};
traverse(root, 0);
return result;
}
private _parseElementToDocuments(root: XmlElement): Document[] {
const nodes = this._getLeafNodesUpToLevel(root, this.splitLevel);
const serializer = new XMLSerializer();
return nodes.map(
(node) =>
new Document({ text: serializer.serializeToString(node).trim() }),
);
}
}
packages/readers/xml/package.json
+14
View File
@@ -0,0 +1,14 @@
{
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
"types": "./dist/index.d.ts",
"exports": {
".": {
"edge-light": "./dist/index.edge-light.js",
"workerd": "./dist/index.workerd.js",
"default": "./dist/index.js"
}
},
"private": true
}
pnpm-lock.yaml
Generated
+62 -54
View File
@@ -1745,6 +1745,9 @@ importers:
'@discoveryjs/json-ext':
specifier: ^0.6.1
version: 0.6.3
'@xmldom/xmldom':
specifier: ^0.9.8
version: 0.9.8
csv-parse:
specifier: ^5.5.6
version: 5.6.0
@@ -6903,6 +6906,10 @@ packages:
resolution: {integrity: sha512-2WALfTl4xo2SkGCYRt6rDTFfk9R1czmBvUQy12gK2KuRKIpWEhcbbzy8EZXtz/jkRqHX8bFEc6FC1HjX4TUWYw==}
engines: {node: '>=10.0.0'}
'@xmldom/xmldom@0.9.8':
resolution: {integrity: sha512-p96FSY54r+WJ50FIOsCOjyj/wavs8921hG5+kVMmZgKcvIKxMXHTrjNJvRgWa/zuX3B6t2lijLNFaOyuxUH+2A==}
engines: {node: '>=14.6'}
'@xtuc/ieee754@1.2.0':
resolution: {integrity: sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==}
@@ -19508,10 +19515,10 @@ snapshots:
transitivePeerDependencies:
- supports-color
'@typescript-eslint/eslint-plugin@8.30.1(@typescript-eslint/parser@8.30.1(eslint@9.29.0(jiti@2.4.2))(typescript@5.8.3))(eslint@9.29.0(jiti@2.4.2))(typescript@5.8.3)':
'@typescript-eslint/eslint-plugin@8.30.1(@typescript-eslint/parser@8.30.1(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.29.0(jiti@2.4.2))(typescript@5.8.3)':
dependencies:
'@eslint-community/regexpp': 4.12.1
'@typescript-eslint/parser': 8.30.1(eslint@9.29.0(jiti@2.4.2))(typescript@5.8.3)
'@typescript-eslint/parser': 8.30.1(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3)
'@typescript-eslint/scope-manager': 8.30.1
'@typescript-eslint/type-utils': 8.30.1(eslint@9.29.0(jiti@2.4.2))(typescript@5.8.3)
'@typescript-eslint/utils': 8.30.1(eslint@9.29.0(jiti@2.4.2))(typescript@5.8.3)
@@ -20281,6 +20288,8 @@ snapshots:
'@xmldom/xmldom@0.8.10': {}
'@xmldom/xmldom@0.9.8': {}
'@xtuc/ieee754@1.2.0': {}
'@xtuc/long@4.2.2': {}
@@ -21835,7 +21844,7 @@ snapshots:
'@typescript-eslint/parser': 8.30.1(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3)
eslint: 9.16.0(jiti@2.4.2)
eslint-import-resolver-node: 0.3.9
eslint-import-resolver-typescript: 3.7.0(eslint-plugin-import@2.31.0)(eslint@9.16.0(jiti@2.4.2))
eslint-import-resolver-typescript: 3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.30.1(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.16.0(jiti@2.4.2)))(eslint@9.16.0(jiti@2.4.2))
eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.30.1(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-typescript@3.7.0)(eslint@9.16.0(jiti@2.4.2))
eslint-plugin-jsx-a11y: 6.10.2(eslint@9.16.0(jiti@2.4.2))
eslint-plugin-react: 7.37.2(eslint@9.16.0(jiti@2.4.2))
@@ -21855,8 +21864,8 @@ snapshots:
'@typescript-eslint/parser': 6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3)
eslint: 9.22.0(jiti@2.4.2)
eslint-import-resolver-node: 0.3.9
eslint-import-resolver-typescript: 3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2)))(eslint@9.22.0(jiti@2.4.2))
eslint-plugin-import: 2.31.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-typescript@3.7.0)(eslint@9.22.0(jiti@2.4.2))
eslint-import-resolver-typescript: 3.7.0(eslint-plugin-import@2.31.0)(eslint@9.22.0(jiti@2.4.2))
eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.30.1(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2))
eslint-plugin-jsx-a11y: 6.10.2(eslint@9.22.0(jiti@2.4.2))
eslint-plugin-react: 7.37.2(eslint@9.22.0(jiti@2.4.2))
eslint-plugin-react-hooks: 5.2.0(eslint@9.22.0(jiti@2.4.2))
@@ -21885,23 +21894,7 @@ snapshots:
transitivePeerDependencies:
- supports-color
eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2)))(eslint@9.22.0(jiti@2.4.2)):
dependencies:
'@nolyfill/is-core-module': 1.0.39
debug: 4.4.0
enhanced-resolve: 5.18.1
eslint: 9.22.0(jiti@2.4.2)
fast-glob: 3.3.3
get-tsconfig: 4.10.0
is-bun-module: 1.3.0
is-glob: 4.0.3
stable-hash: 0.0.4
optionalDependencies:
eslint-plugin-import: 2.31.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-typescript@3.7.0)(eslint@9.22.0(jiti@2.4.2))
transitivePeerDependencies:
- supports-color
eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0)(eslint@9.16.0(jiti@2.4.2)):
eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.30.1(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.16.0(jiti@2.4.2)))(eslint@9.16.0(jiti@2.4.2)):
dependencies:
'@nolyfill/is-core-module': 1.0.39
debug: 4.4.0
@@ -21917,55 +21910,41 @@ snapshots:
transitivePeerDependencies:
- supports-color
eslint-module-utils@2.12.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2)))(eslint@9.22.0(jiti@2.4.2)))(eslint@9.22.0(jiti@2.4.2)):
eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0)(eslint@9.22.0(jiti@2.4.2)):
dependencies:
debug: 3.2.7
optionalDependencies:
'@typescript-eslint/parser': 6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3)
'@nolyfill/is-core-module': 1.0.39
debug: 4.4.0
enhanced-resolve: 5.18.1
eslint: 9.22.0(jiti@2.4.2)
eslint-import-resolver-node: 0.3.9
eslint-import-resolver-typescript: 3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2)))(eslint@9.22.0(jiti@2.4.2))
fast-glob: 3.3.3
get-tsconfig: 4.10.0
is-bun-module: 1.3.0
is-glob: 4.0.3
stable-hash: 0.0.4
optionalDependencies:
eslint-plugin-import: 2.31.0(@typescript-eslint/parser@8.30.1(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2))
transitivePeerDependencies:
- supports-color
eslint-module-utils@2.12.0(@typescript-eslint/parser@8.30.1(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0)(eslint@9.16.0(jiti@2.4.2)):
eslint-module-utils@2.12.0(@typescript-eslint/parser@8.30.1(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.30.1(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.16.0(jiti@2.4.2)))(eslint@9.16.0(jiti@2.4.2)))(eslint@9.16.0(jiti@2.4.2)):
dependencies:
debug: 3.2.7
optionalDependencies:
'@typescript-eslint/parser': 8.30.1(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3)
eslint: 9.16.0(jiti@2.4.2)
eslint-import-resolver-node: 0.3.9
eslint-import-resolver-typescript: 3.7.0(eslint-plugin-import@2.31.0)(eslint@9.16.0(jiti@2.4.2))
eslint-import-resolver-typescript: 3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.30.1(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.16.0(jiti@2.4.2)))(eslint@9.16.0(jiti@2.4.2))
transitivePeerDependencies:
- supports-color
eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-typescript@3.7.0)(eslint@9.22.0(jiti@2.4.2)):
eslint-module-utils@2.12.0(@typescript-eslint/parser@8.30.1(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint@9.22.0(jiti@2.4.2)):
dependencies:
'@rtsao/scc': 1.1.0
array-includes: 3.1.8
array.prototype.findlastindex: 1.2.5
array.prototype.flat: 1.3.3
array.prototype.flatmap: 1.3.3
debug: 3.2.7
doctrine: 2.1.0
optionalDependencies:
'@typescript-eslint/parser': 8.30.1(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3)
eslint: 9.22.0(jiti@2.4.2)
eslint-import-resolver-node: 0.3.9
eslint-module-utils: 2.12.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2)))(eslint@9.22.0(jiti@2.4.2)))(eslint@9.22.0(jiti@2.4.2))
hasown: 2.0.2
is-core-module: 2.16.1
is-glob: 4.0.3
minimatch: 3.1.2
object.fromentries: 2.0.8
object.groupby: 1.0.3
object.values: 1.2.1
semver: 6.3.1
string.prototype.trimend: 1.0.9
tsconfig-paths: 3.15.0
optionalDependencies:
'@typescript-eslint/parser': 6.21.0(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3)
transitivePeerDependencies:
- eslint-import-resolver-typescript
- eslint-import-resolver-webpack
- supports-color
eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.30.1(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-typescript@3.7.0)(eslint@9.16.0(jiti@2.4.2)):
@@ -21979,7 +21958,7 @@ snapshots:
doctrine: 2.1.0
eslint: 9.16.0(jiti@2.4.2)
eslint-import-resolver-node: 0.3.9
eslint-module-utils: 2.12.0(@typescript-eslint/parser@8.30.1(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0)(eslint@9.16.0(jiti@2.4.2))
eslint-module-utils: 2.12.0(@typescript-eslint/parser@8.30.1(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.7.0(eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.30.1(eslint@9.16.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.16.0(jiti@2.4.2)))(eslint@9.16.0(jiti@2.4.2)))(eslint@9.16.0(jiti@2.4.2))
hasown: 2.0.2
is-core-module: 2.16.1
is-glob: 4.0.3
@@ -21997,6 +21976,35 @@ snapshots:
- eslint-import-resolver-webpack
- supports-color
eslint-plugin-import@2.31.0(@typescript-eslint/parser@8.30.1(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.22.0(jiti@2.4.2)):
dependencies:
'@rtsao/scc': 1.1.0
array-includes: 3.1.8
array.prototype.findlastindex: 1.2.5
array.prototype.flat: 1.3.3
array.prototype.flatmap: 1.3.3
debug: 3.2.7
doctrine: 2.1.0
eslint: 9.22.0(jiti@2.4.2)
eslint-import-resolver-node: 0.3.9
eslint-module-utils: 2.12.0(@typescript-eslint/parser@8.30.1(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint-import-resolver-node@0.3.9)(eslint@9.22.0(jiti@2.4.2))
hasown: 2.0.2
is-core-module: 2.16.1
is-glob: 4.0.3
minimatch: 3.1.2
object.fromentries: 2.0.8
object.groupby: 1.0.3
object.values: 1.2.1
semver: 6.3.1
string.prototype.trimend: 1.0.9
tsconfig-paths: 3.15.0
optionalDependencies:
'@typescript-eslint/parser': 8.30.1(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3)
transitivePeerDependencies:
- eslint-import-resolver-typescript
- eslint-import-resolver-webpack
- supports-color
eslint-plugin-jsx-a11y@6.10.2(eslint@9.16.0(jiti@2.4.2)):
dependencies:
aria-query: 5.3.2
@@ -27838,7 +27846,7 @@ snapshots:
typescript-eslint@8.30.1(eslint@9.29.0(jiti@2.4.2))(typescript@5.8.3):
dependencies:
'@typescript-eslint/eslint-plugin': 8.30.1(@typescript-eslint/parser@8.30.1(eslint@9.29.0(jiti@2.4.2))(typescript@5.8.3))(eslint@9.29.0(jiti@2.4.2))(typescript@5.8.3)
'@typescript-eslint/eslint-plugin': 8.30.1(@typescript-eslint/parser@8.30.1(eslint@9.22.0(jiti@2.4.2))(typescript@5.7.3))(eslint@9.29.0(jiti@2.4.2))(typescript@5.8.3)
'@typescript-eslint/parser': 8.30.1(eslint@9.29.0(jiti@2.4.2))(typescript@5.8.3)
'@typescript-eslint/utils': 8.30.1(eslint@9.29.0(jiti@2.4.2))(typescript@5.8.3)
eslint: 9.29.0(jiti@2.4.2)