Compare commits

...

4 Commits

Author SHA1 Message Date
sweep-ai[bot] 5d31beb760 Corrected import statement and added checks in getNodesFromDocument function 2023-08-14 17:43:35 +00:00
sweep-ai[bot] 6c9e84f23c Added validation and error handling for JSON support in Document class 2023-08-14 17:42:59 +00:00
sweep-ai[bot] 599f5d9f9b Modified getNodesFromDocument to handle JSON Documents 2023-08-14 17:37:58 +00:00
sweep-ai[bot] 2426aaffcb Added JSON support to Document class 2023-08-14 17:36:38 +00:00
2 changed files with 34 additions and 2 deletions
+27 -2
View File
@@ -226,9 +226,34 @@ export class IndexNode extends TextNode {
* A document is just a special text node with a docId.
*/
export class Document extends TextNode {
constructor(init?: Partial<Document>) {
constructor(init?: Partial<Document>, type: string = 'text') {
if (!init) {
throw new Error("init parameter is null or undefined");
}
if (type !== 'text' && type !== 'json') {
throw new Error("type must be either 'text' or 'json'");
}
super(init);
Object.assign(this, init);
if (type === 'json') {
this.text = this.jsonToText(init.text);
} else {
Object.assign(this, init);
}
}
private jsonToText(jsonData: any): string {
if (typeof jsonData !== 'object' || jsonData === null) {
throw new Error("jsonData must be an object");
}
let text = '';
for (let key in jsonData) {
if (typeof jsonData[key] === 'object' && jsonData[key] !== null) {
text += `${key}: ${this.jsonToText(jsonData[key])}\n`;
} else {
text += `${key}: ${jsonData[key]}\n`;
}
}
return text;
}
getType() {
+7
View File
@@ -34,6 +34,13 @@ export function getNodesFromDocument(
) {
let nodes: TextNode[] = [];
if (document.type !== 'text' && document.type !== 'json') {
throw new Error("document.type must be either 'text' or 'json'");
}
if (document.type === 'json') {
document.text = document.jsonToText(JSON.parse(document.text));
}
const textSplits = getTextSplitsFromDocument(document, textSplitter);
textSplits.forEach((textSplit) => {