copy over example

feat: added draft for a summarizer of the chat history

.changeset/nasty-seas-care.md

@@ -1,5 +0,0 @@
----
-"llamaindex": patch
----
-
-Add notion loader (thank you @TomPenguin!)

.changeset/odd-mangos-judge.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Chat History summarization (thanks @marcusschlesser)

.changeset/shaggy-cooks-push.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+Notion database support (thanks @TomPenguin)

.changeset/sixty-vans-punch.md

@@ -0,0 +1,5 @@
+---
+"llamaindex": patch
+---
+
+KeywordIndex (thanks @swk777)

apps/simple/CHANGELOG.md

@@ -1,5 +1,12 @@
 # simple
 
+## 0.0.24
+
+### Patch Changes
+
+- Updated dependencies [5bb55bc]
+  - llamaindex@0.0.26
+
 ## 0.0.23
 
 ### Patch Changes

apps/simple/keywordIndex.ts

@@ -0,0 +1,32 @@
+import {
+  Document,
+  KeywordTableIndex,
+  KeywordTableRetrieverMode,
+} from "llamaindex";
+import essay from "./essay";
+
+async function main() {
+  const document = new Document({ text: essay, id_: "essay" });
+  const index = await KeywordTableIndex.fromDocuments([document]);
+
+  const allModes: KeywordTableRetrieverMode[] = [
+    KeywordTableRetrieverMode.DEFAULT,
+    KeywordTableRetrieverMode.SIMPLE,
+    KeywordTableRetrieverMode.RAKE,
+  ];
+  allModes.forEach(async (mode) => {
+    const queryEngine = index.asQueryEngine({
+      retriever: index.asRetriever({
+        mode,
+      }),
+    });
+    const response = await queryEngine.query(
+      "What did the author do growing up?",
+    );
+    console.log(response.toString());
+  });
+}
+
+main().catch((e: Error) => {
+  console.error(e, e.stack);
+});

apps/simple/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.0.23",
+  "version": "0.0.24",
   "private": true,
   "name": "simple",
   "dependencies": {

examples/keywordIndex.ts

@@ -0,0 +1,32 @@
+import {
+  Document,
+  KeywordTableIndex,
+  KeywordTableRetrieverMode,
+} from "llamaindex";
+import essay from "./essay";
+
+async function main() {
+  const document = new Document({ text: essay, id_: "essay" });
+  const index = await KeywordTableIndex.fromDocuments([document]);
+
+  const allModes: KeywordTableRetrieverMode[] = [
+    KeywordTableRetrieverMode.DEFAULT,
+    KeywordTableRetrieverMode.SIMPLE,
+    KeywordTableRetrieverMode.RAKE,
+  ];
+  allModes.forEach(async (mode) => {
+    const queryEngine = index.asQueryEngine({
+      retriever: index.asRetriever({
+        mode,
+      }),
+    });
+    const response = await queryEngine.query(
+      "What did the author do growing up?",
+    );
+    console.log(response.toString());
+  });
+}
+
+main().catch((e: Error) => {
+  console.error(e, e.stack);
+});

examples/notion.ts

@@ -0,0 +1,89 @@
+import { Client } from "@notionhq/client";
+import { program } from "commander";
+import { NotionReader, VectorStoreIndex } from "llamaindex";
+import { stdin as input, stdout as output } from "node:process";
+// readline/promises is still experimental so not in @types/node yet
+// @ts-ignore
+import readline from "node:readline/promises";
+
+program
+  .argument("[page]", "Notion page id (must be provided)")
+  .action(async (page, _options, command) => {
+    // Initializing a client
+
+    if (!process.env.NOTION_TOKEN) {
+      console.log(
+        "No NOTION_TOKEN found in environment variables. You will need to register an integration https://www.notion.com/my-integrations and put it in your NOTION_TOKEN environment variable.",
+      );
+      return;
+    }
+
+    const notion = new Client({
+      auth: process.env.NOTION_TOKEN,
+    });
+
+    if (!page) {
+      const response = await notion.search({
+        filter: {
+          value: "page",
+          property: "object",
+        },
+        sort: {
+          direction: "descending",
+          timestamp: "last_edited_time",
+        },
+      });
+
+      const { results } = response;
+
+      if (results.length === 0) {
+        console.log(
+          "No pages found. You will need to share it with your integration. (tap the three dots on the top right, find Add connections, and add your integration)",
+        );
+        return;
+      } else {
+        const pages = results
+          .map((result) => {
+            if (!("url" in result)) {
+              return null;
+            }
+
+            return {
+              id: result.id,
+              url: result.url,
+            };
+          })
+          .filter((page) => page !== null);
+        console.log("Found pages:");
+        console.table(pages);
+        console.log(`To run, run ts-node ${command.name()} [page id]`);
+        return;
+      }
+    }
+
+    const reader = new NotionReader({ client: notion });
+    const documents = await reader.loadData(page);
+    console.log(documents);
+
+    // Split text and create embeddings. Store them in a VectorStoreIndex
+    const index = await VectorStoreIndex.fromDocuments(documents);
+
+    // Create query engine
+    const queryEngine = index.asQueryEngine();
+
+    const rl = readline.createInterface({ input, output });
+    while (true) {
+      const query = await rl.question("Query: ");
+
+      if (!query) {
+        break;
+      }
+
+      const response = await queryEngine.query(query);
+
+      // Output response
+      console.log(response.toString());
+    }
+  });
+
+program.parse();

packages/core/CHANGELOG.md

@@ -1,5 +1,11 @@
 # llamaindex
 
+## 0.0.26
+
+### Patch Changes
+
+- 5bb55bc: Add notion loader (thank you @TomPenguin!)
+
 ## 0.0.25
 
 ### Patch Changes

packages/core/package.json

@@ -1,14 +1,15 @@
 {
   "name": "llamaindex",
-  "version": "0.0.25",
+  "version": "0.0.26",
   "dependencies": {
     "@anthropic-ai/sdk": "^0.6.2",
     "@notionhq/client": "^2.2.12",
     "lodash": "^4.17.21",
-    "md-utils-ts": "^2.0.0",
+    "notion-md-crawler": "^0.0.2",
     "openai": "^4.3.1",
     "papaparse": "^5.4.1",
     "pdf-parse": "^1.1.1",
+    "rake-modified": "^1.0.8",
     "replicate": "^0.16.1",
     "tiktoken-node": "^0.0.6",
     "uuid": "^9.0.0",

packages/core/src/ChatEngine.ts

@@ -1,4 +1,7 @@
 import { v4 as uuidv4 } from "uuid";
+import { Event } from "./callbacks/CallbackManager";
+import { ChatHistory, SimpleChatHistory } from "./ChatHistory";
+import { ChatMessage, LLM, OpenAI } from "./llm/LLM";
 import { TextNode } from "./Node";
 import {
   CondenseQuestionPrompt,
@@ -11,8 +14,6 @@ import { BaseQueryEngine } from "./QueryEngine";
 import { Response } from "./Response";
 import { BaseRetriever } from "./Retriever";
 import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
-import { Event } from "./callbacks/CallbackManager";
-import { ChatMessage, LLM, OpenAI } from "./llm/LLM";
 
 /**
  * A ChatEngine is used to handle back and forth chats between the application and the LLM.
@@ -188,3 +189,28 @@ export class ContextChatEngine implements ChatEngine {
     this.chatHistory = [];
   }
 }
+
+/**
+ * HistoryChatEngine is a ChatEngine that uses a ChatHistory to keep track of the chat history. This is an example with the same behavior as SimpleChatEngine
+ * TODO: generally use the ChatHistory instead of ChatMessage[] - breaking change
+ */
+export class HistoryChatEngine implements ChatEngine {
+  chatHistory: ChatHistory;
+  llm: LLM;
+
+  constructor(init?: Partial<HistoryChatEngine>) {
+    this.chatHistory = init?.chatHistory ?? new SimpleChatHistory();
+    this.llm = init?.llm ?? new OpenAI();
+  }
+
+  async chat(message: string): Promise<Response> {
+    this.chatHistory.addMessage({ content: message, role: "user" });
+    const response = await this.llm.chat(this.chatHistory.messages);
+    this.chatHistory.addMessage(response.message);
+    return new Response(response.message.content);
+  }
+
+  reset() {
+    this.chatHistory.reset();
+  }
+}

packages/core/src/ChatHistory.ts

@@ -0,0 +1,73 @@
+import { ChatMessage, LLM, OpenAI } from "./llm/LLM";
+import {
+  defaultSummaryPrompt,
+  messagesToHistoryStr,
+  SummaryPrompt,
+} from "./Prompt";
+
+/**
+ * A ChatHistory is used to keep the state of back and forth chat messages
+ */
+export interface ChatHistory {
+  messages: ChatMessage[];
+  /**
+   * Adds a message to the chat history.
+   * @param message
+   */
+  addMessage(message: ChatMessage): Promise<void>;
+
+  /**
+   * Resets the chat history so that it's empty.
+   */
+  reset(): void;
+}
+
+export class SimpleChatHistory implements ChatHistory {
+  messages: ChatMessage[];
+
+  constructor(init?: Partial<SimpleChatHistory>) {
+    this.messages = init?.messages ?? [];
+  }
+
+  async addMessage(message: ChatMessage) {
+    this.messages.push(message);
+  }
+
+  reset() {
+    this.messages = [];
+  }
+}
+
+export class SummaryChatHistory implements ChatHistory {
+  messages: ChatMessage[];
+  summaryPrompt: SummaryPrompt;
+  llm: LLM;
+
+  constructor(init?: Partial<SummaryChatHistory>) {
+    this.messages = init?.messages ?? [];
+    this.summaryPrompt = init?.summaryPrompt ?? defaultSummaryPrompt;
+    this.llm = init?.llm ?? new OpenAI();
+  }
+
+  private async summarize() {
+    const chatHistoryStr = messagesToHistoryStr(this.messages);
+
+    const response = await this.llm.complete(
+      this.summaryPrompt({ context: chatHistoryStr }),
+    );
+
+    this.messages = [{ content: response.message.content, role: "system" }];
+  }
+
+  async addMessage(message: ChatMessage) {
+    // TODO: check if summarization is necessary
+    // TBD what are good conditions, e.g. depending on the context length of the LLM?
+    // for now we just have a dummy implementation at always summarizes the messages
+    await this.summarize();
+    this.messages.push(message);
+  }
+
+  reset() {
+    this.messages = [];
+  }
+}

packages/core/src/Node.ts

@@ -294,5 +294,5 @@ export function jsonToNode(json: any) {
  */
 export interface NodeWithScore {
   node: BaseNode;
-  score: number;
+  score?: number;
 }

packages/core/src/Prompt.ts

@@ -356,3 +356,34 @@ ${context}
 };
 
 export type ContextSystemPrompt = typeof defaultContextSystemPrompt;
+
+export const defaultKeywordExtractPrompt = ({
+  context = "",
+  maxKeywords = 10,
+}) => {
+  return `
+Some text is provided below. Given the text, extract up to ${maxKeywords} keywords from the text. Avoid stopwords.
+---------------------
+${context}
+---------------------
+Provide keywords in the following comma-separated format: 'KEYWORDS: <keywords>'
+`;
+};
+
+export type KeywordExtractPrompt = typeof defaultKeywordExtractPrompt;
+
+export const defaultQueryKeywordExtractPrompt = ({
+  question = "",
+  maxKeywords = 10,
+}) => {
+  return `(
+  "A question is provided below. Given the question, extract up to ${maxKeywords} "
+  "keywords from the text. Focus on extracting the keywords that we can use "
+  "to best lookup answers to the question. Avoid stopwords."
+  "---------------------"
+  "${question}"
+  "---------------------"
+  "Provide keywords in the following comma-separated format: 'KEYWORDS: <keywords>'"
+)`;
+};
+export type QueryKeywordExtractPrompt = typeof defaultQueryKeywordExtractPrompt;

packages/core/src/indices/BaseIndex.ts

@@ -39,6 +39,7 @@ export abstract class IndexStruct {
 export enum IndexStructType {
   SIMPLE_DICT = "simple_dict",
   LIST = "list",
+  KEYWORD_TABLE = "keyword_table",
 }
 
 export class IndexDict extends IndexStruct {
@@ -106,6 +107,36 @@ export class IndexList extends IndexStruct {
   }
 }
 
+// A table of keywords mapping keywords to text chunks.
+export class KeywordTable extends IndexStruct {
+  table: Map<string, Set<string>> = new Map();
+  type: IndexStructType = IndexStructType.KEYWORD_TABLE;
+  addNode(keywords: string[], nodeId: string): void {
+    keywords.forEach((keyword) => {
+      if (!this.table.has(keyword)) {
+        this.table.set(keyword, new Set());
+      }
+      this.table.get(keyword)!.add(nodeId);
+    });
+  }
+
+  deleteNode(keywords: string[], nodeId: string) {
+    keywords.forEach((keyword) => {
+      if (this.table.has(keyword)) {
+        this.table.get(keyword)!.delete(nodeId);
+      }
+    });
+  }
+
+  toJson(): Record<string, unknown> {
+    return {
+      ...super.toJson(),
+      table: this.table,
+      type: this.type,
+    };
+  }
+}
+
 export interface BaseIndexInit<T> {
   serviceContext: ServiceContext;
   storageContext: StorageContext;

packages/core/src/indices/index.ts

@@ -1,3 +1,4 @@
 export * from "./BaseIndex";
+export * from "./keyword";
 export * from "./summary";
 export * from "./vectorStore";

packages/core/src/indices/keyword/KeywordTableIndex.ts

@@ -0,0 +1,269 @@
+import { BaseNode, Document, MetadataMode } from "../../Node";
+import { defaultKeywordExtractPrompt } from "../../Prompt";
+import { BaseQueryEngine, RetrieverQueryEngine } from "../../QueryEngine";
+import { ResponseSynthesizer } from "../../ResponseSynthesizer";
+import { BaseRetriever } from "../../Retriever";
+import {
+  ServiceContext,
+  serviceContextFromDefaults,
+} from "../../ServiceContext";
+import { StorageContext, storageContextFromDefaults } from "../../storage";
+import { BaseDocumentStore } from "../../storage/docStore/types";
+import {
+  BaseIndex,
+  BaseIndexInit,
+  IndexStructType,
+  KeywordTable,
+} from "../BaseIndex";
+import {
+  KeywordTableLLMRetriever,
+  KeywordTableRAKERetriever,
+  KeywordTableSimpleRetriever,
+} from "./KeywordTableIndexRetriever";
+import { extractKeywordsGivenResponse } from "./utils";
+
+export interface KeywordIndexOptions {
+  nodes?: BaseNode[];
+  indexStruct?: KeywordTable;
+  indexId?: string;
+  serviceContext?: ServiceContext;
+  storageContext?: StorageContext;
+}
+export enum KeywordTableRetrieverMode {
+  DEFAULT = "DEFAULT",
+  SIMPLE = "SIMPLE",
+  RAKE = "RAKE",
+}
+
+const KeywordTableRetrieverMap = {
+  [KeywordTableRetrieverMode.DEFAULT]: KeywordTableLLMRetriever,
+  [KeywordTableRetrieverMode.SIMPLE]: KeywordTableSimpleRetriever,
+  [KeywordTableRetrieverMode.RAKE]: KeywordTableRAKERetriever,
+};
+
+/**
+ * The KeywordTableIndex, an index that extracts keywords from each Node and builds a mapping from each keyword to the corresponding Nodes of that keyword.
+ */
+export class KeywordTableIndex extends BaseIndex<KeywordTable> {
+  constructor(init: BaseIndexInit<KeywordTable>) {
+    super(init);
+  }
+
+  static async init(options: KeywordIndexOptions): Promise<KeywordTableIndex> {
+    const storageContext =
+      options.storageContext ?? (await storageContextFromDefaults({}));
+    const serviceContext =
+      options.serviceContext ?? serviceContextFromDefaults({});
+    const { docStore, indexStore } = storageContext;
+
+    // Setup IndexStruct from storage
+    let indexStructs = (await indexStore.getIndexStructs()) as KeywordTable[];
+    let indexStruct: KeywordTable | null;
+
+    if (options.indexStruct && indexStructs.length > 0) {
+      throw new Error(
+        "Cannot initialize index with both indexStruct and indexStore",
+      );
+    }
+
+    if (options.indexStruct) {
+      indexStruct = options.indexStruct;
+    } else if (indexStructs.length == 1) {
+      indexStruct = indexStructs[0];
+    } else if (indexStructs.length > 1 && options.indexId) {
+      indexStruct = (await indexStore.getIndexStruct(
+        options.indexId,
+      )) as KeywordTable;
+    } else {
+      indexStruct = null;
+    }
+
+    // check indexStruct type
+    if (indexStruct && indexStruct.type !== IndexStructType.KEYWORD_TABLE) {
+      throw new Error(
+        "Attempting to initialize KeywordTableIndex with non-keyword table indexStruct",
+      );
+    }
+
+    if (indexStruct) {
+      if (options.nodes) {
+        throw new Error(
+          "Cannot initialize KeywordTableIndex with both nodes and indexStruct",
+        );
+      }
+    } else {
+      if (!options.nodes) {
+        throw new Error(
+          "Cannot initialize KeywordTableIndex without nodes or indexStruct",
+        );
+      }
+      indexStruct = await KeywordTableIndex.buildIndexFromNodes(
+        options.nodes,
+        storageContext.docStore,
+        serviceContext,
+      );
+
+      await indexStore.addIndexStruct(indexStruct);
+    }
+
+    return new KeywordTableIndex({
+      storageContext,
+      serviceContext,
+      docStore,
+      indexStore,
+      indexStruct,
+    });
+  }
+
+  asRetriever(options?: any): BaseRetriever {
+    const { mode = KeywordTableRetrieverMode.DEFAULT, ...otherOptions } =
+      options ?? {};
+    const KeywordTableRetriever =
+      KeywordTableRetrieverMap[mode as KeywordTableRetrieverMode];
+    if (KeywordTableRetriever) {
+      return new KeywordTableRetriever({ index: this, ...otherOptions });
+    }
+    throw new Error(`Unknown retriever mode: ${mode}`);
+  }
+
+  asQueryEngine(options?: {
+    retriever?: BaseRetriever;
+    responseSynthesizer?: ResponseSynthesizer;
+  }): BaseQueryEngine {
+    const { retriever, responseSynthesizer } = options ?? {};
+    return new RetrieverQueryEngine(
+      retriever ?? this.asRetriever(),
+      responseSynthesizer,
+    );
+  }
+
+  static async extractKeywords(
+    text: string,
+    serviceContext: ServiceContext,
+  ): Promise<Set<string>> {
+    const response = await serviceContext.llm.complete(
+      defaultKeywordExtractPrompt({
+        context: text,
+      }),
+    );
+    return extractKeywordsGivenResponse(response.message.content, "KEYWORDS:");
+  }
+
+  /**
+   * High level API: split documents, get keywords, and build index.
+   * @param documents
+   * @param storageContext
+   * @param serviceContext
+   * @returns
+   */
+  static async fromDocuments(
+    documents: Document[],
+    args: {
+      storageContext?: StorageContext;
+      serviceContext?: ServiceContext;
+    } = {},
+  ): Promise<KeywordTableIndex> {
+    let { storageContext, serviceContext } = args;
+    storageContext = storageContext ?? (await storageContextFromDefaults({}));
+    serviceContext = serviceContext ?? serviceContextFromDefaults({});
+    const docStore = storageContext.docStore;
+
+    docStore.addDocuments(documents, true);
+    for (const doc of documents) {
+      docStore.setDocumentHash(doc.id_, doc.hash);
+    }
+
+    const nodes = serviceContext.nodeParser.getNodesFromDocuments(documents);
+    const index = await KeywordTableIndex.init({
+      nodes,
+      storageContext,
+      serviceContext,
+    });
+    return index;
+  }
+
+  /**
+   * Get keywords for nodes and place them into the index.
+   * @param nodes
+   * @param serviceContext
+   * @param vectorStore
+   * @returns
+   */
+  static async buildIndexFromNodes(
+    nodes: BaseNode[],
+    docStore: BaseDocumentStore,
+    serviceContext: ServiceContext,
+  ): Promise<KeywordTable> {
+    const indexStruct = new KeywordTable();
+    await docStore.addDocuments(nodes, true);
+    for (const node of nodes) {
+      const keywords = await KeywordTableIndex.extractKeywords(
+        node.getContent(MetadataMode.LLM),
+        serviceContext,
+      );
+      indexStruct.addNode([...keywords], node.id_);
+    }
+    return indexStruct;
+  }
+
+  async insertNodes(nodes: BaseNode[]) {
+    for (let node of nodes) {
+      const keywords = await KeywordTableIndex.extractKeywords(
+        node.getContent(MetadataMode.LLM),
+        this.serviceContext,
+      );
+      this.indexStruct.addNode([...keywords], node.id_);
+    }
+  }
+
+  deleteNode(nodeId: string): void {
+    const keywordsToDelete: Set<string> = new Set();
+    for (const [keyword, existingNodeIds] of Object.entries(
+      this.indexStruct.table,
+    )) {
+      const index = existingNodeIds.indexOf(nodeId);
+      if (index !== -1) {
+        existingNodeIds.splice(index, 1);
+
+        // Delete keywords that have zero nodes
+        if (existingNodeIds.length === 0) {
+          keywordsToDelete.add(keyword);
+        }
+      }
+    }
+    this.indexStruct.deleteNode([...keywordsToDelete], nodeId);
+  }
+
+  async deleteNodes(nodeIds: string[], deleteFromDocStore: boolean) {
+    nodeIds.forEach((nodeId) => {
+      this.deleteNode(nodeId);
+    });
+
+    if (deleteFromDocStore) {
+      for (const nodeId of nodeIds) {
+        await this.docStore.deleteDocument(nodeId, false);
+      }
+    }
+
+    await this.storageContext.indexStore.addIndexStruct(this.indexStruct);
+  }
+
+  async deleteRefDoc(
+    refDocId: string,
+    deleteFromDocStore?: boolean,
+  ): Promise<void> {
+    const refDocInfo = await this.docStore.getRefDocInfo(refDocId);
+
+    if (!refDocInfo) {
+      return;
+    }
+
+    await this.deleteNodes(refDocInfo.nodeIds, false);
+
+    if (deleteFromDocStore) {
+      await this.docStore.deleteRefDoc(refDocId, false);
+    }
+
+    return;
+  }
+}

packages/core/src/indices/keyword/KeywordTableIndexRetriever.ts

@@ -0,0 +1,119 @@
+import { NodeWithScore } from "../../Node";
+import {
+  defaultKeywordExtractPrompt,
+  defaultQueryKeywordExtractPrompt,
+  KeywordExtractPrompt,
+  QueryKeywordExtractPrompt,
+} from "../../Prompt";
+import { BaseRetriever } from "../../Retriever";
+import { ServiceContext } from "../../ServiceContext";
+import { BaseDocumentStore } from "../../storage/docStore/types";
+import { KeywordTable } from "../BaseIndex";
+import { KeywordTableIndex } from "./KeywordTableIndex";
+import {
+  extractKeywordsGivenResponse,
+  rakeExtractKeywords,
+  simpleExtractKeywords,
+} from "./utils";
+
+// Base Keyword Table Retriever
+abstract class BaseKeywordTableRetriever implements BaseRetriever {
+  protected index: KeywordTableIndex;
+  protected indexStruct: KeywordTable;
+  protected docstore: BaseDocumentStore;
+  protected serviceContext: ServiceContext;
+
+  protected maxKeywordsPerQuery: number; // Maximum number of keywords to extract from query.
+  protected numChunksPerQuery: number; // Maximum number of text chunks to query.
+  protected keywordExtractTemplate: KeywordExtractPrompt; // A Keyword Extraction Prompt
+  protected queryKeywordExtractTemplate: QueryKeywordExtractPrompt; // A Query Keyword Extraction Prompt
+
+  constructor({
+    index,
+    keywordExtractTemplate,
+    queryKeywordExtractTemplate,
+    maxKeywordsPerQuery = 10,
+    numChunksPerQuery = 10,
+  }: {
+    index: KeywordTableIndex;
+    keywordExtractTemplate?: KeywordExtractPrompt;
+    queryKeywordExtractTemplate?: QueryKeywordExtractPrompt;
+    maxKeywordsPerQuery: number;
+    numChunksPerQuery: number;
+  }) {
+    this.index = index;
+    this.indexStruct = index.indexStruct;
+    this.docstore = index.docStore;
+    this.serviceContext = index.serviceContext;
+
+    this.maxKeywordsPerQuery = maxKeywordsPerQuery;
+    this.numChunksPerQuery = numChunksPerQuery;
+    this.keywordExtractTemplate =
+      keywordExtractTemplate || defaultKeywordExtractPrompt;
+    this.queryKeywordExtractTemplate =
+      queryKeywordExtractTemplate || defaultQueryKeywordExtractPrompt;
+  }
+
+  abstract getKeywords(query: string): Promise<string[]>;
+
+  async retrieve(query: string): Promise<NodeWithScore[]> {
+    const keywords = await this.getKeywords(query);
+    const chunkIndicesCount: { [key: string]: number } = {};
+    const filteredKeywords = keywords.filter((keyword) =>
+      this.indexStruct.table.has(keyword),
+    );
+
+    for (let keyword of filteredKeywords) {
+      for (let nodeId of this.indexStruct.table.get(keyword) || []) {
+        chunkIndicesCount[nodeId] = (chunkIndicesCount[nodeId] ?? 0) + 1;
+      }
+    }
+
+    const sortedChunkIndices = Object.keys(chunkIndicesCount)
+      .sort((a, b) => chunkIndicesCount[b] - chunkIndicesCount[a])
+      .slice(0, this.numChunksPerQuery);
+
+    const sortedNodes = await this.docstore.getNodes(sortedChunkIndices);
+
+    return sortedNodes.map((node) => ({ node }));
+  }
+
+  getServiceContext(): ServiceContext {
+    return this.index.serviceContext;
+  }
+}
+
+// Extracts keywords using LLMs.
+export class KeywordTableLLMRetriever extends BaseKeywordTableRetriever {
+  async getKeywords(query: string): Promise<string[]> {
+    const response = await this.serviceContext.llm.complete(
+      this.queryKeywordExtractTemplate({
+        question: query,
+        maxKeywords: this.maxKeywordsPerQuery,
+      }),
+    );
+    const keywords = extractKeywordsGivenResponse(
+      response.message.content,
+      "KEYWORDS:",
+    );
+    return [...keywords];
+  }
+}
+
+// Extracts keywords using simple regex-based keyword extractor.
+export class KeywordTableSimpleRetriever extends BaseKeywordTableRetriever {
+  getKeywords(query: string): Promise<string[]> {
+    return Promise.resolve([
+      ...simpleExtractKeywords(query, this.maxKeywordsPerQuery),
+    ]);
+  }
+}
+
+// Extracts keywords using RAKE keyword extractor
+export class KeywordTableRAKERetriever extends BaseKeywordTableRetriever {
+  getKeywords(query: string): Promise<string[]> {
+    return Promise.resolve([
+      ...rakeExtractKeywords(query, this.maxKeywordsPerQuery),
+    ]);
+  }
+}

packages/core/src/indices/keyword/index.ts

@@ -0,0 +1,9 @@
+export {
+  KeywordTableIndex,
+  KeywordTableRetrieverMode,
+} from "./KeywordTableIndex";
+export {
+  KeywordTableLLMRetriever,
+  KeywordTableRAKERetriever,
+  KeywordTableSimpleRetriever,
+} from "./KeywordTableIndexRetriever";

packages/core/src/indices/keyword/utils.ts

@@ -0,0 +1,81 @@
+// @ts-ignore
+import rake from "rake-modified";
+
+// Get subtokens from a list of tokens., filtering for stopwords.
+export function expandTokensWithSubtokens(tokens: Set<string>): Set<string> {
+  const results: Set<string> = new Set();
+  const regex: RegExp = /\w+/g;
+
+  for (let token of tokens) {
+    results.add(token);
+    const subTokens: RegExpMatchArray | null = token.match(regex);
+    if (subTokens && subTokens.length > 1) {
+      for (let w of subTokens) {
+        results.add(w);
+      }
+    }
+  }
+  return results;
+}
+
+export function extractKeywordsGivenResponse(
+  response: string,
+  startToken: string = "",
+  lowercase: boolean = true,
+): Set<string> {
+  const results: string[] = [];
+  response = response.trim();
+
+  if (response.startsWith(startToken)) {
+    response = response.substring(startToken.length);
+  }
+
+  const keywords: string[] = response.split(",");
+  for (let k of keywords) {
+    let rk: string = k;
+    if (lowercase) {
+      rk = rk.toLowerCase();
+    }
+    results.push(rk.trim());
+  }
+
+  return expandTokensWithSubtokens(new Set(results));
+}
+
+export function simpleExtractKeywords(
+  textChunk: string,
+  maxKeywords?: number,
+): Set<string> {
+  const regex: RegExp = /\w+/g;
+  let tokens: string[] = [...textChunk.matchAll(regex)].map((token) =>
+    token[0].toLowerCase().trim(),
+  );
+
+  // Creating a frequency map
+  const valueCounts: { [key: string]: number } = {};
+  for (let token of tokens) {
+    valueCounts[token] = (valueCounts[token] || 0) + 1;
+  }
+
+  // Sorting tokens by frequency
+  const sortedTokens: string[] = Object.keys(valueCounts).sort(
+    (a, b) => valueCounts[b] - valueCounts[a],
+  );
+
+  const keywords: string[] = maxKeywords
+    ? sortedTokens.slice(0, maxKeywords)
+    : sortedTokens;
+
+  return new Set(keywords);
+}
+
+export function rakeExtractKeywords(
+  textChunk: string,
+  maxKeywords?: number,
+): Set<string> {
+  const keywords = Object.keys(rake(textChunk));
+  const limitedKeywords = maxKeywords
+    ? keywords.slice(0, maxKeywords)
+    : keywords;
+  return new Set(limitedKeywords);
+}

packages/core/src/readers/NotionReader.ts

@@ -1,271 +1,67 @@
-import { Client, collectPaginatedAPI } from "@notionhq/client";
-import * as md from "md-utils-ts";
+import { Client } from "@notionhq/client";
+import { crawler, Crawler, Pages, pageToString } from "notion-md-crawler";
 import { Document } from "../Node";
 import { BaseReader } from "./base";
 
-type NotionClient = InstanceType<typeof Client>;
+type OptionalSerializers = Parameters<Crawler>[number]["serializers"];
 
-// Notion Page
-type NotionPageRetrieveMethod = NotionClient["pages"]["retrieve"];
-type NotionPartialPageObjectResponse = Awaited<
-  ReturnType<NotionPageRetrieveMethod>
->;
-
-// Notion Block
-type NotionBlockListMethod = NotionClient["blocks"]["children"]["list"];
-type NotionBlockListResponse = Awaited<ReturnType<NotionBlockListMethod>>;
-type NotionBlockObjectResponse = NotionBlockListResponse["results"][number];
-type ExtractBlockObjectResponse<T> = T extends { type: string } ? T : never;
-type NotionBlock = ExtractBlockObjectResponse<NotionBlockObjectResponse>;
-type NotionChildPageBlock = Extract<NotionBlock, { type: "child_page" }>;
-type NotionParagraphBlock = Extract<NotionBlock, { type: "paragraph" }>;
-type NotionTableRowBlock = Extract<NotionBlock, { type: "table_row" }>;
-type NotionRichText = NotionParagraphBlock["paragraph"]["rich_text"];
-type NotionAnnotations = NotionRichText[number]["annotations"];
-
-const fetchNotionBlocks = (client: Client) => async (blockId: string) =>
-  collectPaginatedAPI(client.blocks.children.list, {
-    block_id: blockId,
-  });
-
-const fetchNotionPage = (client: Client) => (pageId: string) =>
-  client.pages.retrieve({ page_id: pageId });
-
-type Page = {
-  metadata: {
-    id: string;
-    title: string;
-    createdTime: string;
-    lastEditedTime: string;
-    parentId?: string;
-  };
-  lines: string[];
+/**
+ * Options for initializing the NotionReader class
+ * @typedef {Object} NotionReaderOptions
+ * @property {Client} client - The Notion Client object for API interactions
+ * @property {OptionalSerializers} [serializers] - Option to customize serialization. See [the url](https://github.com/TomPenguin/notion-md-crawler/tree/main) for details.
+ */
+type NotionReaderOptions = {
+  client: Client;
+  serializers?: OptionalSerializers;
 };
 
-type Pages = Record<string, Page>;
-
-const hasType = (block: NotionBlockObjectResponse): block is NotionBlock =>
-  "type" in block;
-
-const blockIs = <T extends NotionBlock["type"]>(
-  block: NotionBlock,
-  type: T,
-): block is Extract<NotionBlock, { type: T }> => block.type === type;
-
-const getCursor = (
-  pageBlock: NotionChildPageBlock,
-  parentId?: string,
-): Page => ({
-  metadata: {
-    id: pageBlock.id,
-    title: pageBlock.child_page.title,
-    createdTime: pageBlock.created_time,
-    lastEditedTime: pageBlock.last_edited_time,
-    parentId,
-  },
-  lines: [],
-});
-
-const annotateText = (text: string, annotations: NotionAnnotations) => {
-  if (annotations.code) text = md.inlineCode(text);
-  if (annotations.bold) text = md.bold(text);
-  if (annotations.italic) text = md.italic(text);
-  if (annotations.strikethrough) text = md.del(text);
-  if (annotations.underline) text = md.underline(text);
-
-  return text;
-};
-
-const richTextToString = (richText: NotionRichText) =>
-  richText
-    .map(({ plain_text, annotations, href }) => {
-      if (plain_text.match(/^\s*$/)) return plain_text;
-
-      const leadingSpaceMatch = plain_text.match(/^(\s*)/);
-      const trailingSpaceMatch = plain_text.match(/(\s*)$/);
-
-      const leading_space = leadingSpaceMatch ? leadingSpaceMatch[0] : "";
-      const trailing_space = trailingSpaceMatch ? trailingSpaceMatch[0] : "";
-
-      const text = plain_text.trim();
-
-      if (text === "") return leading_space + trailing_space;
-
-      const annotatedText = annotateText(text, annotations);
-      const linkedText = href ? md.anchor(annotatedText, href) : annotatedText;
-
-      return leading_space + linkedText + trailing_space;
-    })
-    .join("");
-
-const tableRowToString = (block: NotionTableRowBlock) =>
-  `| ${block.table_row.cells
-    .flatMap((row) => row.map((column) => richTextToString([column])))
-    .join(" | ")} |`;
-
-const blockToString = (block: NotionBlock): string => {
-  switch (block.type) {
-    case "divider":
-      return md.hr();
-    case "equation":
-      return md.equationBlock(block.equation.expression);
-    case "bookmark":
-      return md.anchor(
-        richTextToString(block.bookmark.caption),
-        block.bookmark.url,
-      );
-    case "link_preview":
-      return md.anchor(block.type, block.link_preview.url);
-    case "link_to_page":
-      const href =
-        block.link_to_page.type === "page_id" ? block.link_to_page.page_id : "";
-      return md.anchor(block.type, href);
-    case "child_page":
-      return `[${block.child_page.title}]`;
-    case "child_database":
-      return `[${block.child_database.title}]`;
-    case "paragraph":
-      return richTextToString(block.paragraph.rich_text);
-    case "heading_1":
-      return md.h1(richTextToString(block.heading_1.rich_text));
-    case "heading_2":
-      return md.h2(richTextToString(block.heading_2.rich_text));
-    case "heading_3":
-      return md.h3(richTextToString(block.heading_3.rich_text));
-    case "bulleted_list_item":
-      return md.bullet(richTextToString(block.bulleted_list_item.rich_text));
-    case "numbered_list_item":
-      return md.bullet(richTextToString(block.numbered_list_item.rich_text), 1);
-    case "quote":
-      return md.quote(richTextToString(block.quote.rich_text));
-    case "table_row":
-      return tableRowToString(block);
-    case "to_do":
-      return md.todo(
-        richTextToString(block.to_do.rich_text),
-        block.to_do.checked,
-      );
-    case "template":
-      return richTextToString(block.template.rich_text);
-    case "code":
-      return md.codeBlock(block.code.language)(
-        richTextToString(block.code.rich_text),
-      );
-    case "callout":
-      return md.quote(richTextToString(block.callout.rich_text));
-
-    case "image":
-    case "video":
-    case "audio":
-    case "file":
-    case "pdf":
-    case "table":
-    case "embed":
-    case "breadcrumb":
-    case "synced_block":
-    case "table_of_contents":
-    case "unsupported":
-    default:
-      return "";
-  }
-};
-
-const getNest = (block: NotionBlock, baseNest: number) => {
-  switch (block.type) {
-    // Reset nest
-    case "child_page":
-      return 0;
-
-    // Eliminates unnecessary nests due to NotionBlock structure
-    case "table_row":
-    case "column_list":
-    case "column":
-    case "synced_block":
-      return baseNest;
-
-    default:
-      return baseNest + 1;
-  }
-};
-
-const crawlPages =
-  (client: Client) =>
-  async (
-    blocks: NotionBlockObjectResponse[],
-    cursor: Page,
-    pages: Pages = {},
-    nest = 0,
-  ): Promise<Pages> => {
-    pages[cursor.metadata.id] = pages[cursor.metadata.id] || cursor;
-
-    for (const block of blocks) {
-      if (!hasType(block)) continue;
-
-      const line = md.indent()(blockToString(block), nest);
-      cursor.lines.push(line);
-
-      if (block.has_children) {
-        const blockId = blockIs(block, "synced_block")
-          ? block.synced_block.synced_from?.block_id || block.id
-          : block.id;
-        const childBlocks = await fetchNotionBlocks(client)(blockId);
-        const nextCursor = blockIs(block, "child_page")
-          ? getCursor(block, cursor.metadata.id)
-          : cursor;
-        const childPages = await crawlPages(client)(
-          childBlocks,
-          nextCursor,
-          pages,
-          getNest(block, nest),
-        );
-        pages = { ...pages, ...childPages };
-      }
-    }
-
-    return pages;
-  };
-
-const extractPageTitle = (page: NotionPartialPageObjectResponse) => {
-  if (!("properties" in page)) return "";
-
-  if (page.properties.title.type !== "title") return "";
-
-  return page.properties.title.title[0].plain_text;
-};
-
-const nestHeading = (text: string) => (text.match(/^#+\s/) ? "#" + text : text);
-
-const pagesToDocuments = (pages: Pages): Document[] =>
-  Object.entries(pages).map(([, { lines, metadata }]) => {
-    const title = md.h1(metadata.title);
-    const body = lines.map(nestHeading);
-    const text = [title, ...body].join("\n");
-    return new Document({ text, metadata });
-  });
-
+/**
+ * Notion pages are retrieved recursively and converted to Document objects.
+ * Notion Database can also be loaded, and [the serialization method can be customized](https://github.com/TomPenguin/notion-md-crawler/tree/main).
+ *
+ * [Note] To use this reader, must be created the Notion integration must be created in advance
+ * Please refer to [this document](https://www.notion.so/help/create-integrations-with-the-notion-api) for details.
+ */
 export class NotionReader implements BaseReader {
-  private client: Client;
+  private crawl: ReturnType<Crawler>;
 
-  constructor(options: { client: Client }) {
-    this.client = options.client;
+  /**
+   * Constructor for the NotionReader class
+   * @param {NotionReaderOptions} options - Configuration options for the reader
+   */
+  constructor({ client, serializers }: NotionReaderOptions) {
+    this.crawl = crawler({ client, serializers });
   }
 
-  async loadData(pageId: string): Promise<Document[]> {
-    const rootPage = (await fetchNotionPage(this.client)(pageId)) as any;
-    const rootPageTitle = extractPageTitle(rootPage);
-    const rootBlocks = await fetchNotionBlocks(this.client)(rootPage.id);
+  /**
+   * Converts Pages to an array of Document objects
+   * @param {Pages} pages - The Notion pages to convert (Return value of `loadPages`)
+   * @returns {Document[]} An array of Document objects
+   */
+  toDocuments(pages: Pages): Document[] {
+    return Object.values(pages).map((page) => {
+      const text = pageToString(page);
+      return new Document({ text, metadata: page.metadata });
+    });
+  }
 
-    const cursor: Page = {
-      metadata: {
-        id: rootPage.id,
-        title: rootPageTitle,
-        createdTime: rootPage.created_time,
-        lastEditedTime: rootPage.last_edited_time,
-      },
-      lines: [],
-    };
-    const pages = await crawlPages(this.client)(rootBlocks, cursor);
+  /**
+   * Loads recursively the Notion page with the specified root page ID.
+   * @param {string} rootPageId - The root Notion page ID
+   * @returns {Promise<Pages>} A Promise that resolves to a Pages object(Convertible with the `toDocuments` method)
+   */
+  async loadPages(rootPageId: string): Promise<Pages> {
+    return this.crawl(rootPageId);
+  }
 
-    return pagesToDocuments(pages);
+  /**
+   * Loads recursively Notion pages and converts them to an array of Document objects
+   * @param {string} rootPageId - The root Notion page ID
+   * @returns {Promise<Document[]>} A Promise that resolves to an array of Document objects
+   */
+  async loadData(rootPageId: string): Promise<Document[]> {
+    const pages = await this.loadPages(rootPageId);
+    return this.toDocuments(pages);
   }
 }

packages/core/src/tests/Keyword.test.ts

@@ -0,0 +1,84 @@
+import {
+  rakeExtractKeywords,
+  simpleExtractKeywords,
+} from "../indices/keyword/utils";
+describe("SimpleExtractKeywords", () => {
+  test("should extract unique keywords", () => {
+    const text = "apple banana apple cherry";
+    const result = simpleExtractKeywords(text);
+    expect(result).toEqual(new Set(["apple", "banana", "cherry"]));
+  });
+
+  test("should handle empty string", () => {
+    const text = "";
+    const result = simpleExtractKeywords(text);
+    expect(result).toEqual(new Set());
+  });
+
+  test("should handle case sensitivity", () => {
+    const text = "Apple apple";
+    const result = simpleExtractKeywords(text);
+    expect(result).toEqual(new Set(["apple"]));
+  });
+
+  test("should order keywords by frequency", () => {
+    const text = "apple banana apple cherry banana apple";
+    const result = simpleExtractKeywords(text);
+    expect([...result]).toEqual(["apple", "banana", "cherry"]);
+  });
+
+  test("should respect the maxKeywords parameter", () => {
+    const text = "apple banana apple cherry banana apple orange";
+    const result = simpleExtractKeywords(text, 2);
+    expect(result).toEqual(new Set(["apple", "banana"]));
+  });
+
+  test("should handle non-alphabetic characters", () => {
+    const text = "apple! banana... apple? cherry, orange;";
+    const result = simpleExtractKeywords(text);
+    expect(result).toEqual(new Set(["apple", "banana", "cherry", "orange"]));
+  });
+});
+
+describe("RakeExtractKeywords", () => {
+  const sampleText = `Before college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep.`;
+  test("should return all keywords if maxKeywords is not provided", () => {
+    const result = rakeExtractKeywords(sampleText);
+    expect(result).toEqual(
+      new Set([
+        "strong feelings",
+        "beginning writers",
+        "short stories",
+        "write essays",
+        "stories",
+        "write",
+        "deep",
+        "imagined",
+        "characters",
+        "plot",
+        "hardly",
+        "awful",
+        "probably",
+        "supposed",
+        "wrote",
+        "didn",
+        "programming",
+        "writing",
+        "school",
+        "outside",
+        "main",
+        "college",
+      ]),
+    );
+  });
+
+  test("should respect the maxKeywords parameter", () => {
+    const result = rakeExtractKeywords(sampleText, 2);
+    expect(result).toEqual(new Set(["strong feelings", "beginning writers"]));
+  });
+
+  test("should handle empty return from rake", () => {
+    const result = rakeExtractKeywords("");
+    expect(result).toEqual(new Set());
+  });
+});

pnpm-lock.yaml

@@ -128,9 +128,9 @@ importers:
       lodash:
         specifier: ^4.17.21
         version: 4.17.21
-      md-utils-ts:
-        specifier: ^2.0.0
-        version: 2.0.0
+      notion-md-crawler:
+        specifier: ^0.0.2
+        version: 0.0.2
       openai:
         specifier: ^4.3.1
         version: 4.3.1
@@ -140,6 +140,9 @@ importers:
       pdf-parse:
         specifier: ^1.1.1
         version: 1.1.1
+      rake-modified:
+        specifier: ^1.0.8
+        version: 1.0.8
       replicate:
         specifier: ^0.16.1
         version: 0.16.1
@@ -4670,7 +4673,6 @@ packages:
 
   /any-promise@1.3.0:
     resolution: {integrity: sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==}
-    dev: true
 
   /anymatch@3.1.3:
     resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==}
@@ -7565,6 +7567,13 @@ packages:
       jsonfile: 6.1.0
       universalify: 2.0.0
 
+  /fs-extra@2.1.2:
+    resolution: {integrity: sha512-9ztMtDZtSKC78V8mev+k31qaTabbmuH5jatdvPBMikrFHvw5BqlYnQIn/WGK3WHeRooSTkRvLa2IPlaHjPq5Sg==}
+    dependencies:
+      graceful-fs: 4.2.11
+      jsonfile: 2.4.0
+    dev: false
+
   /fs-extra@7.0.1:
     resolution: {integrity: sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==}
     engines: {node: '>=6 <7 || >=8'}
@@ -7596,6 +7605,16 @@ packages:
     resolution: {integrity: sha512-INM/fWAxMICjttnD0DX1rBvinKskj5G1w+oy/pnm9u/tSlnBrzFonJMcalKJ30P8RRsPzKcCG7Q8l0jx5Fh9YQ==}
     dev: false
 
+  /fs-promise@2.0.3:
+    resolution: {integrity: sha512-oDrTLBQAcRd+p/tSRWvqitKegLPsvqr7aehs5N9ILWFM9az5y5Uh71jKdZ/DTMC4Kel7+GNCQyFCx/IftRv8yg==}
+    deprecated: Use mz or fs-extra^3.0 with Promise Support
+    dependencies:
+      any-promise: 1.3.0
+      fs-extra: 2.1.2
+      mz: 2.7.0
+      thenify-all: 1.6.0
+    dev: false
+
   /fs.realpath@1.0.0:
     resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==}
 
@@ -9436,6 +9455,12 @@ packages:
     resolution: {integrity: sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==}
     dev: true
 
+  /jsonfile@2.4.0:
+    resolution: {integrity: sha512-PKllAqbgLgxHaj8TElYymKCAgrASebJrWpTnEkOaTowt23VKXXN0sUeriJ+eh7y6ufb/CC5ap11pz71/cM0hUw==}
+    optionalDependencies:
+      graceful-fs: 4.2.11
+    dev: false
+
   /jsonfile@4.0.0:
     resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==}
     optionalDependencies:
@@ -9984,7 +10009,6 @@ packages:
       any-promise: 1.3.0
       object-assign: 4.1.1
       thenify-all: 1.6.0
-    dev: true
 
   /nanoid@3.3.6:
     resolution: {integrity: sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==}
@@ -10180,6 +10204,15 @@ packages:
     engines: {node: '>=10'}
     dev: false
 
+  /notion-md-crawler@0.0.2:
+    resolution: {integrity: sha512-lE3/DFMrg7GSbl1sBfDuLVLyxw+yjdarPVm1JGfQ6eONEbNGgO+BdZxpwwZQ1uYeEJurAXMXb/AXT8GKYjKAyg==}
+    dependencies:
+      '@notionhq/client': 2.2.12
+      md-utils-ts: 2.0.0
+    transitivePeerDependencies:
+      - encoding
+    dev: false
+
   /npm-run-path@4.0.1:
     resolution: {integrity: sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==}
     engines: {node: '>=8'}
@@ -11376,6 +11409,13 @@ packages:
     engines: {node: '>=8'}
     dev: false
 
+  /rake-modified@1.0.8:
+    resolution: {integrity: sha512-rj/1t+EyI8Ly52eaCeSy5hoNpdNnDlNQ/+jll2DypR6nkuxotMbaupzwbuMSaXzuSL1I2pYVYy7oPus/Ls49ag==}
+    dependencies:
+      fs-promise: 2.0.3
+      lodash: 4.17.21
+    dev: false
+
   /randombytes@2.1.0:
     resolution: {integrity: sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==}
     dependencies:
@@ -12872,13 +12912,11 @@ packages:
     engines: {node: '>=0.8'}
     dependencies:
       thenify: 3.3.1
-    dev: true
 
   /thenify@3.3.1:
     resolution: {integrity: sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==}
     dependencies:
       any-promise: 1.3.0
-    dev: true
 
   /through@2.3.8:
     resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==}