mirror of
https://github.com/langchain-ai/langgraphjs.git
synced 2026-08-24 21:51:39 -04:00
Hunter Lovell
1e1ecbbcf8
Co-authored-by: Tat Dat Duong <david@duong.cz> Co-authored-by: Christian Bromann <git@bromann.dev> Co-authored-by: Nuno Campos <nuno@langchain.dev> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
119 lines
3.6 KiB
Python
119 lines
3.6 KiB
Python
"""Experimental script to generate consolidated llms text from the docs."""
|
|
|
|
import glob
|
|
import os
|
|
|
|
from mkdocs.structure.files import File
|
|
from mkdocs.structure.pages import Page
|
|
|
|
from notebook_hooks import _on_page_markdown_with_config
|
|
|
|
PARENT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
DOCS_ROOT = os.path.abspath(os.path.join(PARENT_DIR, os.pardir))
|
|
|
|
# Get source directory (parent of HERE / docs)
|
|
SOURCE_DIR = os.path.abspath(os.path.join(DOCS_ROOT, "docs"))
|
|
|
|
# langgraph-js keeps a lot of notebooks in the root directory under examples.
|
|
EXAMPLES_ROOT = os.path.abspath(os.path.join(DOCS_ROOT, os.pardir, "examples"))
|
|
|
|
|
|
def _make_llms_text(output_file: str) -> str:
|
|
"""Generate a consolidated text file from markdown/notebook files for LLM training.
|
|
|
|
Args:
|
|
output_file: Path to output the consolidated text file
|
|
"""
|
|
# Collect all markdown and notebook files
|
|
all_files = [
|
|
os.path.join(EXAMPLES_ROOT, "quickstart/quickstart.ipynb"),
|
|
]
|
|
|
|
all_files.extend(
|
|
glob.glob(os.path.join(SOURCE_DIR, "how-tos/*.md"), recursive=True)
|
|
)
|
|
|
|
all_files.extend(
|
|
glob.glob(os.path.join(SOURCE_DIR, "how-tos/*.ipynb"), recursive=True)
|
|
)
|
|
|
|
all_files.extend(
|
|
glob.glob(os.path.join(SOURCE_DIR, "tutorials/*.md"), recursive=True)
|
|
)
|
|
# Add all concepts
|
|
all_files.extend(
|
|
glob.glob(os.path.join(SOURCE_DIR, "concepts/*.md"), recursive=True)
|
|
)
|
|
all_files.extend(
|
|
glob.glob(os.path.join(SOURCE_DIR, "concepts/*.ipynb"), recursive=True)
|
|
)
|
|
|
|
# Add how-tos from examples
|
|
all_files.extend(
|
|
glob.glob(os.path.join(EXAMPLES_ROOT, "how-tos/*.ipynb"), recursive=True)
|
|
)
|
|
|
|
all_content = []
|
|
|
|
# Process each file
|
|
for abs_path in all_files:
|
|
print(f"Processing {abs_path}")
|
|
|
|
# check if abs path is in EXAMPLES_ROOT
|
|
if abs_path.startswith(EXAMPLES_ROOT):
|
|
rel_path = os.path.relpath(abs_path, EXAMPLES_ROOT)
|
|
src_dir = EXAMPLES_ROOT
|
|
elif abs_path.startswith(SOURCE_DIR):
|
|
rel_path = os.path.relpath(abs_path, SOURCE_DIR)
|
|
src_dir = SOURCE_DIR
|
|
else:
|
|
# raise an exception
|
|
raise ValueError(
|
|
f"Path {abs_path} is not in the source "
|
|
f"directory {SOURCE_DIR} or examples directory {EXAMPLES_ROOT}"
|
|
)
|
|
|
|
|
|
# Create File and Page objects to match mkdocs structure
|
|
file_obj = File(
|
|
path=rel_path, src_dir=src_dir, dest_dir="", use_directory_urls=True
|
|
)
|
|
page = Page(
|
|
title="",
|
|
file=file_obj,
|
|
config={},
|
|
)
|
|
|
|
# Read raw content
|
|
with open(abs_path, "r", encoding="utf-8") as f:
|
|
content = f.read()
|
|
|
|
# Convert to markdown without logic to resolve API references
|
|
processed_content = _on_page_markdown_with_config(
|
|
content, page, add_api_references=False, remove_base64_images=True
|
|
)
|
|
if processed_content:
|
|
# Add file name
|
|
all_content.append(f"---\n{rel_path}\n---")
|
|
# Add content
|
|
all_content.append(processed_content)
|
|
|
|
|
|
# Write consolidated output
|
|
with open(output_file, "w", encoding="utf-8") as f:
|
|
f.write("\n\n".join(all_content))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description=(
|
|
"Generate consolidated text file from markdown/notebook files for LLMs."
|
|
)
|
|
)
|
|
parser.add_argument("output_file", help="Path to output the consolidated text file")
|
|
|
|
args = parser.parse_args()
|
|
_make_llms_text(args.output_file)
|