mirror of
https://github.com/agno-agi/agno.git
synced 2026-10-12 02:10:54 +00:00
cookbook: add knowledge filesystem example and sync progress logs (#10868)
## Summary Add an AgentOS cookbook that syncs Agno documentation into PostgreSQL-backed knowledge pages, indexes them with PgVector and OpenAI embeddings, and exposes the `product-docs` namespace through the agent's filesystem. The agent can search the synced documentation, and the server starts after synchronization succeeds. Add built-in page synchronization logs for lock waiting, discovery, progress every 25 processed pages, and final counts. Long syncs now show progress without a custom CLI or progress callback in the cookbook. ## Type of change - [ ] Bug fix - [ ] New feature - [ ] Breaking change - [x] Improvement - [ ] Model update - [x] Other: Cookbook example --- ## Checklist - [ ] Code complies with style guidelines - [ ] Ran format/validation scripts (`./scripts/format.sh` and `./scripts/validate.sh`) - [x] Self-review completed - [x] Documentation updated (comments, docstrings) - [x] Examples and guides: Relevant cookbook examples have been included or updated (if applicable) - [ ] Tested in clean environment - [ ] Tests added/updated (if applicable) ### Duplicate and AI-Generated PR Check - [x] I have searched existing [open pull requests](https://github.com/agno-agi/agno/pulls) and confirmed that no other PR already addresses this issue - [ ] If a similar PR exists, I have explained below why this PR is a better approach - [x] Check if this PR was entirely AI-generated (by Copilot, Claude Code, Cursor, etc.) --- ## Additional Notes The cookbook uses PostgreSQL at `localhost:5532` and requires `OPENAI_API_KEY` to sync documentation. A partial sync prints its report and stops before starting the server. Embedding retry behavior is unchanged. No tests or cookbook execution were run for this PR. Formatting ran on an isolated copy; its incidental import-order change was reverted. Validation was stopped at the user's request, and no further checks were run.
This commit is contained in:
@@ -0,0 +1,57 @@
|
||||
"""Sync documentation knowledge and serve an agent with a durable filesystem."""
|
||||
|
||||
from agno.agent import Agent
|
||||
from agno.db.postgres import PostgresDb
|
||||
from agno.fs import FileSystem
|
||||
from agno.knowledge.embedder.openai import OpenAIEmbedder
|
||||
from agno.knowledge.knowledge import Knowledge
|
||||
from agno.models.openai import OpenAIResponses
|
||||
from agno.os import AgentOS
|
||||
from agno.vectordb.pgvector import PgVector
|
||||
|
||||
db = PostgresDb(
|
||||
id="filesystem-db",
|
||||
db_url="postgresql+psycopg://ai:ai@localhost:5532/ai",
|
||||
)
|
||||
|
||||
knowledge = Knowledge(
|
||||
content_db=db,
|
||||
page_store=FileSystem(db=db, namespace="product-docs"),
|
||||
vector_db=PgVector(
|
||||
db=db,
|
||||
table_name="product_doc_vectors",
|
||||
embedder=OpenAIEmbedder(id="text-embedding-3-small"),
|
||||
),
|
||||
)
|
||||
|
||||
filesystem_agent = Agent(
|
||||
id="filesystem-agent",
|
||||
name="File System Agent",
|
||||
model=OpenAIResponses(id="gpt-5.6-luna"),
|
||||
db=db,
|
||||
filesystem=FileSystem(db=db, namespace="product-docs"),
|
||||
knowledge=knowledge,
|
||||
search_knowledge=True,
|
||||
instructions=[
|
||||
"Keep durable working notes in your filesystem.",
|
||||
"Search the documentation knowledge base for Agno questions and cite sources.",
|
||||
],
|
||||
markdown=True,
|
||||
)
|
||||
|
||||
|
||||
agent_os = AgentOS(
|
||||
id="filesystem-os",
|
||||
description="AgentOS with a durable PostgreSQL filesystem.",
|
||||
db=db,
|
||||
agents=[filesystem_agent],
|
||||
)
|
||||
app = agent_os.get_app()
|
||||
|
||||
if __name__ == "__main__":
|
||||
knowledge.setup()
|
||||
report = knowledge.sync_pages(url="https://docs.agno.com/llms.txt")
|
||||
print(report.model_dump_json(indent=2))
|
||||
if report.status == "partial":
|
||||
raise RuntimeError("Some documentation pages could not be synchronized")
|
||||
agent_os.serve(app=app)
|
||||
@@ -68,7 +68,7 @@ from agno.knowledge.page.types import (
|
||||
encoded_size,
|
||||
)
|
||||
from agno.utils.bounded import BoundedWorkers, WorkBudget
|
||||
from agno.utils.log import log_info, log_warning
|
||||
from agno.utils.log import log_debug, log_info, log_warning
|
||||
from agno.vectordb.pgvector import PgVector
|
||||
from agno.vectordb.pgvector.index import HNSW
|
||||
|
||||
@@ -1447,6 +1447,7 @@ class PageCoordinator:
|
||||
budget = budget or WorkBudget(3900)
|
||||
source = PageSource(url, public_url, budget)
|
||||
updated = deleted = failed = unknown = processed = discovered = 0
|
||||
log_debug(f"Starting page sync: namespace={self.namespace}; waiting for the sync lock")
|
||||
progress("waiting")
|
||||
errors = []
|
||||
failed_paths: list[str] = []
|
||||
@@ -1475,8 +1476,10 @@ class PageCoordinator:
|
||||
if binding["source"] not in (None, source.url):
|
||||
raise ValueError("filesystem namespace is bound to another documentation source")
|
||||
self._source_attempt(conn, source, "processing")
|
||||
log_debug(f"Discovering documentation pages: namespace={self.namespace}")
|
||||
pages = source.discover()
|
||||
discovered = len(pages)
|
||||
log_debug(f"Discovered {discovered} pages; downloading, embedding, and publishing")
|
||||
progress("discovered")
|
||||
if validate_discovery is not None:
|
||||
with conn.begin():
|
||||
@@ -1541,6 +1544,11 @@ class PageCoordinator:
|
||||
with conn.begin():
|
||||
self._settings(conn, budget)
|
||||
self._attempt(conn, page, "failed")
|
||||
if processed == 1 or processed % 25 == 0 or processed == discovered:
|
||||
log_debug(
|
||||
f"Page sync progress: {processed}/{discovered} processed; "
|
||||
f"updated={updated}, failed={failed}"
|
||||
)
|
||||
progress("publishing", path=page.path)
|
||||
if source.complete and not errors and not failed and not unknown:
|
||||
progress("pruning")
|
||||
@@ -1602,6 +1610,10 @@ class PageCoordinator:
|
||||
with conn.begin():
|
||||
self._settings(conn, budget)
|
||||
self._source_attempt(conn, source, "partial" if errors else "completed", report)
|
||||
log_debug(
|
||||
f"Page sync {report.status}: namespace={self.namespace}; "
|
||||
f"updated={updated}, deleted={deleted}, failed={failed}, unknown={unknown}"
|
||||
)
|
||||
return report
|
||||
except Exception as exc:
|
||||
if acquired and not conn.invalidated and not conn.closed:
|
||||
|
||||
Reference in New Issue
Block a user