cookbook: add knowledge filesystem example and sync progress logs (#10868)

## Summary

Add an AgentOS cookbook that syncs Agno documentation into
PostgreSQL-backed knowledge pages, indexes them with PgVector and OpenAI
embeddings, and exposes the `product-docs` namespace through the agent's
filesystem. The agent can search the synced documentation, and the
server starts after synchronization succeeds.

Add built-in page synchronization logs for lock waiting, discovery,
progress every 25 processed pages, and final counts. Long syncs now show
progress without a custom CLI or progress callback in the cookbook.

## Type of change

- [ ] Bug fix
- [ ] New feature
- [ ] Breaking change
- [x] Improvement
- [ ] Model update
- [x] Other: Cookbook example

---

## Checklist

- [ ] Code complies with style guidelines
- [ ] Ran format/validation scripts (`./scripts/format.sh` and
`./scripts/validate.sh`)
- [x] Self-review completed
- [x] Documentation updated (comments, docstrings)
- [x] Examples and guides: Relevant cookbook examples have been included
or updated (if applicable)
- [ ] Tested in clean environment
- [ ] Tests added/updated (if applicable)

### Duplicate and AI-Generated PR Check

- [x] I have searched existing [open pull
requests](https://github.com/agno-agi/agno/pulls) and confirmed that no
other PR already addresses this issue
- [ ] If a similar PR exists, I have explained below why this PR is a
better approach
- [x] Check if this PR was entirely AI-generated (by Copilot, Claude
Code, Cursor, etc.)

---

## Additional Notes

The cookbook uses PostgreSQL at `localhost:5532` and requires
`OPENAI_API_KEY` to sync documentation. A partial sync prints its report
and stops before starting the server. Embedding retry behavior is
unchanged.

No tests or cookbook execution were run for this PR. Formatting ran on
an isolated copy; its incidental import-order change was reverted.
Validation was stopped at the user's request, and no further checks were
run.
This commit is contained in:
Ayush Jha
2026-10-07 16:34:20 +05:30
committed by GitHub
parent 97d3f588d2
commit f1911c12c0
2 changed files with 70 additions and 1 deletions
@@ -0,0 +1,57 @@
"""Sync documentation knowledge and serve an agent with a durable filesystem."""
from agno.agent import Agent
from agno.db.postgres import PostgresDb
from agno.fs import FileSystem
from agno.knowledge.embedder.openai import OpenAIEmbedder
from agno.knowledge.knowledge import Knowledge
from agno.models.openai import OpenAIResponses
from agno.os import AgentOS
from agno.vectordb.pgvector import PgVector
db = PostgresDb(
id="filesystem-db",
db_url="postgresql+psycopg://ai:ai@localhost:5532/ai",
)
knowledge = Knowledge(
content_db=db,
page_store=FileSystem(db=db, namespace="product-docs"),
vector_db=PgVector(
db=db,
table_name="product_doc_vectors",
embedder=OpenAIEmbedder(id="text-embedding-3-small"),
),
)
filesystem_agent = Agent(
id="filesystem-agent",
name="File System Agent",
model=OpenAIResponses(id="gpt-5.6-luna"),
db=db,
filesystem=FileSystem(db=db, namespace="product-docs"),
knowledge=knowledge,
search_knowledge=True,
instructions=[
"Keep durable working notes in your filesystem.",
"Search the documentation knowledge base for Agno questions and cite sources.",
],
markdown=True,
)
agent_os = AgentOS(
id="filesystem-os",
description="AgentOS with a durable PostgreSQL filesystem.",
db=db,
agents=[filesystem_agent],
)
app = agent_os.get_app()
if __name__ == "__main__":
knowledge.setup()
report = knowledge.sync_pages(url="https://docs.agno.com/llms.txt")
print(report.model_dump_json(indent=2))
if report.status == "partial":
raise RuntimeError("Some documentation pages could not be synchronized")
agent_os.serve(app=app)
+13 -1
View File
@@ -68,7 +68,7 @@ from agno.knowledge.page.types import (
encoded_size,
)
from agno.utils.bounded import BoundedWorkers, WorkBudget
from agno.utils.log import log_info, log_warning
from agno.utils.log import log_debug, log_info, log_warning
from agno.vectordb.pgvector import PgVector
from agno.vectordb.pgvector.index import HNSW
@@ -1447,6 +1447,7 @@ class PageCoordinator:
budget = budget or WorkBudget(3900)
source = PageSource(url, public_url, budget)
updated = deleted = failed = unknown = processed = discovered = 0
log_debug(f"Starting page sync: namespace={self.namespace}; waiting for the sync lock")
progress("waiting")
errors = []
failed_paths: list[str] = []
@@ -1475,8 +1476,10 @@ class PageCoordinator:
if binding["source"] not in (None, source.url):
raise ValueError("filesystem namespace is bound to another documentation source")
self._source_attempt(conn, source, "processing")
log_debug(f"Discovering documentation pages: namespace={self.namespace}")
pages = source.discover()
discovered = len(pages)
log_debug(f"Discovered {discovered} pages; downloading, embedding, and publishing")
progress("discovered")
if validate_discovery is not None:
with conn.begin():
@@ -1541,6 +1544,11 @@ class PageCoordinator:
with conn.begin():
self._settings(conn, budget)
self._attempt(conn, page, "failed")
if processed == 1 or processed % 25 == 0 or processed == discovered:
log_debug(
f"Page sync progress: {processed}/{discovered} processed; "
f"updated={updated}, failed={failed}"
)
progress("publishing", path=page.path)
if source.complete and not errors and not failed and not unknown:
progress("pruning")
@@ -1602,6 +1610,10 @@ class PageCoordinator:
with conn.begin():
self._settings(conn, budget)
self._source_attempt(conn, source, "partial" if errors else "completed", report)
log_debug(
f"Page sync {report.status}: namespace={self.namespace}; "
f"updated={updated}, deleted={deleted}, failed={failed}, unknown={unknown}"
)
return report
except Exception as exc:
if acquired and not conn.invalidated and not conn.closed: