Compare commits

...
7 Commits
125 changed files with 19223 additions and 24 deletions

No files matched your search

+3 -1
View File
@@ -80,4 +80,6 @@ CopilotKit*
issue_details.json
# MCP Registry tokens
.mcpregistry_*
.mcpregistry_*
*.log
+291 -1
View File
@@ -261,6 +261,16 @@ npm install praisonai
| ↳ 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
| ↳ Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
| ↳ Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
| ↳ Middleware System | [Example](examples/middleware/basic_middleware.py) | [📖](https://docs.praison.ai/features/middleware) |
| ↳ Configurable Model | [Example](examples/middleware/configurable_model.py) | [📖](https://docs.praison.ai/features/configurable-model) |
| ↳ Rate Limiter | [Example](examples/middleware/rate_limiter.py) | [📖](https://docs.praison.ai/features/rate-limiter) |
| ↳ Injected Tool State | [Example](examples/middleware/injected_state.py) | [📖](https://docs.praison.ai/features/injected-state) |
| ↳ Shadow Git Checkpoints | [Example](#10-shadow-git-checkpoints) | [📖](https://docs.praison.ai/features/checkpoints) |
| ↳ Background Tasks | [Example](examples/background/basic_background.py) | [📖](https://docs.praison.ai/features/background-tasks) |
| ↳ Policy Engine | [Example](examples/policy/basic_policy.py) | [📖](https://docs.praison.ai/features/policy-engine) |
| ↳ Thinking Budgets | [Example](examples/thinking/basic_thinking.py) | [📖](https://docs.praison.ai/features/thinking-budgets) |
| ↳ Output Styles | [Example](examples/output/basic_output.py) | [📖](https://docs.praison.ai/features/output-styles) |
| ↳ Context Compaction | [Example](examples/compaction/basic_compaction.py) | [📖](https://docs.praison.ai/features/context-compaction) |
| **📊 Monitoring & Management** | | |
| ↳ Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
| ↳ Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
@@ -930,8 +940,191 @@ praisonai checkpoint diff # Show changes
praisonai checkpoint restore abc123 # Restore to checkpoint
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/checkpoints)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/checkpoint)
- [💻 Example](examples/checkpoints/basic_checkpoints.py)
### 11. Field Names Reference (A-I-G-S)
---
### 11. Background Tasks
Run agent tasks asynchronously without blocking:
```python
import asyncio
from praisonaiagents.background import BackgroundRunner, BackgroundConfig
async def main():
config = BackgroundConfig(max_concurrent_tasks=3)
runner = BackgroundRunner(config=config)
async def my_task(name: str) -> str:
await asyncio.sleep(2)
return f"Task {name} completed"
task = await runner.submit(my_task, args=("example",), name="my_task")
await task.wait(timeout=10.0)
print(task.result)
asyncio.run(main())
```
**CLI Commands:**
```bash
praisonai background list # List running tasks
praisonai background status <id> # Check task status
praisonai background cancel <id> # Cancel a task
praisonai background clear # Clear completed tasks
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/background-tasks)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/background)
- [💻 Example](examples/background/basic_background.py)
---
### 12. Policy Engine
Control what agents can and cannot do with policy-based execution:
```python
from praisonaiagents.policy import (
PolicyEngine, Policy, PolicyRule, PolicyAction
)
engine = PolicyEngine()
policy = Policy(
name="no_delete",
rules=[
PolicyRule(
action=PolicyAction.DENY,
resource="tool:delete_*",
reason="Delete operations blocked"
)
]
)
engine.add_policy(policy)
result = engine.check("tool:delete_file", {})
print(f"Allowed: {result.allowed}")
```
**CLI Commands:**
```bash
praisonai policy list # List policies
praisonai policy check "tool:name" # Check if allowed
praisonai policy init # Create template
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/policy-engine)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/policy)
- [💻 Example](examples/policy/basic_policy.py)
---
### 13. Thinking Budgets
Configure token budgets for extended thinking:
```python
from praisonaiagents.thinking import ThinkingBudget, ThinkingTracker
# Use predefined levels
budget = ThinkingBudget.high() # 16,000 tokens
# Track usage
tracker = ThinkingTracker()
session = tracker.start_session(budget_tokens=16000)
tracker.end_session(session, tokens_used=12000)
summary = tracker.get_summary()
print(f"Utilization: {summary['average_utilization']:.1%}")
```
**CLI Commands:**
```bash
praisonai thinking status # Show current budget
praisonai thinking set high # Set budget level
praisonai thinking stats # Show usage statistics
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/thinking-budgets)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/thinking)
- [💻 Example](examples/thinking/basic_thinking.py)
---
### 14. Output Styles
Configure how agents format their responses:
```python
from praisonaiagents.output import OutputStyle, OutputFormatter
# Use preset styles
style = OutputStyle.concise()
formatter = OutputFormatter(style)
# Format output
text = "# Hello\n\nThis is **bold** text."
plain = formatter.format(text)
print(plain)
```
**CLI Commands:**
```bash
praisonai output status # Show current style
praisonai output set concise # Set output style
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/output-styles)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/output-style)
- [💻 Example](examples/output/basic_output.py)
---
### 15. Context Compaction
Automatically manage context window size:
```python
from praisonaiagents.compaction import (
ContextCompactor, CompactionStrategy
)
compactor = ContextCompactor(
max_tokens=4000,
strategy=CompactionStrategy.SLIDING,
preserve_recent=3
)
messages = [...] # Your conversation history
compacted, result = compactor.compact(messages)
print(f"Compression: {result.compression_ratio:.1%}")
```
**CLI Commands:**
```bash
praisonai compaction status # Show settings
praisonai compaction set sliding # Set strategy
praisonai compaction stats # Show statistics
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/context-compaction)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/compaction)
- [💻 Example](examples/compaction/basic_compaction.py)
---
### 16. Field Names Reference (A-I-G-S)
PraisonAI accepts both old (agents.yaml) and new (workflow.yaml) field names. Use the **canonical names** for new projects:
@@ -2535,6 +2728,103 @@ PraisonAI provides zero-dependency persistent memory for agents. For detailed ex
---
## 💾 Persistence (Databases)
Enable automatic conversation persistence with 2 lines of code:
```python
from praisonaiagents import Agent, db
agent = Agent(
name="Assistant",
db=db(database_url="postgresql://localhost/mydb"), # db(...) shortcut
session_id="my-session" # Optional: defaults to per-hour ID (YYYYMMDDHH)
)
agent.chat("Hello!") # Auto-persists messages, runs, traces
```
### Persistence CLI Commands
| Command | Description |
|---------|-------------|
| `praisonai persistence doctor` | Validate DB connectivity |
| `praisonai persistence run` | Run agent with persistence |
| `praisonai persistence resume` | Resume existing session |
| `praisonai persistence export` | Export session to JSONL |
| `praisonai persistence import` | Import session from JSONL |
| `praisonai persistence migrate` | Apply schema migrations |
| `praisonai persistence status` | Show schema status |
### Databases Table
| Database | Store Type | Install | Example | Docs |
|----------|------------|---------|---------|------|
| PostgreSQL | Conversation | `pip install "praisonai[tools]"` | [simple_db_agent.py](examples/persistence/simple_db_agent.py) | [docs](https://docs.praison.ai/docs/databases/postgres) |
| MySQL | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| SQLite | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| SingleStore | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Supabase | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| SurrealDB | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Qdrant | Knowledge | `pip install "praisonai[tools]"` | [knowledge_qdrant.py](examples/persistence/knowledge_qdrant.py) | [docs](https://docs.praison.ai/docs/databases/qdrant) |
| ChromaDB | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Pinecone | Knowledge | `pip install pinecone` | [pinecone_wow.py](examples/vector/pinecone_wow.py) | [docs](https://docs.praison.ai/docs/databases/pinecone) |
| Weaviate | Knowledge | `pip install weaviate-client` | [weaviate_wow.py](examples/vector/weaviate_wow.py) | [docs](https://docs.praison.ai/docs/databases/weaviate) |
| LanceDB | Knowledge | `pip install lancedb` | [lancedb_real_wow.py](examples/vector/lancedb_real_wow.py) | [docs](https://docs.praison.ai/docs/databases/lancedb) |
| Milvus | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| PGVector | Knowledge | `pip install psycopg2-binary` | [pgvector_real_wow.py](examples/vector/pgvector_real_wow.py) | [docs](https://docs.praison.ai/docs/databases/pgvector) |
| Redis Vector | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Cassandra | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| ClickHouse | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Redis | State | `pip install "praisonai[tools]"` | [state_redis.py](examples/persistence/state_redis.py) | [docs](https://docs.praison.ai/docs/databases/redis) |
| MongoDB | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| DynamoDB | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Firestore | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Upstash | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Memory | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
---
## 🔧 Tools Table
Install all tools with: `pip install "praisonai[tools]"`
| Tool | Category | Import | Docs |
|------|----------|--------|------|
| Tavily | Web Search | `from praisonai_tools import TavilyTool` | [docs](https://docs.praison.ai/docs/tools/external/tavily) |
| DuckDuckGo | Web Search | `from praisonai_tools import DuckDuckGoTool` | [docs](https://docs.praison.ai/docs/tools/external/duckduckgo) |
| Exa | Web Search | `from praisonai_tools import ExaTool` | [docs](https://docs.praison.ai/docs/tools/external/exa) |
| Serper | Web Search | `from praisonai_tools import SerperTool` | [docs](https://docs.praison.ai/docs/tools/external/serper) |
| Jina | Web Reader | `from praisonai_tools import JinaTool` | [docs](https://docs.praison.ai/docs/tools/external/jina) |
| Firecrawl | Web Scraping | `from praisonai_tools import FirecrawlTool` | [docs](https://docs.praison.ai/docs/tools/external/firecrawl) |
| Crawl4AI | Web Scraping | `from praisonai_tools import Crawl4AITool` | [docs](https://docs.praison.ai/docs/tools/external/crawl4ai) |
| Wikipedia | Knowledge | `from praisonai_tools import WikipediaTool` | [docs](https://docs.praison.ai/docs/tools/external/wikipedia) |
| ArXiv | Research | `from praisonai_tools import ArxivTool` | [docs](https://docs.praison.ai/docs/tools/external/arxiv) |
| HackerNews | News | `from praisonai_tools import HackerNewsTool` | [docs](https://docs.praison.ai/docs/tools/external/hackernews) |
| YouTube | Media | `from praisonai_tools import YouTubeTool` | [docs](https://docs.praison.ai/docs/tools/external/youtube) |
| Weather | Data | `from praisonai_tools import WeatherTool` | [docs](https://docs.praison.ai/docs/tools/external/weather) |
| PostgreSQL | Database | `from praisonai_tools import PostgresTool` | [docs](https://docs.praison.ai/docs/tools/external/postgres) |
| MySQL | Database | `from praisonai_tools import MySQLTool` | [docs](https://docs.praison.ai/docs/tools/external/mysql) |
| SQLite | Database | `from praisonai_tools import SQLiteTool` | [docs](https://docs.praison.ai/docs/tools/external/sqlite) |
| MongoDB | Database | `from praisonai_tools import MongoDBTool` | [docs](https://docs.praison.ai/docs/tools/external/mongodb) |
| Redis | Database | `from praisonai_tools import RedisTool` | [docs](https://docs.praison.ai/docs/tools/external/redis) |
| Qdrant | Vector DB | `from praisonai_tools import QdrantTool` | [docs](https://docs.praison.ai/docs/tools/external/qdrant) |
| GitHub | DevOps | `from praisonai_tools import GitHubTool` | [docs](https://docs.praison.ai/docs/tools/external/github) |
| Slack | Communication | `from praisonai_tools import SlackTool` | [docs](https://docs.praison.ai/docs/tools/external/slack) |
| Discord | Communication | `from praisonai_tools import DiscordTool` | [docs](https://docs.praison.ai/docs/tools/external/discord) |
| Telegram | Communication | `from praisonai_tools import TelegramTool` | [docs](https://docs.praison.ai/docs/tools/external/telegram) |
| Email | Communication | `from praisonai_tools import EmailTool` | [docs](https://docs.praison.ai/docs/tools/external/email) |
| Notion | Productivity | `from praisonai_tools import NotionTool` | [docs](https://docs.praison.ai/docs/tools/external/notion) |
| File | File System | `from praisonai_tools import FileTool` | [docs](https://docs.praison.ai/docs/tools/external/file) |
| Shell | System | `from praisonai_tools import ShellTool` | [docs](https://docs.praison.ai/docs/tools/external/shell) |
| Python | Code | `from praisonai_tools import PythonTool` | [docs](https://docs.praison.ai/docs/tools/external/python) |
| JSON | Data | `from praisonai_tools import JSONTool` | [docs](https://docs.praison.ai/docs/tools/external/json) |
| CSV | Data | `from praisonai_tools import CSVTool` | [docs](https://docs.praison.ai/docs/tools/external/csv) |
| Calculator | Math | `from praisonai_tools import CalculatorTool` | [docs](https://docs.praison.ai/docs/tools/external/calculator) |
> See [full tools documentation](https://docs.praison.ai/docs/tools/tools) for all 100+ available tools.
---
## 🎓 Video Tutorials
Learn PraisonAI through our comprehensive video series:
+1 -1
View File
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
# Install Python packages (using latest versions)
RUN pip install --no-cache-dir \
praisonai_tools \
"praisonai>=2.4.0" \
"praisonai>=2.5.0" \
"praisonai[chat]" \
"embedchain[github,youtube]"
+1 -1
View File
@@ -20,7 +20,7 @@ RUN mkdir -p /root/.praison
# Install Python packages (using latest versions)
RUN pip install --no-cache-dir \
praisonai_tools \
"praisonai>=2.4.0" \
"praisonai>=2.5.0" \
"praisonai[ui]" \
"praisonai[chat]" \
"praisonai[realtime]" \
+1 -1
View File
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
# Install Python packages (using latest versions)
RUN pip install --no-cache-dir \
praisonai_tools \
"praisonai>=2.4.0" \
"praisonai>=2.5.0" \
"praisonai[ui]" \
"praisonai[crewai]"
+93
View File
@@ -0,0 +1,93 @@
# Persistence CLI Commands
## Quick Reference
```bash
# Check database connectivity
praisonai persistence doctor \
--conversation-url postgresql://localhost:5432/praisonai \
--state-url redis://localhost:6379
# Run agent with persistence
praisonai persistence run \
--conversation-url postgresql://localhost:5432/praisonai \
--session-id my-session
# Resume a session
praisonai persistence resume \
--conversation-url postgresql://localhost:5432/praisonai \
--session-id my-session
# Export session to file
praisonai persistence export \
--conversation-url postgresql://localhost:5432/praisonai \
--session-id my-session \
--output session_backup.jsonl
# Import session from file
praisonai persistence import \
--conversation-url postgresql://localhost:5432/praisonai \
--file session_backup.jsonl
# Check schema status
praisonai persistence status \
--conversation-url postgresql://localhost:5432/praisonai \
--state-url redis://localhost:6379
# Apply migrations
praisonai persistence migrate \
--conversation-url postgresql://localhost:5432/praisonai \
--state-url redis://localhost:6379
```
## Environment Variables
Instead of CLI flags, you can use environment variables:
```bash
export PRAISON_CONVERSATION_URL=postgresql://localhost:5432/praisonai
export PRAISON_STATE_URL=redis://localhost:6379
export PRAISON_KNOWLEDGE_URL=http://localhost:6333
# Then run without URL flags
praisonai persistence doctor
praisonai persistence status
```
## Docker Setup
```bash
# Start all services
docker run -d --name praison-postgres \
-e POSTGRES_PASSWORD=praison123 \
-e POSTGRES_DB=praisonai \
-p 5432:5432 postgres:16
docker run -d --name praison-redis \
-p 6379:6379 redis:7
docker run -d --name praison-qdrant \
-p 6333:6333 -p 6334:6334 qdrant/qdrant
# Verify
docker ps
```
## Example Workflow
```bash
# 1. Start services
docker-compose up -d
# 2. Check connectivity
praisonai persistence doctor
# 3. Run agent
praisonai persistence run --session-id demo-session
# 4. Export for backup
praisonai persistence export --session-id demo-session --output backup.jsonl
# 5. Import to another environment
praisonai persistence import --file backup.jsonl
```
+124
View File
@@ -0,0 +1,124 @@
# PraisonAI Persistence CLI - End-to-End Guide
This guide demonstrates the CLI commands for database persistence.
## Prerequisites
```bash
pip install "praisonai[tools]"
# Start Docker containers
docker run -d --name praison-postgres -p 5432:5432 \
-e POSTGRES_PASSWORD=praison123 \
-e POSTGRES_DB=praisonai \
postgres:16
docker run -d --name praison-qdrant -p 6333:6333 qdrant/qdrant
docker run -d --name praison-redis -p 6379:6379 redis:7
```
## 1. Doctor Command - Validate Connectivity
```bash
# Check all stores
praisonai persistence doctor \
--conversation-url "postgresql://postgres:praison123@localhost:5432/praisonai" \
--knowledge-url "http://localhost:6333" \
--state-url "redis://localhost:6379"
```
**Expected Output:**
```
==================================================
PraisonAI Persistence Doctor
==================================================
[Conversation Store] Testing: postgresql://postgres:****@localhost:5432/praisonai
✅ Connected (postgres)
[Knowledge Store] Testing: http://localhost:6333
✅ Connected (qdrant)
[State Store] Testing: redis://localhost:6379
✅ Connected (redis)
==================================================
Results: 3/3 stores connected successfully
==================================================
```
## 2. Run Command - Execute Agent with Persistence
```bash
# Run agent with persistence (dry-run first)
praisonai persistence run \
--session-id "cli-demo-001" \
--conversation-url "postgresql://postgres:praison123@localhost:5432/praisonai" \
--dry-run \
"Hello, my name is Bob"
# Actually run it
praisonai persistence run \
--session-id "cli-demo-001" \
--conversation-url "postgresql://postgres:praison123@localhost:5432/praisonai" \
"Hello, my name is Bob"
```
**Expected Output:**
```
[Session: cli-demo-001]
Agent: Hello Bob! Nice to meet you. How can I help you today?
```
## 3. Resume Command - Continue a Session
```bash
# Show history
praisonai persistence resume \
--session-id "cli-demo-001" \
--conversation-url "postgresql://postgres:praison123@localhost:5432/praisonai" \
--show-history
# Continue conversation
praisonai persistence resume \
--session-id "cli-demo-001" \
--conversation-url "postgresql://postgres:praison123@localhost:5432/praisonai" \
--continue "What's my name?"
```
**Expected Output:**
```
[Session: cli-demo-001]
Messages in history: 2
--- Conversation History ---
[USER] Hello, my name is Bob
[ASSISTANT] Hello Bob! Nice to meet you...
--- End History ---
Agent: Your name is Bob!
```
## Environment Variables
Instead of passing URLs on command line, use environment variables:
```bash
export PRAISON_CONVERSATION_URL="postgresql://postgres:praison123@localhost:5432/praisonai"
export PRAISON_KNOWLEDGE_URL="http://localhost:6333"
export PRAISON_STATE_URL="redis://localhost:6379"
export OPENAI_API_KEY="your-key"
# Now commands are simpler
praisonai persistence doctor --all
praisonai persistence run --session-id "my-session" "Hello!"
```
## Full CLI Reference
```
praisonai persistence --help
praisonai persistence doctor --help
praisonai persistence run --help
praisonai persistence resume --help
```
+55
View File
@@ -0,0 +1,55 @@
"""
Basic Middleware Example - PraisonAI Agents
Demonstrates before/after hooks and wrap decorators for model and tool calls.
"""
from praisonaiagents import Agent, tool
from praisonaiagents.hooks import (
before_model, after_model, wrap_model_call,
before_tool, after_tool, wrap_tool_call,
InvocationContext, ModelRequest, ModelResponse
)
# Simple tool for demonstration
@tool
def get_weather(city: str) -> str:
"""Get weather for a city."""
return f"Sunny, 22°C in {city}"
# Before model hook - adds context
@before_model
def add_context(request: ModelRequest) -> ModelRequest:
print(f"[before_model] Adding context to request")
return request
# After model hook - logs response
@after_model
def log_response(response: ModelResponse) -> ModelResponse:
print(f"[after_model] Response received")
return response
# Wrap tool call - retry on error
@wrap_tool_call
def retry_on_error(tool_call, call_next):
print(f"[wrap_tool_call] Executing tool")
try:
return call_next(tool_call)
except Exception as e:
print(f"[wrap_tool_call] Retrying after error: {e}")
return call_next(tool_call)
# Create agent with hooks
agent = Agent(
name="WeatherBot",
instructions="You help with weather queries.",
tools=[get_weather],
hooks=[add_context, log_response, retry_on_error]
)
if __name__ == "__main__":
# Test the tool directly
result = get_weather("London")
print(f"Weather: {result}")
print("\n✓ Middleware example complete")
+31
View File
@@ -0,0 +1,31 @@
"""
Configurable Model Example - PraisonAI Agents
Demonstrates runtime model switching without recreating the agent.
"""
from praisonaiagents import Agent
# Create agent with configurable model enabled
agent = Agent(
name="FlexBot",
instructions="You are a helpful assistant.",
llm="gpt-4o-mini",
llm_config={"configurable": True}
)
if __name__ == "__main__":
# Default model call
print("Using default model (gpt-4o-mini)...")
# response = agent.chat("Say hello in 5 words")
# Override model per-call
print("\nOverriding to different model...")
# response = agent.chat("Say hello in 5 words", config={"model": "gpt-4o"})
# Override temperature
print("\nOverriding temperature...")
# response = agent.chat("Say hello creatively", config={"temperature": 0.9})
print("\n✓ Configurable model example complete")
print("Note: Uncomment agent.chat() calls to run with API key")
+50
View File
@@ -0,0 +1,50 @@
"""
Injected State Example - PraisonAI Agents
Demonstrates injecting agent state into tools without exposing it in the schema.
"""
from praisonaiagents import Agent, tool
from praisonaiagents.tools import Injected
from praisonaiagents.tools.injected import AgentState, with_injection_context
# Tool with injected state - state param is NOT in the public schema
@tool
def show_context(query: str, state: Injected[dict]) -> str:
"""Show the current agent context."""
session_id = state.get('session_id', 'unknown')
agent_id = state.get('agent_id', 'unknown')
return f"Query: {query}, Session: {session_id}, Agent: {agent_id}"
# Create agent with the tool
agent = Agent(
name="ContextBot",
instructions="You help show context information.",
tools=[show_context],
session_id="my-session-123"
)
if __name__ == "__main__":
# Verify injected param is not in schema
schema = show_context.get_schema()
params = schema['function']['parameters']['properties']
print(f"Schema params: {list(params.keys())}")
assert 'state' not in params, "state should NOT be in schema"
print("✓ 'state' correctly excluded from schema")
# Test with manual injection context
mock_state = AgentState(
agent_id="test-agent",
run_id="run-1",
session_id="session-abc"
)
with with_injection_context(mock_state):
result = show_context(query="hello")
print(f"Result: {result}")
# Test via agent.execute_tool
result = agent.execute_tool("show_context", {"query": "test"})
print(f"Agent result: {result}")
print("\n✓ Injected state example complete")
+29
View File
@@ -0,0 +1,29 @@
"""
Rate Limiter Example - PraisonAI Agents
Demonstrates token bucket rate limiting for LLM API calls.
"""
from praisonaiagents import Agent
from praisonaiagents.llm import RateLimiter
# Create rate limiter: 60 requests per minute
limiter = RateLimiter(requests_per_minute=60, burst=5)
# Create agent with rate limiter
agent = Agent(
name="RateLimitedBot",
instructions="You are a helpful assistant.",
rate_limiter=limiter
)
if __name__ == "__main__":
print(f"Rate limiter: {limiter}")
print(f"Available tokens: {limiter.available_tokens}")
# Demonstrate rate limiting
for i in range(3):
limiter.acquire()
print(f"Request {i+1} acquired, tokens left: {limiter.available_tokens:.1f}")
print("\n✓ Rate limiter example complete")
@@ -0,0 +1,122 @@
"""
Multi-Agent Persistence Example
Demonstrates how multiple agents can share or isolate conversation
history using the persistence layer.
"""
from praisonai.persistence.factory import create_conversation_store, create_state_store
from praisonai.persistence.conversation.base import ConversationSession, ConversationMessage
import uuid
def main():
# Create stores
conv_store = create_conversation_store("sqlite", path=":memory:")
state_store = create_state_store("memory")
print("=== Multi-Agent Persistence Demo ===\n")
# Scenario 1: Shared session (agents collaborate)
print("--- Scenario 1: Shared Session ---")
shared_session_id = f"shared-{uuid.uuid4().hex[:8]}"
session = ConversationSession(
session_id=shared_session_id,
user_id="user-123",
name="Shared Conversation"
)
conv_store.create_session(session)
# Agent 1 (Researcher) adds context
conv_store.add_message(shared_session_id, ConversationMessage(
role="assistant",
content="[Researcher] I found relevant information about the topic.",
metadata={"agent_id": "researcher", "agent_role": "research"}
))
# Agent 2 (Writer) uses the context
conv_store.add_message(shared_session_id, ConversationMessage(
role="assistant",
content="[Writer] Based on the research, here's a summary...",
metadata={"agent_id": "writer", "agent_role": "writing"}
))
# Agent 3 (Editor) reviews
conv_store.add_message(shared_session_id, ConversationMessage(
role="assistant",
content="[Editor] I've reviewed and polished the content.",
metadata={"agent_id": "editor", "agent_role": "editing"}
))
messages = conv_store.get_messages(shared_session_id)
print(f"Shared session has {len(messages)} messages from different agents")
for msg in messages:
agent = msg.metadata.get("agent_id", "unknown") if msg.metadata else "unknown"
print(f" - {agent}: {msg.content[:50]}...")
# Scenario 2: Isolated sessions (agents work independently)
print("\n--- Scenario 2: Isolated Sessions ---")
agent_sessions = {}
for agent_name in ["planner", "executor", "validator"]:
session_id = f"{agent_name}-{uuid.uuid4().hex[:8]}"
agent_sessions[agent_name] = session_id
conv_store.create_session(ConversationSession(
session_id=session_id,
agent_id=agent_name,
name=f"{agent_name.title()} Session"
))
# Each agent has its own conversation
conv_store.add_message(session_id, ConversationMessage(
role="user",
content=f"Task for {agent_name}"
))
conv_store.add_message(session_id, ConversationMessage(
role="assistant",
content=f"[{agent_name.title()}] Processing task..."
))
print("Created isolated sessions for each agent:")
for agent, sid in agent_sessions.items():
msgs = conv_store.get_messages(sid)
print(f" - {agent}: {len(msgs)} messages")
# Scenario 3: Shared state across agents
print("\n--- Scenario 3: Shared State ---")
# Agents share state for coordination
state_store.set("workflow:status", {"phase": "research", "progress": 0})
state_store.set("workflow:context", {"topic": "AI Agents", "deadline": "2024-12-31"})
# Agent 1 updates progress
state = state_store.get("workflow:status")
state["progress"] = 33
state["phase"] = "writing"
state_store.set("workflow:status", state)
print(f"Agent 1 updated state: {state}")
# Agent 2 reads and updates
state = state_store.get("workflow:status")
state["progress"] = 66
state["phase"] = "editing"
state_store.set("workflow:status", state)
print(f"Agent 2 updated state: {state}")
# Agent 3 completes
state = state_store.get("workflow:status")
state["progress"] = 100
state["phase"] = "complete"
state_store.set("workflow:status", state)
print(f"Agent 3 completed: {state}")
# Cleanup
conv_store.close()
state_store.close()
print("\n✅ Multi-agent persistence demo complete!")
if __name__ == "__main__":
main()
+40
View File
@@ -0,0 +1,40 @@
"""Multi-Agent Shared Session - DB Persistence Test"""
import sys
sys.path.insert(0, 'src/praisonai')
from praisonai.persistence import create_conversation_store
from praisonai.persistence.conversation.base import ConversationSession, ConversationMessage
# Create shared session for multiple agents
store = create_conversation_store("sqlite", path="/tmp/multi_agent.db")
session_id = "shared-session-123"
# Create session
session = ConversationSession(session_id=session_id, agent_id="multi")
try:
store.create_session(session)
except Exception:
pass
# Agent 1 writes
msg1 = ConversationMessage(session_id=session_id, role="user", content="Agent1: Hello")
store.add_message(session_id, msg1)
msg2 = ConversationMessage(session_id=session_id, role="assistant", content="Agent1: Hi there!")
store.add_message(session_id, msg2)
# Agent 2 writes to same session
msg3 = ConversationMessage(session_id=session_id, role="user", content="Agent2: What did Agent1 say?")
store.add_message(session_id, msg3)
msg4 = ConversationMessage(session_id=session_id, role="assistant", content="Agent2: Agent1 said Hello")
store.add_message(session_id, msg4)
# Verify both agents' messages are in session
messages = store.get_messages(session_id)
print(f"Total messages in shared session: {len(messages)}")
assert len(messages) >= 4, "Should have messages from both agents"
# Verify content
contents = [m.content for m in messages]
assert any("Agent1" in c for c in contents), "Should have Agent1 messages"
assert any("Agent2" in c for c in contents), "Should have Agent2 messages"
print("PASSED: Multi-agent shared session persistence")
@@ -0,0 +1,16 @@
"""Multi-Agent Shared Session Demo"""
from praisonaiagents import Agent
from praisonai.persistence import create_conversation_store
store = create_conversation_store("sqlite", path="/tmp/multi_agent.db")
agent1 = Agent(name="Researcher", llm="gpt-4o-mini", db=store, session_id="shared-session")
agent2 = Agent(name="Writer", llm="gpt-4o-mini", db=store, session_id="shared-session")
r1 = agent1.chat("Research topic: AI in healthcare. Key finding: reduces diagnosis time by 50%")
print(f"Researcher: {r1[:50]}...")
r2 = agent2.chat("Based on the research, what was the key finding?")
print(f"Writer: {r2[:50]}...")
assert "50" in r2 or "diagnosis" in r2.lower(), "Writer should see Researcher's context"
print("PASSED: Multi-agent shared session works")
+48
View File
@@ -0,0 +1,48 @@
"""
Basic Observability Usage Example
This example demonstrates how to use the observability module
with PraisonAI agents.
Usage:
# Set your provider's API key first
export LANGFUSE_PUBLIC_KEY=pk-lf-xxx
export LANGFUSE_SECRET_KEY=sk-lf-xxx
python basic_usage.py
"""
from praisonai_tools.observability import obs
from praisonaiagents import Agent
# Initialize observability (auto-detects provider from env vars)
obs.init()
# Or specify a provider explicitly
# obs.init(provider="langfuse", project_name="my-project")
# Create an agent
agent = Agent(
instructions="You are a helpful assistant.",
model="gpt-4o-mini",
)
# Use trace context manager for complete workflow tracing
with obs.trace("chat-session", session_id="user-123"):
# Use span context manager for individual operations
with obs.span("user-query", kind=obs.SpanKind.AGENT):
response = agent.chat("What is the capital of France?")
print(response)
# Log LLM calls manually if needed
obs.log_llm_call(
model="gpt-4o-mini",
input_messages="What is 2+2?",
output="4",
input_tokens=10,
output_tokens=1,
)
# Check diagnostics
print("\nDiagnostics:")
print(obs.doctor())
@@ -0,0 +1,48 @@
"""
Langfuse Integration Example
This example shows how to use Langfuse for LLM observability.
Setup:
1. Sign up at https://langfuse.com/
2. Get your API keys from the project settings
3. Set environment variables:
export LANGFUSE_PUBLIC_KEY=pk-lf-xxx
export LANGFUSE_SECRET_KEY=sk-lf-xxx
4. Install dependencies:
pip install opentelemetry-sdk opentelemetry-exporter-otlp
Usage:
python langfuse_example.py
"""
import os
from praisonai_tools.observability import obs
from praisonaiagents import Agent
# Initialize Langfuse
success = obs.init(
provider="langfuse",
project_name="praisonai-demo",
)
if not success:
print("Failed to initialize Langfuse. Check your API keys.")
print("Required: LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY")
exit(1)
print("Langfuse initialized successfully!")
print(f"View traces at: https://cloud.langfuse.com/")
# Create agent
agent = Agent(
instructions="You are a helpful coding assistant.",
model="gpt-4o-mini",
)
# Run with tracing
with obs.trace("coding-session", user_id="developer-1"):
response = agent.chat("Write a Python function to calculate fibonacci numbers")
print(response)
print("\nCheck Langfuse dashboard for traces!")
@@ -0,0 +1,22 @@
"""MLflow Local Observability Demo (no API key needed)"""
import mlflow
import time
mlflow.set_tracking_uri("file:///tmp/mlflow_demo")
mlflow.set_experiment("praisonai-demo")
with mlflow.start_run(run_name="agent-trace"):
mlflow.log_param("agent_name", "Assistant")
mlflow.log_param("model", "gpt-4o-mini")
# Simulate agent work
start = time.time()
time.sleep(0.1)
latency = time.time() - start
mlflow.log_metric("latency_ms", latency * 1000)
mlflow.log_metric("tokens_used", 150)
print(f"Logged run to MLflow: latency={latency*1000:.1f}ms")
print("PASSED: MLflow local observability works")
@@ -0,0 +1,53 @@
"""
Multi-Agent Tracing Example
This example shows how to trace multi-agent workflows with
parent/child span relationships.
Usage:
# Set your provider's API key
export LANGFUSE_PUBLIC_KEY=pk-lf-xxx
export LANGFUSE_SECRET_KEY=sk-lf-xxx
python multi_agent_tracing.py
"""
from praisonai_tools.observability import obs
from praisonai_tools.observability.base import SpanKind
from praisonaiagents import Agent, PraisonAIAgents
# Initialize observability
obs.init()
# Create agents
researcher = Agent(
name="Researcher",
instructions="You research topics and provide detailed information.",
model="gpt-4o-mini",
)
writer = Agent(
name="Writer",
instructions="You write clear, engaging content based on research.",
model="gpt-4o-mini",
)
# Trace the multi-agent workflow
with obs.trace("content-creation", metadata={"workflow": "research-write"}):
# Research phase
with obs.span("research-phase", kind=SpanKind.AGENT) as research_span:
research_span.attributes["agent"] = "Researcher"
research_result = researcher.chat("Research the benefits of AI in healthcare")
# Writing phase
with obs.span("writing-phase", kind=SpanKind.AGENT) as write_span:
write_span.attributes["agent"] = "Writer"
write_span.attributes["input_length"] = len(str(research_result))
final_content = writer.chat(f"Write a blog post based on: {research_result}")
print("Final Content:")
print(final_content)
# Check trace info
print("\nDiagnostics:", obs.doctor())
+79
View File
@@ -0,0 +1,79 @@
"""
Qdrant Knowledge Store Example for PraisonAI.
Demonstrates vector insert + search through the knowledge store.
Requirements:
pip install "praisonai[tools]"
Docker Setup:
docker run -d --name praison-qdrant -p 6333:6333 qdrant/qdrant
Run:
python knowledge_qdrant.py
Expected Output:
=== Qdrant Knowledge Store Demo ===
Created collection: demo_knowledge
Inserted 3 documents
Search results: 2 documents found
"""
from praisonai.persistence.factory import create_knowledge_store
from praisonai.persistence.knowledge.base import KnowledgeDocument
import random
# Create Qdrant knowledge store
store = create_knowledge_store("qdrant", url="http://localhost:6333")
collection_name = "demo_knowledge"
dimension = 384 # Embedding dimension
# Create collection
print("=== Qdrant Knowledge Store Demo ===")
if store.collection_exists(collection_name):
store.delete_collection(collection_name)
store.create_collection(collection_name, dimension=dimension)
print(f"Created collection: {collection_name}")
# Create sample documents with embeddings
random.seed(42)
documents = [
KnowledgeDocument(
id="doc-1",
content="Python is a versatile programming language used for web development, data science, and AI.",
embedding=[random.random() for _ in range(dimension)],
metadata={"category": "programming", "language": "python"}
),
KnowledgeDocument(
id="doc-2",
content="Machine learning enables computers to learn patterns from data without explicit programming.",
embedding=[random.random() for _ in range(dimension)],
metadata={"category": "ai", "topic": "ml"}
),
KnowledgeDocument(
id="doc-3",
content="PostgreSQL is a powerful open-source relational database system.",
embedding=[random.random() for _ in range(dimension)],
metadata={"category": "database", "type": "sql"}
),
]
# Insert documents
ids = store.insert(collection_name, documents)
print(f"Inserted {len(ids)} documents")
# Search for similar documents
query_embedding = [random.random() for _ in range(dimension)]
results = store.search(collection_name, query_embedding, limit=2)
print(f"Search results: {len(results)} documents found")
for doc in results:
print(f" - {doc.id}: {doc.content[:50]}...")
# Cleanup
store.delete_collection(collection_name)
store.close()
print("\n=== Demo Complete ===")
+43
View File
@@ -0,0 +1,43 @@
"""
Minimal Agent with Database Persistence
This is the simplest way to add persistence to your agent.
Messages, runs, and traces are automatically saved.
Run:
python minimal_agent_db.py
Expected output:
- Agent responds to your question
- Data is persisted to PostgreSQL
- Session can be resumed later with same session_id
"""
from praisonaiagents import Agent, db
# Create database connection (PostgreSQL + Redis)
my_db = db(
database_url="postgresql://postgres:praison123@localhost:5432/praisonai",
state_url="redis://localhost:6379" # Optional: for state/tracing
)
# Create agent with persistence
agent = Agent(
name="Assistant",
instructions="You are a helpful assistant. Be concise.",
db=my_db,
# session_id is optional - defaults to per-hour ID (YYYYMMDDHH-hash)
# Set explicitly for session continuity across runs:
session_id="my-persistent-session"
)
# Chat - messages are automatically persisted
response = agent.chat("What is the capital of France?")
print(f"Response: {response}")
# Verify persistence
data = my_db.export_session("my-persistent-session")
print(f"\nMessages stored: {len(data.get('messages', []))}")
# Clean up
my_db.close()
@@ -0,0 +1,61 @@
"""
PostgreSQL with Runs and Traces
Demonstrates full persistence with run tracking and tracing.
Run:
python postgres_runs_traces.py
Expected output:
- Agent responds with tool usage
- Runs and traces are persisted
- Export shows full session data
"""
from praisonaiagents import Agent, db
def get_weather(city: str) -> str:
"""Get weather for a city."""
return f"Weather in {city}: Sunny, 72°F"
# Create database with PostgreSQL + Redis
my_db = db(
database_url="postgresql://postgres:praison123@localhost:5432/praisonai",
state_url="redis://localhost:6379"
)
# Create agent with tools
agent = Agent(
name="WeatherBot",
instructions="You help with weather. Use the get_weather tool when asked.",
tools=[get_weather],
db=my_db,
session_id="weather-session-001"
)
# First chat - triggers run tracking
print("=== First Question ===")
response1 = agent.chat("What's the weather in Tokyo?")
print(f"Response: {response1}\n")
# Second chat - same session, new run
print("=== Second Question ===")
response2 = agent.chat("How about New York?")
print(f"Response: {response2}\n")
# Export session data
print("=== Session Data ===")
data = my_db.export_session("weather-session-001")
print(f"Session ID: {data.get('session_id')}")
print(f"Messages: {len(data.get('messages', []))}")
print(f"Agent: {data.get('agent_name')}")
# Show messages
print("\n=== Messages ===")
for msg in data.get('messages', []):
role = msg.get('role', 'unknown')
content = msg.get('content', '')[:60]
print(f" [{role}] {content}...")
my_db.close()
print("\n✅ Done")
+83
View File
@@ -0,0 +1,83 @@
"""
Redis State Store
Use Redis for state management, tracing, and caching.
Run:
python redis_state.py
Expected output:
- State operations (set/get/delete)
- Tracing operations
"""
from praisonaiagents import db
# Create Redis-only database (for state/tracing)
redis_db = db.RedisDB(host="localhost", port=6379)
print("=== Redis State Store ===")
# Test state operations
print("\n1. State Operations:")
redis_db._init_stores() # Initialize stores
# Set a value
redis_db._state_store.set("user:123:preferences", {
"theme": "dark",
"language": "en"
})
print(" Set user preferences")
# Get the value
prefs = redis_db._state_store.get("user:123:preferences")
print(f" Got preferences: {prefs}")
# Set with TTL (expires in 60 seconds)
redis_db._state_store.set("temp:token", "abc123", ttl=60)
print(" Set temporary token (60s TTL)")
# Delete
redis_db._state_store.delete("user:123:preferences")
print(" Deleted preferences")
# Test tracing
print("\n2. Tracing Operations:")
# Start a trace
trace_id = "trace-example-001"
redis_db.on_trace_start(
trace_id=trace_id,
session_id="example-session",
agent_name="ExampleAgent",
metadata={"purpose": "demo"}
)
print(f" Started trace: {trace_id}")
# Add spans
span_id = "span-llm-001"
redis_db.on_span_start(
span_id=span_id,
trace_id=trace_id,
name="llm_call",
attributes={"model": "gpt-4", "temperature": 0.7}
)
print(f" Started span: {span_id}")
redis_db.on_span_end(
span_id=span_id,
status="ok",
attributes={"tokens": 150, "latency_ms": 450}
)
print(f" Ended span: {span_id}")
# End trace
redis_db.on_trace_end(
trace_id=trace_id,
status="ok",
metadata={"total_spans": 1}
)
print(f" Ended trace: {trace_id}")
redis_db.close()
print("\n✅ Done")
+66
View File
@@ -0,0 +1,66 @@
"""
Session Resume Example
Demonstrates how to resume a conversation from a previous session.
Run:
python session_resume.py
Expected output:
- First run: Creates new session
- Second run: Resumes with history
"""
from praisonaiagents import Agent, db
# Create database
my_db = db(
database_url="postgresql://postgres:praison123@localhost:5432/praisonai",
state_url="redis://localhost:6379"
)
SESSION_ID = "resume-demo-session"
# Create agent with explicit session_id
agent = Agent(
name="MemoryBot",
instructions="You remember our conversation. Be helpful and concise.",
db=my_db,
session_id=SESSION_ID
)
# Check if this is a new or resumed session
data = my_db.export_session(SESSION_ID)
existing_messages = len(data.get('messages', []))
if existing_messages == 0:
print("=== NEW SESSION ===")
print("This is a new conversation.\n")
response = agent.chat("My name is Alice and I love Python programming.")
print(f"Response: {response}\n")
response = agent.chat("What's my favorite programming language?")
print(f"Response: {response}\n")
else:
print(f"=== RESUMED SESSION ({existing_messages} previous messages) ===")
print("Continuing previous conversation.\n")
# Show previous messages
print("Previous messages:")
for msg in data.get('messages', [])[:4]:
role = msg.get('role', 'unknown')
content = msg.get('content', '')[:60]
print(f" [{role}] {content}...")
print()
# Continue conversation
response = agent.chat("What's my name and what do I like?")
print(f"Response: {response}\n")
# Show final state
final_data = my_db.export_session(SESSION_ID)
print(f"Total messages now: {len(final_data.get('messages', []))}")
my_db.close()
print("✅ Done")
+58
View File
@@ -0,0 +1,58 @@
"""
Simple Database Persistence Example for PraisonAI Agents.
This example shows how to enable automatic conversation persistence
with just 2 lines of code.
Requirements:
pip install "praisonai[tools]"
Docker Setup:
docker run -d --name praison-postgres -p 5432:5432 \
-e POSTGRES_PASSWORD=praison123 \
-e POSTGRES_DB=praisonai \
postgres:16
Environment:
export OPENAI_API_KEY=your-key
Run:
python simple_agent_postgres.py
Expected Output:
=== First Run ===
Agent: Nice to meet you, Alice! Python is a great choice...
Session ID: demo-session-001
Messages persisted: 2
"""
from praisonaiagents import Agent, db
# Create database adapter (simplified import)
db_instance = db(
database_url="postgresql://postgres:praison123@localhost:5432/praisonai"
)
# Create agent with persistence enabled
agent = Agent(
name="Assistant",
instructions="You are a helpful assistant. Keep responses brief.",
db=db_instance,
session_id="demo-session-001", # Same session_id = resume conversation
verbose=False
)
# Chat with persistence
print("=== First Run ===")
response = agent.chat("My name is Alice and I love Python programming.")
print(f"Agent: {response}\n")
# Show session info
print(f"Session ID: {agent.session_id}")
print(f"Messages in history: {len(agent.chat_history)}")
# Close DB connection
db_instance.close()
print("\n=== Session persisted! ===")
print("Run again with same session_id to resume.")
+41
View File
@@ -0,0 +1,41 @@
"""
SQLite Local Database
Use SQLite for simple local persistence without external services.
Run:
python sqlite_local.py
Expected output:
- Agent responds
- Data persisted to local SQLite file
"""
from praisonaiagents import Agent, db
import tempfile
import os
# Create SQLite database (local file)
db_path = os.path.join(tempfile.gettempdir(), "my_agent.db")
my_db = db.SQLiteDB(path=db_path)
print(f"Database: {db_path}")
# Create agent
agent = Agent(
name="LocalBot",
instructions="You are a helpful assistant.",
db=my_db,
session_id="local-session"
)
# Chat
response = agent.chat("Hello! What can you help me with?")
print(f"Response: {response}")
# Verify
data = my_db.export_session("local-session")
print(f"\nMessages stored: {len(data.get('messages', []))}")
my_db.close()
print("✅ Done")
+67
View File
@@ -0,0 +1,67 @@
"""
Redis State Store Example for PraisonAI.
Demonstrates set/get/delete + TTL operations.
Requirements:
pip install "praisonai[tools]"
Docker Setup:
docker run -d --name praison-redis -p 6379:6379 redis:7
Run:
python state_redis.py
Expected Output:
=== Redis State Store Demo ===
Set key: session:user123:preferences
Get value: {'theme': 'dark', 'language': 'en'}
TTL remaining: 3600 seconds
Hash operations: {'name': 'Alice', 'role': 'admin'}
"""
from praisonai.persistence.factory import create_state_store
# Create Redis state store
store = create_state_store("redis", url="redis://localhost:6379")
print("=== Redis State Store Demo ===")
# Basic set/get
key = "session:user123:preferences"
value = {"theme": "dark", "language": "en", "notifications": True}
store.set(key, value)
print(f"Set key: {key}")
retrieved = store.get(key)
print(f"Get value: {retrieved}")
# Set with TTL (expires in 1 hour)
ttl_key = "session:user123:token"
store.set(ttl_key, "abc123xyz", ttl=3600)
remaining = store.ttl(ttl_key)
print(f"TTL remaining: {remaining} seconds")
# Hash operations (for structured data)
hash_key = "user:123:profile"
store.hset(hash_key, "name", "Alice")
store.hset(hash_key, "role", "admin")
store.hset(hash_key, "last_login", "2024-12-24")
profile = store.hgetall(hash_key)
print(f"Hash operations: {profile}")
# Check existence
exists = store.exists(key)
print(f"Key exists: {exists}")
# Delete keys
store.delete(key)
store.delete(ttl_key)
store.delete(hash_key)
print("Cleaned up test keys")
store.close()
print("\n=== Demo Complete ===")
+69
View File
@@ -0,0 +1,69 @@
#!/usr/bin/env python3
"""Run all examples and collect PASS/SKIP/FAIL results."""
import subprocess
import sys
import os
from pathlib import Path
EXAMPLES_DIR = Path(__file__).parent
RESULTS = []
def run_example(path: Path) -> tuple:
"""Run an example and return (name, status, reason)."""
name = f"{path.parent.name}/{path.name}"
try:
result = subprocess.run(
[sys.executable, str(path)],
capture_output=True,
text=True,
timeout=60,
env={**os.environ, "PYTHONPATH": str(EXAMPLES_DIR.parent / "src")}
)
output = result.stdout + result.stderr
if "PASSED" in output:
return (name, "PASS", "")
elif "SKIPPED" in output:
return (name, "SKIP", output.split("SKIPPED:")[-1].strip()[:50])
elif result.returncode != 0:
error = output.strip().split("\n")[-1][:80]
return (name, "FAIL", error)
else:
return (name, "PASS", "")
except subprocess.TimeoutExpired:
return (name, "FAIL", "Timeout")
except Exception as e:
return (name, "FAIL", str(e)[:50])
def main():
print("=" * 70)
print("EXAMPLE RUNNER - PASS/SKIP/FAIL SUMMARY")
print("=" * 70)
# Find all example files
examples = []
for subdir in ["db", "vector", "observability", "multi_agent", "workflows"]:
subpath = EXAMPLES_DIR / subdir
if subpath.exists():
examples.extend(sorted(subpath.glob("*_wow.py")))
print(f"\nFound {len(examples)} examples to run\n")
for ex in examples:
name, status, reason = run_example(ex)
RESULTS.append((name, status, reason))
icon = {"PASS": "✅", "SKIP": "⏭️", "FAIL": "❌"}.get(status, "?")
reason_str = f" ({reason})" if reason else ""
print(f"{icon} {status:4} | {name:45} {reason_str}")
# Summary
print("\n" + "=" * 70)
passed = sum(1 for _, s, _ in RESULTS if s == "PASS")
skipped = sum(1 for _, s, _ in RESULTS if s == "SKIP")
failed = sum(1 for _, s, _ in RESULTS if s == "FAIL")
print(f"SUMMARY: {passed} PASS | {skipped} SKIP | {failed} FAIL | {len(RESULTS)} TOTAL")
print("=" * 70)
return 0 if failed == 0 else 1
if __name__ == "__main__":
sys.exit(main())
+44
View File
@@ -0,0 +1,44 @@
"""ArXiv Tool Example.
This example demonstrates how to use the ArXiv tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
Usage:
python tool.py
Note: No API key required!
"""
from praisonai_tools import ArxivTool
def main():
# Initialize ArXiv tool (no API key needed)
arxiv = ArxivTool()
# Example 1: Search ArXiv
print("=" * 60)
print("Example 1: Search ArXiv")
print("=" * 60)
results = arxiv.search("transformer neural networks", max_results=3)
if results and "error" in results[0]:
print(f"Error: {results[0]['error']}")
else:
print(f"Found {len(results)} papers:")
for paper in results:
print(f"\n Title: {paper.get('title', 'N/A')[:60]}...")
authors = paper.get('authors', [])
if authors:
print(f" Authors: {', '.join(authors[:2])}...")
print(f" Published: {paper.get('published', 'N/A')}")
print(f" PDF: {paper.get('pdf_url', 'N/A')}")
print("\n✅ ArXiv tool working correctly!")
if __name__ == "__main__":
main()
+53
View File
@@ -0,0 +1,53 @@
"""Calculator Tool Example.
This example demonstrates how to use the Calculator tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
Usage:
python tool.py
Note: No API key required!
"""
from praisonai_tools import CalculatorTool
def main():
# Initialize Calculator tool
calc = CalculatorTool()
# Example 1: Basic arithmetic
print("=" * 60)
print("Example 1: Basic Arithmetic")
print("=" * 60)
expressions = [
"2 + 2",
"10 * 5",
"100 / 4",
"2 ** 10",
"(10 + 5) * 2 - 3"
]
for expr in expressions:
result = calc.evaluate(expr)
print(f" {expr} = {result.get('result', result)}")
# Example 2: Percentage calculation
print("\n" + "=" * 60)
print("Example 2: Percentage Calculation")
print("=" * 60)
result = calc.evaluate("250 * 0.15")
print(f" 15% of 250 = {result.get('result', result)}")
result = calc.evaluate("1000 * (1 + 0.05) ** 3")
print(f" $1000 at 5% for 3 years = ${result.get('result', 0):.2f}")
print("\n✅ Calculator tool working correctly!")
if __name__ == "__main__":
main()
+75
View File
@@ -0,0 +1,75 @@
"""DuckDuckGo Search Tool Example.
This example demonstrates how to use the DuckDuckGo search tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
Usage:
python tool.py
Note: No API key required!
"""
from praisonai_tools import DuckDuckGoTool
def main():
# Initialize DuckDuckGo tool (no API key needed)
ddg = DuckDuckGoTool()
# Example 1: Web search
print("=" * 60)
print("Example 1: Web Search")
print("=" * 60)
results = ddg.search("Python programming best practices", max_results=3)
if results and "error" in results[0]:
print(f"Error: {results[0]['error']}")
else:
print(f"Found {len(results)} results:")
for r in results:
print(f" - {r['title']}")
print(f" URL: {r['url']}")
print(f" Snippet: {r['snippet'][:100]}...")
print()
# Example 2: News search
print("=" * 60)
print("Example 2: News Search")
print("=" * 60)
news = ddg.news("artificial intelligence", max_results=3)
if news and "error" in news[0]:
print(f"Error: {news[0]['error']}")
else:
print(f"Found {len(news)} news articles:")
for n in news:
print(f" - {n['title']}")
print(f" Source: {n.get('source', 'N/A')}")
print(f" Date: {n.get('date', 'N/A')}")
print()
# Example 3: Image search
print("=" * 60)
print("Example 3: Image Search")
print("=" * 60)
images = ddg.images("sunset landscape", max_results=3)
if images and "error" in images[0]:
print(f"Error: {images[0]['error']}")
else:
print(f"Found {len(images)} images:")
for img in images:
print(f" - {img['title']}")
print(f" Image URL: {img['image_url'][:60]}...")
print()
print("✅ DuckDuckGo tool working correctly!")
if __name__ == "__main__":
main()
+48
View File
@@ -0,0 +1,48 @@
"""Exa Search Tool Example.
This example demonstrates how to use the Exa search tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
export EXA_API_KEY=your_api_key
Usage:
python tool.py
"""
import os
from praisonai_tools import ExaTool
def main():
# Check for API key
if not os.getenv("EXA_API_KEY"):
print("Error: EXA_API_KEY environment variable not set")
print("Get your API key from https://exa.ai/")
return
# Initialize Exa tool
exa = ExaTool()
# Example 1: Neural search
print("=" * 60)
print("Example 1: Neural Search")
print("=" * 60)
results = exa.search("best practices for Python development", num_results=3)
if results and "error" in results[0]:
print(f"Error: {results[0]['error']}")
else:
print(f"Found {len(results)} results:")
for r in results:
print(f" - {r.get('title', 'N/A')[:50]}...")
print(f" URL: {r.get('url', 'N/A')[:60]}...")
print(f" Score: {r.get('score', 'N/A')}")
print()
print("✅ Exa tool working correctly!")
if __name__ == "__main__":
main()
+45
View File
@@ -0,0 +1,45 @@
"""Firecrawl Tool Example.
This example demonstrates how to use the Firecrawl tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
export FIRECRAWL_API_KEY=your_api_key
Usage:
python tool.py
"""
import os
from praisonai_tools import FirecrawlTool
def main():
# Check for API key
if not os.getenv("FIRECRAWL_API_KEY"):
print("Error: FIRECRAWL_API_KEY environment variable not set")
print("Get your API key from https://firecrawl.dev/")
return
# Initialize Firecrawl tool
firecrawl = FirecrawlTool()
# Example 1: Scrape a page
print("=" * 60)
print("Example 1: Scrape a Page")
print("=" * 60)
result = firecrawl.scrape("https://example.com")
if "error" in result:
print(f"Error: {result['error']}")
else:
print(f"URL: {result.get('url', 'N/A')}")
markdown = result.get('markdown', '')
print(f"Content preview: {markdown[:300]}...")
print("\n✅ Firecrawl tool working correctly!")
if __name__ == "__main__":
main()
+42
View File
@@ -0,0 +1,42 @@
"""Hacker News Tool Example.
This example demonstrates how to use the Hacker News tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
Usage:
python tool.py
Note: No API key required!
"""
from praisonai_tools import HackerNewsTool
def main():
# Initialize Hacker News tool
hn = HackerNewsTool()
# Example 1: Get top stories
print("=" * 60)
print("Example 1: Top Stories")
print("=" * 60)
stories = hn.get_top_stories(limit=5)
if stories and "error" in stories[0]:
print(f"Error: {stories[0]['error']}")
else:
print(f"Found {len(stories)} top stories:")
for story in stories:
print(f" - {story.get('title', 'N/A')[:50]}...")
print(f" Score: {story.get('score', 'N/A')} | Comments: {story.get('descendants', 'N/A')}")
print(f" URL: {story.get('url', 'N/A')[:60]}...")
print()
print("✅ Hacker News tool working correctly!")
if __name__ == "__main__":
main()
+40
View File
@@ -0,0 +1,40 @@
"""Jina Reader Tool Example.
This example demonstrates how to use the Jina Reader tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
Usage:
python tool.py
Note: Works without API key (with rate limits)
"""
from praisonai_tools import JinaTool
def main():
# Initialize Jina tool (works without API key)
jina = JinaTool()
# Example 1: Read a URL
print("=" * 60)
print("Example 1: Read URL Content")
print("=" * 60)
result = jina.read("https://example.com")
if "error" in result:
print(f"Error: {result['error']}")
else:
print(f"URL: {result.get('url', 'N/A')}")
print(f"Title: {result.get('title', 'N/A')}")
content = result.get('content', '')
print(f"Content preview: {content[:300]}...")
print("\n✅ Jina tool working correctly!")
if __name__ == "__main__":
main()
+56
View File
@@ -0,0 +1,56 @@
"""PostgreSQL Tool Example.
This example demonstrates how to use the PostgreSQL tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
Docker: docker run -d --name postgres -e POSTGRES_PASSWORD=praison123 -e POSTGRES_DB=praisonai -p 5432:5432 postgres:16
Usage:
python tool.py
"""
from praisonai_tools import PostgresTool
def main():
# Initialize PostgreSQL tool
pg = PostgresTool(
host="localhost",
port=5432,
database="praisonai",
user="postgres",
password="praison123"
)
# Example 1: List tables
print("=" * 60)
print("Example 1: List Tables")
print("=" * 60)
tables = pg.list_tables()
if isinstance(tables, dict) and "error" in tables:
print(f"Error: {tables['error']}")
else:
print(f"Found {len(tables)} tables:")
for t in tables[:10]:
print(f" - {t.get('table_name', t)}")
# Example 2: Query
print("\n" + "=" * 60)
print("Example 2: Execute Query")
print("=" * 60)
result = pg.query("SELECT version()")
if isinstance(result, dict) and "error" in result:
print(f"Error: {result['error']}")
else:
print(f"PostgreSQL version: {result}")
print("\n✅ PostgreSQL tool working correctly!")
if __name__ == "__main__":
main()
+47
View File
@@ -0,0 +1,47 @@
"""Serper Tool Example.
This example demonstrates how to use the Serper (Google Search) tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
export SERPER_API_KEY=your_api_key
Usage:
python tool.py
"""
import os
from praisonai_tools import SerperTool
def main():
# Check for API key
if not os.getenv("SERPER_API_KEY"):
print("Error: SERPER_API_KEY environment variable not set")
print("Get your API key from https://serper.dev/")
return
# Initialize Serper tool
serper = SerperTool()
# Example 1: Web search
print("=" * 60)
print("Example 1: Google Web Search")
print("=" * 60)
results = serper.search("Python programming tutorials", max_results=3)
if results and "error" in results[0]:
print(f"Error: {results[0]['error']}")
else:
print(f"Found {len(results)} results:")
for r in results:
print(f" - {r.get('title', 'N/A')[:50]}...")
print(f" URL: {r.get('link', 'N/A')[:60]}...")
print()
print("✅ Serper tool working correctly!")
if __name__ == "__main__":
main()
+57
View File
@@ -0,0 +1,57 @@
"""Tavily Search Tool Example.
This example demonstrates how to use the Tavily search tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
export TAVILY_API_KEY=your_api_key
Usage:
python tool.py
"""
import os
from praisonai_tools import TavilyTool
def main():
# Check for API key
if not os.getenv("TAVILY_API_KEY"):
print("Error: TAVILY_API_KEY environment variable not set")
print("Get your API key from https://tavily.com/")
return
# Initialize Tavily tool
tavily = TavilyTool()
# Example 1: Basic search
print("=" * 60)
print("Example 1: Basic Search")
print("=" * 60)
results = tavily.search("What is quantum computing?", max_results=3)
if "error" in results:
print(f"Error: {results['error']}")
else:
print(f"Query: {results.get('query')}")
if "answer" in results:
print(f"\nAI Answer: {results['answer'][:200]}...")
print(f"\nResults ({len(results.get('results', []))}):")
for r in results.get("results", []):
print(f" - {r['title']}")
print(f" URL: {r['url']}")
print(f" Score: {r.get('score', 'N/A')}")
# Example 2: Search context for RAG
print("\n" + "=" * 60)
print("Example 2: Search Context (for RAG)")
print("=" * 60)
context = tavily.search_context("Python best practices 2024")
print(f"Context (first 300 chars): {context[:300]}...")
print("\n✅ Tavily tool working correctly!")
if __name__ == "__main__":
main()
+69
View File
@@ -0,0 +1,69 @@
"""Wikipedia Tool Example.
This example demonstrates how to use the Wikipedia tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
Usage:
python tool.py
Note: No API key required!
"""
from praisonai_tools import WikipediaTool
def main():
# Initialize Wikipedia tool (no API key needed)
wiki = WikipediaTool()
# Example 1: Search Wikipedia
print("=" * 60)
print("Example 1: Search Wikipedia")
print("=" * 60)
results = wiki.search("Python programming", max_results=5)
if results and "error" in results[0]:
print(f"Error: {results[0]['error']}")
else:
print(f"Found {len(results)} results:")
for r in results:
print(f" - {r['title']}")
# Example 2: Get page summary
print("\n" + "=" * 60)
print("Example 2: Get Page Summary")
print("=" * 60)
summary = wiki.summary("Artificial intelligence", sentences=3)
if "error" in summary:
print(f"Error: {summary['error']}")
else:
print(f"Title: {summary['title']}")
print(f"Summary: {summary['summary'][:300]}...")
# Example 3: Get full page
print("\n" + "=" * 60)
print("Example 3: Get Full Page")
print("=" * 60)
page = wiki.get_page("Machine learning")
if "error" in page:
print(f"Error: {page['error']}")
if "options" in page:
print(f"Options: {page['options'][:5]}")
else:
print(f"Title: {page['title']}")
print(f"URL: {page['url']}")
print(f"Categories: {page['categories'][:3]}")
print(f"Content preview: {page['content'][:200]}...")
print("\n✅ Wikipedia tool working correctly!")
if __name__ == "__main__":
main()
+21
View File
@@ -0,0 +1,21 @@
"""ChromaDB Vector Store - Basic Test"""
import sys
sys.path.insert(0, 'src/praisonai')
from praisonai.persistence import create_knowledge_store
from praisonai.persistence.knowledge.base import KnowledgeDocument
store = create_knowledge_store("chroma", path="/tmp/chroma_demo")
try:
store.delete_collection("demo")
except Exception:
pass
store.create_collection("demo", dimension=1536)
docs = [
KnowledgeDocument(id="1", content="Python is a programming language", embedding=[0.1]*1536),
KnowledgeDocument(id="2", content="JavaScript runs in browsers", embedding=[0.2]*1536)
]
store.insert("demo", docs)
results = store.search("demo", query_embedding=[0.1]*1536, limit=1)
print(f"Found: {len(results)} results")
assert len(results) >= 1
print("PASSED: ChromaDB vector store")
+31
View File
@@ -0,0 +1,31 @@
"""LanceDB Vector Store - Local Test"""
import sys
import os
import shutil
try:
import lancedb
except ImportError:
print("SKIPPED: LanceDB - lancedb not installed")
sys.exit(0)
# Clean up previous test
db_path = "/tmp/lancedb_test"
if os.path.exists(db_path):
shutil.rmtree(db_path)
db = lancedb.connect(db_path)
# Create table with vectors
data = [
{"id": "1", "text": "Machine learning is AI", "vector": [0.1] * 128},
{"id": "2", "text": "Deep learning uses neural nets", "vector": [0.2] * 128},
]
table = db.create_table("demo", data)
# Search
results = table.search([0.1] * 128).limit(1).to_list()
print(f"Found: {len(results)} results")
print(f"Top result: {results[0]['text'][:30]}...")
assert len(results) >= 1
print("PASSED: LanceDB vector store")
+20
View File
@@ -0,0 +1,20 @@
"""LanceDB Vector Store - Basic Test"""
import sys
sys.path.insert(0, 'src/praisonai')
try:
from praisonai.persistence import create_knowledge_store
from praisonai.persistence.knowledge.base import KnowledgeDocument
store = create_knowledge_store("lancedb", path="/tmp/lancedb_demo")
store.create_collection("demo", dimension=1536)
docs = [
KnowledgeDocument(id="doc1", content="The Eiffel Tower is in Paris", embedding=[0.1]*1536),
KnowledgeDocument(id="doc2", content="The Statue of Liberty is in New York", embedding=[0.2]*1536)
]
store.insert("demo", docs)
results = store.search("demo", query_embedding=[0.1]*1536, limit=1)
print(f"Found: {len(results)} results")
assert len(results) >= 1
print("PASSED: LanceDB vector store")
except ImportError as e:
print(f"SKIPPED: LanceDB - {e}")
+59
View File
@@ -0,0 +1,59 @@
"""PGVector Vector Store - Docker Test"""
import sys
import os
# Docker: docker run -d --name pgvector -e POSTGRES_PASSWORD=postgres -p 5433:5432 pgvector/pgvector:pg16
url = os.getenv("PGVECTOR_URL", "postgresql://postgres:postgres@localhost:5433/postgres")
try:
import psycopg2
except ImportError:
print("SKIPPED: PGVector - psycopg2 not installed")
sys.exit(0)
try:
conn = psycopg2.connect(url)
cur = conn.cursor()
# Ensure extension
cur.execute("CREATE EXTENSION IF NOT EXISTS vector")
conn.commit()
# Create test table
cur.execute("DROP TABLE IF EXISTS test_vectors")
cur.execute("""
CREATE TABLE test_vectors (
id SERIAL PRIMARY KEY,
content TEXT,
embedding vector(128)
)
""")
conn.commit()
# Insert vectors
cur.execute("""
INSERT INTO test_vectors (content, embedding) VALUES
('Machine learning is AI', %s),
('Deep learning uses neural nets', %s)
""", ([0.1] * 128, [0.2] * 128))
conn.commit()
# Search (cosine similarity)
cur.execute("""
SELECT content, embedding <=> %s::vector AS distance
FROM test_vectors
ORDER BY distance
LIMIT 1
""", ([0.1] * 128,))
row = cur.fetchone()
print(f"Top result: {row[0][:30]}... (distance: {row[1]:.4f})")
# Cleanup
cur.execute("DROP TABLE test_vectors")
conn.commit()
conn.close()
print("PASSED: PGVector vector store")
except Exception as e:
print(f"SKIPPED: PGVector - {e}")
+23
View File
@@ -0,0 +1,23 @@
"""PGVector Knowledge Store - Basic Test (requires Docker)"""
import sys
import os
sys.path.insert(0, 'src/praisonai')
from praisonai.persistence import create_knowledge_store
from praisonai.persistence.knowledge.base import KnowledgeDocument
# Docker: docker run -d --name pgvector -e POSTGRES_PASSWORD=postgres -p 5433:5432 pgvector/pgvector:pg16
url = os.getenv("PGVECTOR_URL", "postgresql://postgres:postgres@localhost:5433/postgres")
try:
store = create_knowledge_store("pgvector", url=url)
store.create_collection("demo", dimension=1536)
docs = [
KnowledgeDocument(id="1", content="Kubernetes orchestrates containers", embedding=[0.1]*1536),
KnowledgeDocument(id="2", content="Docker packages applications", embedding=[0.2]*1536)
]
store.insert("demo", docs)
results = store.search("demo", query_embedding=[0.1]*1536, limit=1)
print(f"Found: {len(results)} results")
assert len(results) >= 1
print("PASSED: PGVector vector store")
except Exception as e:
print(f"SKIPPED: PGVector - {e}")
+27
View File
@@ -0,0 +1,27 @@
"""Pinecone Vector Store - Real API Test"""
import os
import sys
sys.path.insert(0, 'src/praisonai')
# Requires: export PINECONE_API_KEY=...
api_key = os.getenv("PINECONE_API_KEY")
if not api_key:
print("SKIPPED: Pinecone - PINECONE_API_KEY not set")
sys.exit(0)
from pinecone import Pinecone
pc = Pinecone(api_key=api_key)
indexes = pc.list_indexes()
print(f"Pinecone connected! Found {len(list(indexes))} indexes")
# Use existing 'test' index or create one
index_name = "test"
if index_name not in [idx.name for idx in pc.list_indexes()]:
print(f"SKIPPED: Pinecone - index '{index_name}' not found")
sys.exit(0)
index = pc.Index(index_name)
stats = index.describe_index_stats()
print(f"Index '{index_name}' has {stats.total_vector_count} vectors")
print("PASSED: Pinecone vector store")
+23
View File
@@ -0,0 +1,23 @@
"""Qdrant Vector Store - Basic Test (requires Docker)"""
import sys, os
sys.path.insert(0, 'src/praisonai')
from praisonai.persistence import create_knowledge_store
from praisonai.persistence.knowledge.base import KnowledgeDocument
# Docker: docker run -d --name qdrant -p 6333:6333 qdrant/qdrant
url = os.getenv("QDRANT_URL", "http://localhost:6333")
store = create_knowledge_store("qdrant", url=url)
try:
store.delete_collection("demo")
except Exception:
pass
store.create_collection("demo", dimension=1536)
docs = [
KnowledgeDocument(id="1", content="Machine learning is AI", embedding=[0.1]*1536),
KnowledgeDocument(id="2", content="Deep learning uses neural nets", embedding=[0.2]*1536)
]
store.insert("demo", docs)
results = store.search("demo", query_embedding=[0.1]*1536, limit=1)
print(f"Found: {len(results)} results")
assert len(results) >= 1
print("PASSED: Qdrant vector store")
+24
View File
@@ -0,0 +1,24 @@
"""Weaviate Cloud Vector Store - Real API Test"""
import os
import sys
# Requires: export WEAVIATE_URL=... and WEAVIATE_API_KEY=...
weaviate_url = os.getenv("WEAVIATE_URL", "https://thhoyczshsawyt16ot3uw.c0.europe-west3.gcp.weaviate.cloud")
weaviate_key = os.getenv("WEAVIATE_API_KEY")
if not weaviate_key:
print("SKIPPED: Weaviate - WEAVIATE_API_KEY not set")
sys.exit(0)
import weaviate
from weaviate.classes.init import Auth
client = weaviate.connect_to_weaviate_cloud(
cluster_url=weaviate_url,
auth_credentials=Auth.api_key(weaviate_key)
)
print(f"Weaviate connected: {client.is_ready()}")
collections = client.collections.list_all()
print(f"Found {len(collections)} collections")
client.close()
print("PASSED: Weaviate cloud vector store")
@@ -54,6 +54,8 @@ except ImportError:
MCP = None
from .session import Session
from .memory.memory import Memory
from .db import db
from .obs import obs
# Workflows - import from dedicated workflows module
from .workflows import (
Workflow, WorkflowStep, WorkflowContext, StepResult,
@@ -257,6 +259,8 @@ __all__ = [
'AutoAgents',
'Session',
'Memory',
'db',
'obs',
'display_interaction',
'display_self_reflection',
'display_instruction',
@@ -252,7 +252,12 @@ class Agent:
history_in_context: Optional[int] = None,
auto_save: Optional[str] = None,
skills: Optional[List[str]] = None,
skills_dirs: Optional[List[str]] = None
skills_dirs: Optional[List[str]] = None,
db: Optional[Any] = None,
session_id: Optional[str] = None,
hooks: Optional[List[Any]] = None,
llm_config: Optional[Dict[str, Any]] = None,
rate_limiter: Optional[Any] = None
):
"""Initialize an Agent instance.
@@ -393,6 +398,17 @@ class Agent:
# Flag to track if final result has been displayed to prevent duplicates
self._final_display_shown = False
# Store hooks for middleware system (zero overhead when empty)
self._hooks = hooks or []
self._middleware_manager = None # Lazy init
# Store llm_config for configurable model switching
self._llm_config = llm_config or {}
self._llm_configurable = self._llm_config.get('configurable', False)
# Store rate limiter (optional, zero overhead when None)
self._rate_limiter = rate_limiter
# Store OpenAI client parameters for lazy initialization
self._openai_api_key = api_key
self._openai_base_url = base_url
@@ -615,6 +631,11 @@ Your Goal: {self.goal}
self._skill_manager = None # Lazy loaded
self._skills_initialized = False
# Database persistence (lazy - no imports until used)
self._db = db
self._session_id = session_id
self._db_initialized = False
@property
def console(self):
"""Lazily initialize Rich Console only when needed."""
@@ -1615,8 +1636,32 @@ Your Goal: {self.goal}"""
def execute_tool(self, function_name, arguments):
"""
Execute a tool dynamically based on the function name and arguments.
Injects agent state for tools with Injected[T] parameters.
"""
logging.debug(f"{self.name} executing tool {function_name} with arguments: {arguments}")
# Set up injection context for tools with Injected parameters
from ..tools.injected import AgentState, with_injection_context
state = AgentState(
agent_id=self.name,
run_id=getattr(self, '_current_run_id', 'unknown'),
session_id=getattr(self, '_session_id', None) or 'default',
last_user_message=self.chat_history[-1].get('content') if self.chat_history else None,
metadata={'agent_name': self.name}
)
# Execute within injection context
return self._execute_tool_with_context(function_name, arguments, state)
def _execute_tool_with_context(self, function_name, arguments, state):
"""Execute tool within injection context."""
from ..tools.injected import with_injection_context
with with_injection_context(state):
return self._execute_tool_impl(function_name, arguments)
def _execute_tool_impl(self, function_name, arguments):
"""Internal tool execution implementation."""
# Check if approval is required for this tool
from ..approval import is_approval_required, console_approval_callback, get_risk_level, mark_approved, ApprovalDecision, get_approval_callback
@@ -1957,7 +2002,112 @@ Your Goal: {self.goal}"""
# expand=False
# )
def chat(self, prompt, temperature=1.0, tools=None, output_json=None, output_pydantic=None, reasoning_steps=False, stream=None, task_name=None, task_description=None, task_id=None):
def _init_db_session(self):
"""Initialize DB session if db adapter is provided (lazy, first chat only)."""
if self._db is None or self._db_initialized:
return
# Generate session_id if not provided: default to per-hour ID (YYYYMMDDHH-agentname)
if self._session_id is None:
import hashlib
from datetime import datetime, timezone
# Per-hour session ID: YYYYMMDDHH (UTC) + agent name hash for uniqueness
hour_str = datetime.now(timezone.utc).strftime("%Y%m%d%H")
agent_hash = hashlib.md5(self.name.encode()).hexdigest()[:6]
self._session_id = f"{hour_str}-{agent_hash}"
# Call db adapter's on_agent_start to get previous messages
try:
history = self._db.on_agent_start(
agent_name=self.name,
session_id=self._session_id,
user_id=self.user_id,
metadata={"role": self.role, "goal": self.goal}
)
# Restore chat history from previous session
if history:
for msg in history:
self.chat_history.append({
"role": msg.role,
"content": msg.content
})
logging.info(f"Resumed session {self._session_id} with {len(history)} messages")
except Exception as e:
logging.warning(f"Failed to initialize DB session: {e}")
self._db_initialized = True
self._current_run_id = None # Track current run
def _start_run(self, input_content: str):
"""Start a new run (turn) for persistence tracking."""
if self._db is None:
return
import uuid
self._current_run_id = f"run-{uuid.uuid4().hex[:12]}"
try:
if hasattr(self._db, 'on_run_start'):
self._db.on_run_start(
session_id=self._session_id,
run_id=self._current_run_id,
input_content=input_content,
metadata={"agent_name": self.name}
)
except Exception as e:
logging.warning(f"Failed to start run: {e}")
def _end_run(self, output_content: str, status: str = "completed", metrics: dict = None):
"""End the current run (turn)."""
if self._db is None or self._current_run_id is None:
return
try:
if hasattr(self._db, 'on_run_end'):
self._db.on_run_end(
session_id=self._session_id,
run_id=self._current_run_id,
output_content=output_content,
status=status,
metrics=metrics or {},
metadata={"agent_name": self.name}
)
except Exception as e:
logging.warning(f"Failed to end run: {e}")
self._current_run_id = None
def _persist_message(self, role: str, content: str):
"""Persist a message to the DB if adapter is provided."""
if self._db is None:
return
try:
if role == "user":
self._db.on_user_message(self._session_id, content)
elif role == "assistant":
self._db.on_agent_message(self._session_id, content)
except Exception as e:
logging.warning(f"Failed to persist message: {e}")
@property
def session_id(self) -> Optional[str]:
"""Get the current session ID."""
return self._session_id
def chat(self, prompt, temperature=1.0, tools=None, output_json=None, output_pydantic=None, reasoning_steps=False, stream=None, task_name=None, task_description=None, task_id=None, config=None):
# Apply rate limiter if configured (before any LLM call)
if self._rate_limiter is not None:
self._rate_limiter.acquire()
# Initialize DB session on first chat (lazy)
self._init_db_session()
# Start a new run for this chat turn
prompt_str = prompt if isinstance(prompt, str) else str(prompt)
self._start_run(prompt_str)
# Reset the final display flag for each new conversation
self._final_display_shown = False
@@ -2054,6 +2204,8 @@ Your Goal: {self.goal}"""
self.chat_history[-1].get("content") == normalized_content):
# Add user message to chat history BEFORE LLM call so handoffs can access it
self.chat_history.append({"role": "user", "content": normalized_content})
# Persist user message to DB
self._persist_message("user", normalized_content)
try:
# Pass everything to LLM class
@@ -2083,6 +2235,8 @@ Your Goal: {self.goal}"""
)
self.chat_history.append({"role": "assistant", "content": response_text})
# Persist assistant message to DB
self._persist_message("assistant", response_text)
# Log completion time if in debug mode
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
@@ -2127,6 +2281,8 @@ Your Goal: {self.goal}"""
self.chat_history[-1].get("content") == normalized_content):
# Add user message to chat history BEFORE LLM call so handoffs can access it
self.chat_history.append({"role": "user", "content": normalized_content})
# Persist user message to DB (OpenAI path)
self._persist_message("user", normalized_content)
reflection_count = 0
start_time = time.time()
@@ -2166,6 +2322,8 @@ Your Goal: {self.goal}"""
# Add to chat history and return raw response
# User message already added before LLM call via _build_messages
self.chat_history.append({"role": "assistant", "content": response_text})
# Persist assistant message to DB
self._persist_message("assistant", response_text)
# Apply guardrail validation even for JSON output
try:
validated_response = self._apply_guardrail_with_retry(response_text, original_prompt, temperature, tools, task_name, task_description, task_id)
@@ -2181,6 +2339,8 @@ Your Goal: {self.goal}"""
if not self.self_reflect:
# User message already added before LLM call via _build_messages
self.chat_history.append({"role": "assistant", "content": response_text})
# Persist assistant message to DB (non-reflect path)
self._persist_message("assistant", response_text)
if self.verbose:
logging.debug(f"Agent {self.name} final response: {response_text}")
# Return only reasoning content if reasoning_steps is True
@@ -2269,11 +2429,13 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
validated_response = self._apply_guardrail_with_retry(response_text, original_prompt, temperature, tools, task_name, task_description, task_id)
# Execute callback after validation
self._execute_callback_and_display(original_prompt, validated_response, time.time() - start_time, task_name, task_description, task_id)
self._end_run(validated_response, "completed", {"duration_ms": (time.time() - start_time) * 1000})
return validated_response
except Exception as e:
logging.error(f"Agent {self.name}: Guardrail validation failed after reflection: {e}")
# Rollback chat history on guardrail failure
self.chat_history = self.chat_history[:chat_history_length]
self._end_run(None, "error", {"error": str(e)})
return None
# Check if we've hit max reflections
@@ -65,6 +65,23 @@ __all__ = [
"AfterAgentInput",
"SessionStartInput",
"SessionEndInput",
# Middleware types
"InvocationContext",
"ModelRequest",
"ModelResponse",
"ToolRequest",
"ToolResponse",
# Middleware decorators
"before_model",
"after_model",
"wrap_model_call",
"before_tool",
"after_tool",
"wrap_tool_call",
# Middleware utilities
"MiddlewareChain",
"AsyncMiddlewareChain",
"MiddlewareManager",
]
@@ -94,4 +111,27 @@ def __getattr__(name: str):
)
return locals()[name]
# Middleware types
if name in ("InvocationContext", "ModelRequest", "ModelResponse",
"ToolRequest", "ToolResponse"):
from .middleware import (
InvocationContext, ModelRequest, ModelResponse,
ToolRequest, ToolResponse
)
return locals()[name]
# Middleware decorators
if name in ("before_model", "after_model", "wrap_model_call",
"before_tool", "after_tool", "wrap_tool_call"):
from .middleware import (
before_model, after_model, wrap_model_call,
before_tool, after_tool, wrap_tool_call
)
return locals()[name]
# Middleware utilities
if name in ("MiddlewareChain", "AsyncMiddlewareChain", "MiddlewareManager"):
from .middleware import MiddlewareChain, AsyncMiddlewareChain, MiddlewareManager
return locals()[name]
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -0,0 +1,439 @@
"""
Middleware System for PraisonAI Agents.
Provides wrap_model_call and wrap_tool_call patterns for intercepting
and modifying agent behavior at model/tool call boundaries.
Zero Performance Impact:
- All middleware is optional
- Empty middleware chain is O(1) fast path
- No overhead when middleware not registered
Usage:
from praisonaiagents.hooks import before_model, wrap_model_call, wrap_tool_call
@before_model
def add_context(request):
request.messages.append({"role": "system", "content": "Extra"})
return request
@wrap_model_call
def retry_on_error(request, call_next):
for _ in range(3):
try:
return call_next(request)
except Exception:
pass
raise RuntimeError("All retries failed")
agent = Agent(name="Test", hooks=[add_context, retry_on_error])
"""
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, List, Optional, TypeVar, Generic
from functools import wraps
@dataclass
class InvocationContext:
"""Context passed through middleware chain.
Contains identifiers for multi-agent safety and optional metadata.
"""
agent_id: str
run_id: str
session_id: str
tool_name: Optional[str] = None
model_name: Optional[str] = None
metadata: Dict[str, Any] = field(default_factory=dict)
@dataclass
class ModelRequest:
"""Request data for model calls."""
messages: List[Dict[str, Any]]
model: str
temperature: float = 1.0
context: Optional[InvocationContext] = None
tools: Optional[List[Any]] = None
extra: Dict[str, Any] = field(default_factory=dict)
@dataclass
class ModelResponse:
"""Response data from model calls."""
content: str
model: str
usage: Optional[Dict[str, int]] = None
context: Optional[InvocationContext] = None
tool_calls: Optional[List[Any]] = None
extra: Dict[str, Any] = field(default_factory=dict)
@dataclass
class ToolRequest:
"""Request data for tool calls."""
tool_name: str
arguments: Dict[str, Any]
context: Optional[InvocationContext] = None
extra: Dict[str, Any] = field(default_factory=dict)
@dataclass
class ToolResponse:
"""Response data from tool calls."""
tool_name: str
result: Any
error: Optional[str] = None
context: Optional[InvocationContext] = None
extra: Dict[str, Any] = field(default_factory=dict)
# Type aliases for middleware functions
T = TypeVar('T')
BeforeModelFn = Callable[[ModelRequest], ModelRequest]
AfterModelFn = Callable[[ModelResponse], ModelResponse]
WrapModelCallFn = Callable[[ModelRequest, Callable[[ModelRequest], ModelResponse]], ModelResponse]
BeforeToolFn = Callable[[ToolRequest], ToolRequest]
AfterToolFn = Callable[[ToolResponse], ToolResponse]
WrapToolCallFn = Callable[[ToolRequest, Callable[[ToolRequest], ToolResponse]], ToolResponse]
class MiddlewareChain:
"""Synchronous middleware chain executor.
Composes middleware functions in registration order.
First registered middleware wraps all subsequent ones.
"""
def __init__(self, middlewares: Optional[List[Callable]] = None):
self._middlewares = middlewares or []
def execute(self, request: Any, final_handler: Callable[[Any], Any]) -> Any:
"""Execute the middleware chain with a final handler.
Args:
request: The request object to pass through
final_handler: The actual operation to perform
Returns:
The result from the chain
"""
if not self._middlewares:
return final_handler(request)
# Build the chain from inside out
# Last middleware wraps final_handler
# First middleware is outermost
def build_chain(index: int) -> Callable[[Any], Any]:
if index >= len(self._middlewares):
return final_handler
middleware = self._middlewares[index]
next_handler = build_chain(index + 1)
def handler(req: Any) -> Any:
return middleware(req, next_handler)
return handler
chain = build_chain(0)
return chain(request)
def __len__(self) -> int:
return len(self._middlewares)
class AsyncMiddlewareChain:
"""Asynchronous middleware chain executor.
Same as MiddlewareChain but for async middleware and handlers.
"""
def __init__(self, middlewares: Optional[List[Callable]] = None):
self._middlewares = middlewares or []
async def execute(self, request: Any, final_handler: Callable) -> Any:
"""Execute the async middleware chain.
Args:
request: The request object to pass through
final_handler: The actual async operation to perform
Returns:
The result from the chain
"""
if not self._middlewares:
return await final_handler(request)
# Build the chain from inside out
def build_chain(index: int) -> Callable:
if index >= len(self._middlewares):
return final_handler
middleware = self._middlewares[index]
next_handler = build_chain(index + 1)
async def handler(req: Any) -> Any:
return await middleware(req, next_handler)
return handler
chain = build_chain(0)
return await chain(request)
def __len__(self) -> int:
return len(self._middlewares)
# Decorator functions for tagging middleware
def before_model(func: BeforeModelFn) -> BeforeModelFn:
"""Decorator to mark a function as a before_model hook.
The function receives a ModelRequest and should return a (possibly modified) ModelRequest.
Example:
@before_model
def add_context(request):
request.messages.append({"role": "system", "content": "Extra"})
return request
"""
func._hook_type = 'before_model'
return func
def after_model(func: AfterModelFn) -> AfterModelFn:
"""Decorator to mark a function as an after_model hook.
The function receives a ModelResponse and should return a (possibly modified) ModelResponse.
Example:
@after_model
def log_response(response):
print(f"Model returned: {response.content[:50]}")
return response
"""
func._hook_type = 'after_model'
return func
def wrap_model_call(func: WrapModelCallFn) -> WrapModelCallFn:
"""Decorator to mark a function as a wrap_model_call middleware.
The function receives (request, call_next) and should call call_next(request)
to continue the chain, or return early to short-circuit.
Example:
@wrap_model_call
def retry_on_error(request, call_next):
for _ in range(3):
try:
return call_next(request)
except Exception:
pass
raise RuntimeError("All retries failed")
"""
func._hook_type = 'wrap_model_call'
return func
def before_tool(func: BeforeToolFn) -> BeforeToolFn:
"""Decorator to mark a function as a before_tool hook.
The function receives a ToolRequest and should return a (possibly modified) ToolRequest.
Example:
@before_tool
def validate_args(request):
if 'dangerous' in request.arguments:
raise ValueError("Dangerous argument detected")
return request
"""
func._hook_type = 'before_tool'
return func
def after_tool(func: AfterToolFn) -> AfterToolFn:
"""Decorator to mark a function as an after_tool hook.
The function receives a ToolResponse and should return a (possibly modified) ToolResponse.
Example:
@after_tool
def log_result(response):
print(f"Tool {response.tool_name} returned: {response.result}")
return response
"""
func._hook_type = 'after_tool'
return func
def wrap_tool_call(func: WrapToolCallFn) -> WrapToolCallFn:
"""Decorator to mark a function as a wrap_tool_call middleware.
The function receives (request, call_next) and should call call_next(request)
to continue the chain, or return early to short-circuit.
Example:
@wrap_tool_call
def retry_flaky_tool(request, call_next):
last_error = None
for _ in range(3):
try:
return call_next(request)
except Exception as e:
last_error = e
raise last_error
"""
func._hook_type = 'wrap_tool_call'
return func
def get_hook_type(func: Callable) -> Optional[str]:
"""Get the hook type of a decorated function."""
return getattr(func, '_hook_type', None)
def is_middleware(func: Callable) -> bool:
"""Check if a function is a middleware (wrap_* type)."""
hook_type = get_hook_type(func)
return hook_type in ('wrap_model_call', 'wrap_tool_call')
def categorize_hooks(hooks: List[Callable]) -> Dict[str, List[Callable]]:
"""Categorize hooks by their type.
Returns:
Dict with keys: before_model, after_model, wrap_model_call,
before_tool, after_tool, wrap_tool_call
"""
categories = {
'before_model': [],
'after_model': [],
'wrap_model_call': [],
'before_tool': [],
'after_tool': [],
'wrap_tool_call': [],
}
for hook in hooks or []:
hook_type = get_hook_type(hook)
if hook_type and hook_type in categories:
categories[hook_type].append(hook)
return categories
class MiddlewareManager:
"""Manages middleware for an agent.
Provides methods to execute before/after hooks and middleware chains
for both model and tool calls.
"""
def __init__(self, hooks: Optional[List[Callable]] = None):
self._hooks = hooks or []
self._categorized = categorize_hooks(self._hooks)
# Pre-build chains for efficiency
self._model_chain = MiddlewareChain(self._categorized['wrap_model_call'])
self._tool_chain = MiddlewareChain(self._categorized['wrap_tool_call'])
@property
def has_model_hooks(self) -> bool:
"""Check if any model hooks are registered."""
return bool(
self._categorized['before_model'] or
self._categorized['after_model'] or
self._categorized['wrap_model_call']
)
@property
def has_tool_hooks(self) -> bool:
"""Check if any tool hooks are registered."""
return bool(
self._categorized['before_tool'] or
self._categorized['after_tool'] or
self._categorized['wrap_tool_call']
)
def run_before_model(self, request: ModelRequest) -> ModelRequest:
"""Run all before_model hooks."""
for hook in self._categorized['before_model']:
request = hook(request)
return request
def run_after_model(self, response: ModelResponse) -> ModelResponse:
"""Run all after_model hooks (in reverse order)."""
for hook in reversed(self._categorized['after_model']):
response = hook(response)
return response
def run_before_tool(self, request: ToolRequest) -> ToolRequest:
"""Run all before_tool hooks."""
for hook in self._categorized['before_tool']:
request = hook(request)
return request
def run_after_tool(self, response: ToolResponse) -> ToolResponse:
"""Run all after_tool hooks (in reverse order)."""
for hook in reversed(self._categorized['after_tool']):
response = hook(response)
return response
def execute_model_call(
self,
request: ModelRequest,
model_fn: Callable[[ModelRequest], ModelResponse]
) -> ModelResponse:
"""Execute a model call with all hooks and middleware.
Order:
1. before_model hooks (in order)
2. wrap_model_call middleware chain
3. actual model call
4. after_model hooks (in reverse order)
"""
# Fast path: no hooks
if not self.has_model_hooks:
return model_fn(request)
# Run before hooks
request = self.run_before_model(request)
# Build wrapped handler that includes after hooks
def wrapped_model_fn(req: ModelRequest) -> ModelResponse:
response = model_fn(req)
return self.run_after_model(response)
# Execute through middleware chain
return self._model_chain.execute(request, wrapped_model_fn)
def execute_tool_call(
self,
request: ToolRequest,
tool_fn: Callable[[ToolRequest], ToolResponse]
) -> ToolResponse:
"""Execute a tool call with all hooks and middleware.
Order:
1. before_tool hooks (in order)
2. wrap_tool_call middleware chain
3. actual tool call
4. after_tool hooks (in reverse order)
"""
# Fast path: no hooks
if not self.has_tool_hooks:
return tool_fn(request)
# Run before hooks
request = self.run_before_tool(request)
# Build wrapped handler that includes after hooks
def wrapped_tool_fn(req: ToolRequest) -> ToolResponse:
response = tool_fn(req)
return self.run_after_tool(response)
# Execute through middleware chain
return self._tool_chain.execute(request, wrapped_tool_fn)
@@ -27,6 +27,7 @@ from .model_router import (
TaskComplexity,
create_routing_agent
)
from .rate_limiter import RateLimiter
__all__ = [
"LLM",
@@ -46,5 +47,6 @@ __all__ = [
"ModelRouter",
"ModelProfile",
"TaskComplexity",
"create_routing_agent"
"create_routing_agent",
"RateLimiter"
]
@@ -0,0 +1,147 @@
"""
Rate Limiter for PraisonAI Agents.
Provides token bucket rate limiting for LLM API calls.
Zero Performance Impact:
- When rate_limiter is None, no overhead
- Uses monotonic clock for accuracy
- Injectable clock/sleep for deterministic testing
Usage:
from praisonaiagents.llm import RateLimiter
from praisonaiagents import Agent
limiter = RateLimiter(requests_per_minute=60)
agent = Agent(name="Test", rate_limiter=limiter)
"""
import time
import asyncio
from dataclasses import dataclass, field
from typing import Optional, Callable
@dataclass
class RateLimiter:
"""Token bucket rate limiter for API calls.
Uses the token bucket algorithm:
- Tokens are added at a fixed rate (requests_per_minute / 60 per second)
- Each request consumes one token
- If no tokens available, wait until one is available
- Burst allows multiple requests in quick succession
Args:
requests_per_minute: Maximum requests per minute
tokens_per_minute: Optional token-based rate limiting (for future use)
burst: Maximum burst size (default: 1)
Example:
limiter = RateLimiter(requests_per_minute=60) # 1 req/sec
limiter.acquire() # Blocks if rate exceeded
# Async usage
await limiter.acquire_async()
"""
requests_per_minute: int
tokens_per_minute: Optional[int] = None
burst: int = 1
# Internal state
_tokens: float = field(default=None, init=False, repr=False)
_last_update: float = field(default=None, init=False, repr=False)
_lock: asyncio.Lock = field(default=None, init=False, repr=False)
# Injectable functions for testing
_get_time: Callable[[], float] = field(default=None, init=False, repr=False)
_sleep: Callable[[float], None] = field(default=None, init=False, repr=False)
_async_sleep: Callable[[float], None] = field(default=None, init=False, repr=False)
def __post_init__(self):
"""Initialize internal state."""
self._tokens = float(self.burst)
self._last_update = time.monotonic()
self._lock = asyncio.Lock()
# Default implementations (can be overridden for testing)
if self._get_time is None:
self._get_time = time.monotonic
if self._sleep is None:
self._sleep = time.sleep
if self._async_sleep is None:
self._async_sleep = asyncio.sleep
@property
def _rate(self) -> float:
"""Tokens per second."""
return self.requests_per_minute / 60.0
def _refill(self) -> None:
"""Refill tokens based on elapsed time."""
now = self._get_time()
elapsed = now - self._last_update
self._tokens = min(self.burst, self._tokens + elapsed * self._rate)
self._last_update = now
def _wait_time(self) -> float:
"""Calculate time to wait for next token."""
if self._tokens >= 1.0:
return 0.0
return (1.0 - self._tokens) / self._rate
def acquire(self) -> None:
"""Acquire a token, blocking if necessary.
This is the synchronous version. Use acquire_async() for async code.
"""
self._refill()
wait = self._wait_time()
if wait > 0:
self._sleep(wait)
self._refill()
self._tokens -= 1.0
async def acquire_async(self) -> None:
"""Acquire a token asynchronously, waiting if necessary.
This is the async version. Use acquire() for sync code.
"""
async with self._lock:
self._refill()
wait = self._wait_time()
if wait > 0:
await self._async_sleep(wait)
self._refill()
self._tokens -= 1.0
def try_acquire(self) -> bool:
"""Try to acquire a token without blocking.
Returns:
True if token acquired, False if rate limit exceeded
"""
self._refill()
if self._tokens >= 1.0:
self._tokens -= 1.0
return True
return False
def reset(self) -> None:
"""Reset the rate limiter to initial state."""
self._tokens = float(self.burst)
self._last_update = self._get_time()
@property
def available_tokens(self) -> float:
"""Get current available tokens (for monitoring)."""
self._refill()
return self._tokens
def __repr__(self) -> str:
return f"RateLimiter(rpm={self.requests_per_minute}, burst={self.burst})"
@@ -0,0 +1,240 @@
"""
Observability interface for PraisonAI Agents.
This module provides the protocol and types for observability/tracing.
Implementations are provided by the wrapper layer (praisonai_tools.observability).
Usage (simplest - recommended):
from praisonaiagents import Agent, obs
agent = Agent(
name="Assistant",
observability=obs.auto(), # Auto-detect from env vars
)
agent.chat("Hello!") # Auto-traces to configured provider
Alternative (explicit provider):
observability=obs.langfuse() # Langfuse
observability=obs.langsmith() # LangSmith
observability=obs.agentops() # AgentOps
observability=obs.arize() # Arize Phoenix
observability=obs.datadog() # Datadog
"""
from typing import Any, Dict, List, Optional, Protocol, runtime_checkable
from dataclasses import dataclass, field
import time
@dataclass
class Span:
"""Represents a span in a trace."""
span_id: str
trace_id: str
name: str
parent_span_id: Optional[str] = None
start_time: float = field(default_factory=time.time)
end_time: Optional[float] = None
attributes: Dict[str, Any] = field(default_factory=dict)
status: str = "ok"
@dataclass
class Trace:
"""Represents a complete trace."""
trace_id: str
session_id: Optional[str] = None
agent_name: Optional[str] = None
user_id: Optional[str] = None
start_time: float = field(default_factory=time.time)
end_time: Optional[float] = None
spans: List[Span] = field(default_factory=list)
metadata: Dict[str, Any] = field(default_factory=dict)
status: str = "ok"
@runtime_checkable
class ObservabilityAdapter(Protocol):
"""Protocol for observability adapters."""
def on_trace_start(
self,
trace_id: str,
session_id: Optional[str] = None,
agent_name: Optional[str] = None,
user_id: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
) -> None:
"""Called when a trace starts."""
...
def on_trace_end(
self,
trace_id: str,
status: str = "ok",
metadata: Optional[Dict[str, Any]] = None,
) -> None:
"""Called when a trace ends."""
...
def on_span_start(
self,
span_id: str,
trace_id: str,
name: str,
parent_span_id: Optional[str] = None,
attributes: Optional[Dict[str, Any]] = None,
) -> None:
"""Called when a span starts."""
...
def on_span_end(
self,
span_id: str,
status: str = "ok",
attributes: Optional[Dict[str, Any]] = None,
) -> None:
"""Called when a span ends."""
...
def on_llm_call(
self,
span_id: str,
model: str,
messages: List[Dict[str, Any]],
response: Optional[str] = None,
tokens: Optional[Dict[str, int]] = None,
latency_ms: Optional[float] = None,
) -> None:
"""Called when an LLM call is made."""
...
def on_tool_call(
self,
span_id: str,
tool_name: str,
args: Dict[str, Any],
result: Any,
latency_ms: Optional[float] = None,
) -> None:
"""Called when a tool is invoked."""
...
def flush(self) -> None:
"""Flush any pending data."""
...
def close(self) -> None:
"""Close the adapter and release resources."""
...
class _LazyObsModule:
"""
Lazy proxy for observability backends.
Allows `from praisonaiagents import obs` without importing heavy deps.
Actual provider classes are loaded only when accessed.
"""
_PROVIDERS = {
"langfuse": "praisonai_tools.observability.providers.langfuse_provider",
"langsmith": "praisonai_tools.observability.providers.langsmith_provider",
"agentops": "praisonai_tools.observability.providers.agentops_provider",
"arize": "praisonai_tools.observability.providers.arize_phoenix_provider",
"phoenix": "praisonai_tools.observability.providers.arize_phoenix_provider",
"datadog": "praisonai_tools.observability.providers.datadog_provider",
"mlflow": "praisonai_tools.observability.providers.mlflow_provider",
"openlit": "praisonai_tools.observability.providers.openlit_provider",
"opik": "praisonai_tools.observability.providers.opik_provider",
"portkey": "praisonai_tools.observability.providers.portkey_provider",
"traceloop": "praisonai_tools.observability.providers.traceloop_provider",
"weave": "praisonai_tools.observability.providers.weave_provider",
"braintrust": "praisonai_tools.observability.providers.braintrust_provider",
"langtrace": "praisonai_tools.observability.providers.langtrace_provider",
"langwatch": "praisonai_tools.observability.providers.langwatch_provider",
"maxim": "praisonai_tools.observability.providers.maxim_provider",
}
def __getattr__(self, name: str):
if name in self._PROVIDERS:
return self._create_provider_factory(name)
raise AttributeError(f"module 'obs' has no attribute {name!r}")
def _create_provider_factory(self, provider_name: str):
"""Create a factory function for a provider."""
def factory(**kwargs):
try:
import importlib
module = importlib.import_module(self._PROVIDERS[provider_name])
# Get the provider class (naming convention: XxxProvider)
class_name = f"{provider_name.title().replace('_', '')}Provider"
provider_cls = getattr(module, class_name, None)
if provider_cls:
return provider_cls(**kwargs)
# Fallback: try to find any class ending with Provider
for attr_name in dir(module):
if attr_name.endswith("Provider"):
return getattr(module, attr_name)(**kwargs)
raise AttributeError(f"No provider class found in {module}")
except ImportError as e:
raise ImportError(
f"Observability provider '{provider_name}' requires praisonai-tools. "
f"Install with: pip install praisonai-tools\n"
f"Original error: {e}"
) from e
return factory
def auto(self, **kwargs):
"""
Auto-detect observability provider from environment variables.
Checks for common env vars like LANGFUSE_PUBLIC_KEY, LANGSMITH_API_KEY, etc.
Returns None if no provider is detected.
"""
import os
# Check for common provider env vars
provider_env_map = {
"LANGFUSE_PUBLIC_KEY": "langfuse",
"LANGSMITH_API_KEY": "langsmith",
"AGENTOPS_API_KEY": "agentops",
"ARIZE_API_KEY": "arize",
"DATADOG_API_KEY": "datadog",
"MLFLOW_TRACKING_URI": "mlflow",
"OPENLIT_API_KEY": "openlit",
"OPIK_API_KEY": "opik",
"PORTKEY_API_KEY": "portkey",
"TRACELOOP_API_KEY": "traceloop",
"WANDB_API_KEY": "weave",
"BRAINTRUST_API_KEY": "braintrust",
}
for env_var, provider in provider_env_map.items():
if os.getenv(env_var):
return self._create_provider_factory(provider)(**kwargs)
return None
def __repr__(self):
return "<module 'praisonaiagents.obs' (lazy observability backends)>"
def __call__(self, provider: str = "auto", **kwargs):
"""
Shortcut: obs("langfuse") or obs() for auto-detection.
"""
if provider == "auto":
return self.auto(**kwargs)
return self._create_provider_factory(provider)(**kwargs)
# Singleton lazy module instance
obs = _LazyObsModule()
__all__ = [
"ObservabilityAdapter",
"Span",
"Trace",
"obs",
]
@@ -2,6 +2,9 @@
from importlib import import_module
from typing import Any
# Export Injected type for tool state injection
from .injected import Injected
# Map of function names to their module and class (if any)
TOOL_MAPPINGS = {
# Direct functions
@@ -286,4 +289,4 @@ def __getattr__(name: str) -> Any:
method = getattr(_instances[class_name], name)
return method
__all__ = list(TOOL_MAPPINGS.keys())
__all__ = list(TOOL_MAPPINGS.keys()) + ['Injected']
@@ -14,6 +14,14 @@ Usage:
@tool(name="web_search", description="Search the internet")
def search(query: str, max_results: int = 5) -> list:
return [...]
# With injected state:
from praisonaiagents.tools import Injected
@tool
def my_tool(query: str, state: Injected[dict]) -> str:
'''Tool with injected state.'''
return f"session={state.get('session_id')}"
"""
import inspect
@@ -22,12 +30,19 @@ import logging
from typing import Any, Callable, Dict, Optional, Union, get_type_hints
from .base import BaseTool
from .injected import (
is_injected_type,
get_injected_params,
inject_state_into_kwargs,
filter_injected_from_schema
)
class FunctionTool(BaseTool):
"""A BaseTool wrapper for plain functions.
Created automatically by the @tool decorator.
Supports Injected[T] parameters for state injection.
"""
def __init__(
@@ -42,6 +57,9 @@ class FunctionTool(BaseTool):
self.description = description or func.__doc__ or f"Tool: {self.name}"
self.version = version
# Detect injected parameters
self._injected_params = get_injected_params(func)
# Generate schema from the original function, not run()
self.parameters = self._generate_schema_from_func(func)
@@ -55,8 +73,16 @@ class FunctionTool(BaseTool):
if not self.description:
self.description = self.__class__.__doc__ or f"Tool: {self.name}"
@property
def injected_params(self) -> Dict[str, Any]:
"""Get the injected parameters for this tool."""
return self._injected_params
def _generate_schema_from_func(self, func: Callable) -> Dict[str, Any]:
"""Generate JSON Schema from the wrapped function's signature."""
"""Generate JSON Schema from the wrapped function's signature.
Injected parameters are excluded from the schema.
"""
schema = {
"type": "object",
"properties": {},
@@ -71,8 +97,17 @@ class FunctionTool(BaseTool):
if param_name in ('self', 'cls'):
continue
# Skip injected parameters - they don't go in schema
if param_name in self._injected_params:
continue
# Get type hint
param_type = hints.get(param_name, Any)
# Double-check it's not an Injected type
if is_injected_type(param_type):
continue
json_type = BaseTool._python_type_to_json(param_type)
schema["properties"][param_name] = {"type": json_type}
@@ -86,11 +121,18 @@ class FunctionTool(BaseTool):
return schema
def run(self, **kwargs) -> Any:
"""Execute the wrapped function."""
"""Execute the wrapped function with injected state."""
# Inject state for any Injected parameters
kwargs = inject_state_into_kwargs(kwargs, self._injected_params)
return self._func(**kwargs)
def __call__(self, *args, **kwargs) -> Any:
"""Allow calling with positional args like the original function."""
"""Allow calling with positional args like the original function.
Injects state for Injected parameters.
"""
# Inject state for any Injected parameters
kwargs = inject_state_into_kwargs(kwargs, self._injected_params)
return self._func(*args, **kwargs)
@@ -0,0 +1,213 @@
"""
Injected State for PraisonAI Tools.
Provides a mechanism for tools to receive injected state (agent context)
without exposing it in the tool's public schema.
Zero Performance Impact:
- When no Injected params, no overhead
- Context is thread-local for safety
- Lazy evaluation of state
Usage:
from praisonaiagents import tool
from praisonaiagents.tools import Injected
@tool
def my_tool(query: str, state: Injected[dict]) -> str:
'''Tool with injected state.'''
session_id = state.get('session_id')
return f"Query: {query}, Session: {session_id}"
"""
from dataclasses import dataclass, field
from typing import Any, Dict, Generic, List, Optional, TypeVar, get_type_hints, get_origin, get_args
from contextvars import ContextVar
from contextlib import contextmanager
T = TypeVar('T')
class Injected(Generic[T]):
"""Type marker for injected parameters.
Parameters annotated with Injected[T] will:
- Not appear in the tool's public schema
- Be automatically injected at runtime with the current agent state
Supported types:
- Injected[dict] - Receives state as a dictionary
- Injected[AgentState] - Receives state as AgentState dataclass
Example:
@tool
def my_tool(query: str, state: Injected[dict]) -> str:
return f"session={state.get('session_id')}"
"""
pass
@dataclass
class AgentState:
"""State object injected into tools.
Contains context about the current agent execution.
"""
agent_id: str
run_id: str
session_id: str
last_user_message: Optional[str] = None
last_agent_message: Optional[str] = None
memory: Optional[Any] = None
previous_tool_results: List[Any] = field(default_factory=list)
metadata: Dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> Dict[str, Any]:
"""Convert to dictionary for Injected[dict] usage."""
return {
'agent_id': self.agent_id,
'run_id': self.run_id,
'session_id': self.session_id,
'last_user_message': self.last_user_message,
'last_agent_message': self.last_agent_message,
'previous_tool_results': self.previous_tool_results,
'metadata': self.metadata,
}
# Thread-local context for current state
_current_state: ContextVar[Optional[AgentState]] = ContextVar('current_state', default=None)
def get_current_state() -> Optional[AgentState]:
"""Get the current injection state."""
return _current_state.get()
def set_current_state(state: Optional[AgentState]) -> None:
"""Set the current injection state."""
_current_state.set(state)
@contextmanager
def with_injection_context(state: AgentState):
"""Context manager for setting injection state.
Usage:
with with_injection_context(state):
result = my_tool(query="test")
"""
token = _current_state.set(state)
try:
yield
finally:
_current_state.reset(token)
def is_injected_type(annotation: Any) -> bool:
"""Check if a type annotation is Injected[T]."""
origin = get_origin(annotation)
if origin is Injected:
return True
# Handle string annotations
if isinstance(annotation, str) and 'Injected' in annotation:
return True
return False
def get_injected_type(annotation: Any) -> Optional[type]:
"""Get the inner type T from Injected[T]."""
origin = get_origin(annotation)
if origin is Injected:
args = get_args(annotation)
if args:
return args[0]
return None
def resolve_injected_value(annotation: Any, state: Optional[AgentState]) -> Any:
"""Resolve the value to inject based on annotation type.
Args:
annotation: The Injected[T] annotation
state: The current agent state
Returns:
The resolved value (dict or AgentState)
"""
if state is None:
# Return empty dict if no state available
return {}
inner_type = get_injected_type(annotation)
if inner_type is dict or inner_type is Dict:
return state.to_dict()
elif inner_type is AgentState:
return state
else:
# Default to dict
return state.to_dict()
def get_injected_params(func) -> Dict[str, Any]:
"""Get all injected parameters from a function.
Returns:
Dict mapping param name to its Injected[T] annotation
"""
try:
hints = get_type_hints(func)
except Exception:
# If we can't get hints, return empty
return {}
injected = {}
for name, hint in hints.items():
if name == 'return':
continue
if is_injected_type(hint):
injected[name] = hint
return injected
def filter_injected_from_schema(properties: Dict[str, Any], required: List[str],
injected_params: Dict[str, Any]) -> tuple:
"""Remove injected parameters from schema.
Args:
properties: Schema properties dict
required: List of required param names
injected_params: Dict of injected param names
Returns:
Tuple of (filtered_properties, filtered_required)
"""
filtered_props = {k: v for k, v in properties.items() if k not in injected_params}
filtered_required = [r for r in required if r not in injected_params]
return filtered_props, filtered_required
def inject_state_into_kwargs(kwargs: Dict[str, Any], injected_params: Dict[str, Any]) -> Dict[str, Any]:
"""Inject state values into kwargs for injected parameters.
Args:
kwargs: The original kwargs
injected_params: Dict of injected param names to annotations
Returns:
Updated kwargs with injected values
"""
if not injected_params:
return kwargs
state = get_current_state()
result = kwargs.copy()
for param_name, annotation in injected_params.items():
if param_name not in result:
result[param_name] = resolve_injected_value(annotation, state)
return result
+1 -1
View File
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "praisonaiagents"
version = "0.2.0"
version = "0.3.0"
description = "Praison AI agents for completing complex tasks with Self Reflection Agents"
readme = "README.md"
requires-python = ">=3.10"
@@ -0,0 +1,234 @@
"""
TDD Tests for Configurable Model (runtime model switching).
These tests are written FIRST before implementation.
"""
import pytest
from unittest.mock import Mock, patch
class TestConfigurableModelBasic:
"""Test basic configurable model functionality."""
def test_agent_accepts_llm_config_configurable(self):
"""Agent should accept llm_config with configurable=True."""
from praisonaiagents import Agent
agent = Agent(
name="Test",
instructions="Test agent",
llm="gpt-4o",
llm_config={"configurable": True}
)
assert agent is not None
assert getattr(agent, '_llm_configurable', False) is True
def test_agent_chat_accepts_config_parameter(self):
"""Agent.chat() should accept config parameter for model override."""
from praisonaiagents import Agent
agent = Agent(
name="Test",
instructions="Test agent",
llm="gpt-4o",
llm_config={"configurable": True}
)
# Should not raise - config parameter should be accepted
# We'll mock the actual LLM call
with patch.object(agent, '_chat_completion', return_value="mocked"):
result = agent.chat(
"Hello",
config={"model": "claude-3-5-sonnet"}
)
def test_config_model_override(self):
"""config.model should override default model for that call."""
from praisonaiagents import Agent
agent = Agent(
name="Test",
instructions="Test agent",
llm="gpt-4o",
llm_config={"configurable": True}
)
# Track which model was used
used_model = [None]
original_chat = agent._chat_completion
def mock_chat(*args, **kwargs):
# The model should be accessible somehow
used_model[0] = getattr(agent, '_current_model', agent.llm)
return "response"
with patch.object(agent, '_chat_completion', side_effect=mock_chat):
agent.chat("Hello", config={"model": "claude-3-5-sonnet"})
# Model should have been switched for this call
# Implementation will set _current_model or similar
def test_config_does_not_mutate_default(self):
"""Per-call config should not change agent's default model."""
from praisonaiagents import Agent
agent = Agent(
name="Test",
instructions="Test agent",
llm="gpt-4o",
llm_config={"configurable": True}
)
original_llm = agent.llm
with patch.object(agent, '_chat_completion', return_value="mocked"):
agent.chat("Hello", config={"model": "claude-3-5-sonnet"})
# Default should be unchanged
assert agent.llm == original_llm
def test_config_temperature_override(self):
"""config.temperature should override for that call."""
from praisonaiagents import Agent
agent = Agent(
name="Test",
instructions="Test agent",
llm="gpt-4o",
llm_config={"configurable": True}
)
# Should accept temperature in config
with patch.object(agent, '_chat_completion', return_value="mocked") as mock:
agent.chat("Hello", config={"temperature": 0.2})
# Temperature should be passed through
class TestConfigurableModelCaching:
"""Test model client caching for configurable models."""
def test_model_client_cached(self):
"""Model clients should be cached to avoid recreation."""
from praisonaiagents.llm import LLM
llm = LLM(model="gpt-4o", configurable=True)
# First call with model A
# Second call with model A should reuse client
# This is internal behavior - we test via call count or similar
def test_cache_size_limited(self):
"""Model client cache should have a size limit."""
from praisonaiagents.llm import LLM
llm = LLM(model="gpt-4o", configurable=True, cache_size=3)
# Should not grow unbounded
class TestConfigurableModelThreadSafety:
"""Test thread safety of configurable models."""
def test_concurrent_calls_different_models(self):
"""Concurrent calls with different models should not interfere."""
import threading
from praisonaiagents import Agent
agent = Agent(
name="Test",
instructions="Test agent",
llm="gpt-4o",
llm_config={"configurable": True}
)
results = {}
errors = []
def call_with_model(model_name, thread_id):
try:
with patch.object(agent, '_chat_completion', return_value=f"response_{model_name}"):
result = agent.chat("Hello", config={"model": model_name})
results[thread_id] = result
except Exception as e:
errors.append(e)
threads = [
threading.Thread(target=call_with_model, args=("model_a", 1)),
threading.Thread(target=call_with_model, args=("model_b", 2)),
]
for t in threads:
t.start()
for t in threads:
t.join()
assert len(errors) == 0
class TestConfigurableModelValidation:
"""Test validation of configurable model settings."""
def test_config_without_configurable_flag_ignored(self):
"""If llm_config.configurable is False, config param should be ignored."""
from praisonaiagents import Agent
agent = Agent(
name="Test",
instructions="Test agent",
llm="gpt-4o"
# No llm_config or configurable=False
)
# Should still work, just ignore the config
with patch.object(agent, '_chat_completion', return_value="mocked"):
result = agent.chat("Hello", config={"model": "other"})
# Should use default model, not "other"
def test_invalid_config_keys_ignored(self):
"""Unknown keys in config should be ignored, not error."""
from praisonaiagents import Agent
agent = Agent(
name="Test",
instructions="Test agent",
llm="gpt-4o",
llm_config={"configurable": True}
)
# Should not raise for unknown keys
with patch.object(agent, '_chat_completion', return_value="mocked"):
result = agent.chat("Hello", config={"unknown_key": "value"})
class TestConfigurableModelProvider:
"""Test provider switching in configurable models."""
def test_config_provider_override(self):
"""config.provider should allow switching providers."""
from praisonaiagents import Agent
agent = Agent(
name="Test",
instructions="Test agent",
llm="gpt-4o",
llm_config={"configurable": True}
)
# Should accept provider in config
with patch.object(agent, '_chat_completion', return_value="mocked"):
agent.chat("Hello", config={"provider": "anthropic", "model": "claude-3-5-sonnet"})
def test_model_string_with_provider_prefix(self):
"""Model string like 'anthropic/claude-3' should work."""
from praisonaiagents import Agent
agent = Agent(
name="Test",
instructions="Test agent",
llm="gpt-4o",
llm_config={"configurable": True}
)
with patch.object(agent, '_chat_completion', return_value="mocked"):
agent.chat("Hello", config={"model": "anthropic/claude-3-5-sonnet"})
@@ -0,0 +1,367 @@
"""
TDD Tests for InjectedState for Tools.
These tests are written FIRST before implementation.
"""
import pytest
from unittest.mock import Mock, patch
class TestInjectedTypeExists:
"""Test that Injected type marker exists."""
def test_injected_type_exists(self):
"""Injected type should exist in praisonaiagents.tools."""
from praisonaiagents.tools import Injected
# Should be a generic type marker
assert Injected is not None
def test_injected_generic_usage(self):
"""Injected should work as a generic type annotation."""
from praisonaiagents.tools import Injected
# Should be usable as Injected[dict], Injected[SomeType], etc.
def my_tool(query: str, state: Injected[dict]) -> str:
return query
# Should have annotations
annotations = my_tool.__annotations__
assert 'state' in annotations
def test_injected_with_agent_state(self):
"""Injected[AgentState] should work."""
from praisonaiagents.tools import Injected
from praisonaiagents.tools.injected import AgentState
def my_tool(query: str, state: Injected[AgentState]) -> str:
return query
annotations = my_tool.__annotations__
assert 'state' in annotations
class TestAgentStateType:
"""Test AgentState type for injection."""
def test_agent_state_exists(self):
"""AgentState type should exist."""
from praisonaiagents.tools.injected import AgentState
assert AgentState is not None
def test_agent_state_has_required_fields(self):
"""AgentState should have agent_id, run_id, session_id."""
from praisonaiagents.tools.injected import AgentState
state = AgentState(
agent_id="agent-123",
run_id="run-456",
session_id="session-789"
)
assert state.agent_id == "agent-123"
assert state.run_id == "run-456"
assert state.session_id == "session-789"
def test_agent_state_optional_fields(self):
"""AgentState should have optional fields."""
from praisonaiagents.tools.injected import AgentState
state = AgentState(
agent_id="a",
run_id="r",
session_id="s",
last_user_message="Hello",
last_agent_message="Hi there",
memory=Mock(),
previous_tool_results=["result1", "result2"]
)
assert state.last_user_message == "Hello"
assert state.last_agent_message == "Hi there"
assert state.memory is not None
assert len(state.previous_tool_results) == 2
class TestToolDecoratorInjection:
"""Test @tool decorator handles Injected parameters."""
def test_tool_decorator_detects_injected(self):
"""@tool should detect Injected parameters."""
from praisonaiagents import tool
from praisonaiagents.tools import Injected
@tool
def my_tool(query: str, state: Injected[dict]) -> str:
"""A tool with injected state."""
return f"query={query}, session={state.get('session_id')}"
# Should have metadata about injected params
assert hasattr(my_tool, '_injected_params') or hasattr(my_tool, 'injected_params')
def test_injected_params_not_in_schema(self):
"""Injected parameters should not appear in tool schema."""
from praisonaiagents import tool
from praisonaiagents.tools import Injected
@tool
def my_tool(query: str, state: Injected[dict]) -> str:
"""A tool with injected state."""
return query
schema = my_tool.get_schema()
params = schema['function']['parameters']['properties']
# 'query' should be in schema
assert 'query' in params
# 'state' should NOT be in schema (it's injected)
assert 'state' not in params
def test_injected_params_not_required(self):
"""Injected parameters should not be in required list."""
from praisonaiagents import tool
from praisonaiagents.tools import Injected
@tool
def my_tool(query: str, state: Injected[dict]) -> str:
"""A tool with injected state."""
return query
schema = my_tool.get_schema()
required = schema['function']['parameters'].get('required', [])
assert 'state' not in required
assert 'query' in required
class TestToolExecutionInjection:
"""Test that injected state is provided at runtime."""
def test_injected_state_provided_at_runtime(self):
"""When tool is called, injected state should be provided."""
from praisonaiagents import tool
from praisonaiagents.tools import Injected
from praisonaiagents.tools.injected import AgentState
received_state = [None]
@tool
def my_tool(query: str, state: Injected[AgentState]) -> str:
"""A tool with injected state."""
received_state[0] = state
return f"session={state.session_id}"
# Simulate execution with injection context
from praisonaiagents.tools.injected import with_injection_context
mock_state = AgentState(
agent_id="a",
run_id="r",
session_id="test-session"
)
with with_injection_context(mock_state):
result = my_tool(query="hello")
assert received_state[0] is not None
assert received_state[0].session_id == "test-session"
def test_injected_dict_state(self):
"""Injected[dict] should provide dict with state info."""
from praisonaiagents import tool
from praisonaiagents.tools import Injected
from praisonaiagents.tools.injected import with_injection_context, AgentState
received_state = [None]
@tool
def my_tool(query: str, state: Injected[dict]) -> str:
"""A tool with injected dict state."""
received_state[0] = state
return "ok"
mock_state = AgentState(
agent_id="a",
run_id="r",
session_id="s"
)
with with_injection_context(mock_state):
my_tool(query="test")
assert isinstance(received_state[0], dict)
assert 'session_id' in received_state[0]
class TestAgentToolExecution:
"""Test Agent executes tools with injection."""
def test_agent_injects_state_to_tools(self):
"""Agent should inject state when executing tools."""
from praisonaiagents import Agent, tool
from praisonaiagents.tools import Injected
received_session = [None]
@tool
def show_state(query: str, state: Injected[dict]) -> str:
"""Show the injected state."""
received_session[0] = state.get('session_id')
return f"session={state.get('session_id')}"
agent = Agent(
name="Test",
instructions="Test agent",
tools=[show_state],
session_id="my-session-123"
)
# Execute the tool directly through agent
result = agent.execute_tool("show_state", {"query": "test"})
# State should have been injected
assert received_session[0] == "my-session-123"
class TestInjectionContextManager:
"""Test injection context manager."""
def test_with_injection_context_exists(self):
"""with_injection_context should exist."""
from praisonaiagents.tools.injected import with_injection_context
assert with_injection_context is not None
def test_context_manager_sets_state(self):
"""Context manager should set current state."""
from praisonaiagents.tools.injected import (
with_injection_context,
get_current_state,
AgentState
)
state = AgentState(agent_id="a", run_id="r", session_id="s")
with with_injection_context(state):
current = get_current_state()
assert current is state
# After context, should be None
assert get_current_state() is None
def test_nested_contexts(self):
"""Nested contexts should work correctly."""
from praisonaiagents.tools.injected import (
with_injection_context,
get_current_state,
AgentState
)
state1 = AgentState(agent_id="a1", run_id="r1", session_id="s1")
state2 = AgentState(agent_id="a2", run_id="r2", session_id="s2")
with with_injection_context(state1):
assert get_current_state().agent_id == "a1"
with with_injection_context(state2):
assert get_current_state().agent_id == "a2"
# Back to state1
assert get_current_state().agent_id == "a1"
class TestAsyncInjection:
"""Test async tool injection."""
@pytest.mark.asyncio
async def test_async_tool_injection(self):
"""Async tools should also receive injected state."""
from praisonaiagents import tool
from praisonaiagents.tools import Injected
from praisonaiagents.tools.injected import with_injection_context, AgentState
received_state = [None]
@tool
async def async_tool(query: str, state: Injected[dict]) -> str:
"""Async tool with injected state."""
received_state[0] = state
return "ok"
state = AgentState(agent_id="a", run_id="r", session_id="async-session")
with with_injection_context(state):
await async_tool(query="test")
assert received_state[0] is not None
assert received_state[0].get('session_id') == "async-session"
class TestInjectionZeroOverhead:
"""Test that injection has zero overhead when not used."""
def test_no_injected_params_no_overhead(self):
"""Tools without Injected params should have no overhead."""
from praisonaiagents import tool
@tool
def simple_tool(query: str) -> str:
"""Simple tool without injection."""
return query
# Should work normally without any injection context
result = simple_tool(query="hello")
assert result == "hello"
def test_schema_generation_fast(self):
"""Schema generation should be fast even with Injected params."""
from praisonaiagents import tool
from praisonaiagents.tools import Injected
import time
@tool
def my_tool(a: str, b: int, c: Injected[dict], d: float) -> str:
"""Tool with mixed params."""
return a
start = time.monotonic()
for _ in range(100):
my_tool.get_schema()
elapsed = time.monotonic() - start
# Should be very fast
assert elapsed < 0.1 # 100 calls in < 100ms
class TestBackwardCompatibility:
"""Test backward compatibility with existing tools."""
def test_existing_tools_still_work(self):
"""Existing tools without Injected should continue to work."""
from praisonaiagents import tool
@tool
def legacy_tool(query: str, count: int = 5) -> str:
"""Legacy tool."""
return f"{query} x {count}"
result = legacy_tool(query="test", count=3)
assert result == "test x 3"
def test_existing_tool_schema_unchanged(self):
"""Existing tool schemas should be unchanged."""
from praisonaiagents import tool
@tool
def legacy_tool(query: str, count: int = 5) -> str:
"""Legacy tool."""
return f"{query} x {count}"
schema = legacy_tool.get_schema()
params = schema['function']['parameters']['properties']
assert 'query' in params
assert 'count' in params
assert len(params) == 2
@@ -0,0 +1,403 @@
"""
TDD Tests for Middleware System (wrap_model_call, wrap_tool_call).
These tests are written FIRST before implementation.
"""
import pytest
from unittest.mock import Mock, MagicMock, patch, AsyncMock
from dataclasses import dataclass
from typing import Any, Dict, Optional
class TestMiddlewareTypes:
"""Test middleware type definitions."""
def test_invocation_context_exists(self):
"""InvocationContext dataclass should exist with required fields."""
from praisonaiagents.hooks.middleware import InvocationContext
ctx = InvocationContext(
agent_id="agent-123",
run_id="run-456",
session_id="session-789"
)
assert ctx.agent_id == "agent-123"
assert ctx.run_id == "run-456"
assert ctx.session_id == "session-789"
def test_invocation_context_optional_fields(self):
"""InvocationContext should have optional tool_name, model_name, metadata."""
from praisonaiagents.hooks.middleware import InvocationContext
ctx = InvocationContext(
agent_id="a",
run_id="r",
session_id="s",
tool_name="my_tool",
model_name="gpt-4o",
metadata={"key": "value"}
)
assert ctx.tool_name == "my_tool"
assert ctx.model_name == "gpt-4o"
assert ctx.metadata == {"key": "value"}
def test_model_request_exists(self):
"""ModelRequest dataclass for before_model hooks."""
from praisonaiagents.hooks.middleware import ModelRequest
req = ModelRequest(
messages=[{"role": "user", "content": "Hello"}],
model="gpt-4o",
temperature=0.7,
context=Mock()
)
assert req.messages == [{"role": "user", "content": "Hello"}]
assert req.model == "gpt-4o"
assert req.temperature == 0.7
def test_model_response_exists(self):
"""ModelResponse dataclass for after_model hooks."""
from praisonaiagents.hooks.middleware import ModelResponse
resp = ModelResponse(
content="Hello back!",
model="gpt-4o",
usage={"prompt_tokens": 10, "completion_tokens": 5},
context=Mock()
)
assert resp.content == "Hello back!"
assert resp.model == "gpt-4o"
def test_tool_request_exists(self):
"""ToolRequest dataclass for before_tool hooks."""
from praisonaiagents.hooks.middleware import ToolRequest
req = ToolRequest(
tool_name="get_weather",
arguments={"city": "Paris"},
context=Mock()
)
assert req.tool_name == "get_weather"
assert req.arguments == {"city": "Paris"}
def test_tool_response_exists(self):
"""ToolResponse dataclass for after_tool hooks."""
from praisonaiagents.hooks.middleware import ToolResponse
resp = ToolResponse(
tool_name="get_weather",
result="Sunny, 22°C",
error=None,
context=Mock()
)
assert resp.result == "Sunny, 22°C"
assert resp.error is None
class TestMiddlewareDecorators:
"""Test middleware decorator functions."""
def test_before_model_decorator_exists(self):
"""before_model decorator should exist and tag functions."""
from praisonaiagents.hooks import before_model
@before_model
def my_hook(request):
return request
assert hasattr(my_hook, '_hook_type')
assert my_hook._hook_type == 'before_model'
def test_after_model_decorator_exists(self):
"""after_model decorator should exist and tag functions."""
from praisonaiagents.hooks import after_model
@after_model
def my_hook(response):
return response
assert hasattr(my_hook, '_hook_type')
assert my_hook._hook_type == 'after_model'
def test_wrap_model_call_decorator_exists(self):
"""wrap_model_call decorator should exist and tag functions."""
from praisonaiagents.hooks import wrap_model_call
@wrap_model_call
def my_middleware(request, call_next):
return call_next(request)
assert hasattr(my_middleware, '_hook_type')
assert my_middleware._hook_type == 'wrap_model_call'
def test_before_tool_decorator_exists(self):
"""before_tool decorator should exist and tag functions."""
from praisonaiagents.hooks import before_tool
@before_tool
def my_hook(request):
return request
assert hasattr(my_hook, '_hook_type')
assert my_hook._hook_type == 'before_tool'
def test_after_tool_decorator_exists(self):
"""after_tool decorator should exist and tag functions."""
from praisonaiagents.hooks import after_tool
@after_tool
def my_hook(response):
return response
assert hasattr(my_hook, '_hook_type')
assert my_hook._hook_type == 'after_tool'
def test_wrap_tool_call_decorator_exists(self):
"""wrap_tool_call decorator should exist and tag functions."""
from praisonaiagents.hooks import wrap_tool_call
@wrap_tool_call
def my_middleware(request, call_next):
return call_next(request)
assert hasattr(my_middleware, '_hook_type')
assert my_middleware._hook_type == 'wrap_tool_call'
class TestMiddlewareChaining:
"""Test middleware chain composition."""
def test_wrap_model_call_chain_order(self):
"""wrap_model_call hooks should compose in registration order."""
from praisonaiagents.hooks.middleware import MiddlewareChain
calls = []
def mw1(req, call_next):
calls.append("mw1_before")
result = call_next(req)
calls.append("mw1_after")
return result
def mw2(req, call_next):
calls.append("mw2_before")
result = call_next(req)
calls.append("mw2_after")
return result
def final_handler(req):
calls.append("final")
return "response"
chain = MiddlewareChain([mw1, mw2])
result = chain.execute({"test": True}, final_handler)
assert calls == ["mw1_before", "mw2_before", "final", "mw2_after", "mw1_after"]
assert result == "response"
def test_empty_middleware_chain(self):
"""Empty middleware chain should just call final handler."""
from praisonaiagents.hooks.middleware import MiddlewareChain
def final_handler(req):
return "direct"
chain = MiddlewareChain([])
result = chain.execute({"test": True}, final_handler)
assert result == "direct"
def test_middleware_can_modify_request(self):
"""Middleware should be able to modify request before passing on."""
from praisonaiagents.hooks.middleware import MiddlewareChain
def add_context(req, call_next):
req["extra"] = "added"
return call_next(req)
def final_handler(req):
return req.get("extra")
chain = MiddlewareChain([add_context])
result = chain.execute({"original": True}, final_handler)
assert result == "added"
def test_middleware_can_modify_response(self):
"""Middleware should be able to modify response after call_next."""
from praisonaiagents.hooks.middleware import MiddlewareChain
def uppercase_response(req, call_next):
result = call_next(req)
return result.upper()
def final_handler(req):
return "hello"
chain = MiddlewareChain([uppercase_response])
result = chain.execute({}, final_handler)
assert result == "HELLO"
def test_middleware_can_short_circuit(self):
"""Middleware can return early without calling call_next."""
from praisonaiagents.hooks.middleware import MiddlewareChain
def block_middleware(req, call_next):
if req.get("block"):
return "blocked"
return call_next(req)
def final_handler(req):
return "reached"
chain = MiddlewareChain([block_middleware])
# Should be blocked
result = chain.execute({"block": True}, final_handler)
assert result == "blocked"
# Should pass through
result = chain.execute({"block": False}, final_handler)
assert result == "reached"
class TestMiddlewareRetry:
"""Test retry logic in middleware."""
def test_wrap_tool_call_retry_on_error(self):
"""wrap_tool_call should support retry patterns."""
from praisonaiagents.hooks.middleware import MiddlewareChain
attempt_count = [0]
def retry_middleware(req, call_next):
last_error = None
for i in range(3):
try:
return call_next(req)
except Exception as e:
last_error = e
raise last_error
def flaky_handler(req):
attempt_count[0] += 1
if attempt_count[0] < 3:
raise RuntimeError("flaky")
return "success"
chain = MiddlewareChain([retry_middleware])
result = chain.execute({}, flaky_handler)
assert result == "success"
assert attempt_count[0] == 3
class TestAsyncMiddleware:
"""Test async middleware support."""
@pytest.mark.asyncio
async def test_async_wrap_model_call(self):
"""Async middleware should work with async handlers."""
from praisonaiagents.hooks.middleware import AsyncMiddlewareChain
calls = []
async def async_mw(req, call_next):
calls.append("before")
result = await call_next(req)
calls.append("after")
return result
async def async_handler(req):
calls.append("handler")
return "async_result"
chain = AsyncMiddlewareChain([async_mw])
result = await chain.execute({}, async_handler)
assert calls == ["before", "handler", "after"]
assert result == "async_result"
@pytest.mark.asyncio
async def test_async_middleware_chain_order(self):
"""Async middleware should maintain order."""
from praisonaiagents.hooks.middleware import AsyncMiddlewareChain
order = []
async def mw1(req, call_next):
order.append(1)
result = await call_next(req)
order.append(-1)
return result
async def mw2(req, call_next):
order.append(2)
result = await call_next(req)
order.append(-2)
return result
async def handler(req):
order.append(0)
return "done"
chain = AsyncMiddlewareChain([mw1, mw2])
await chain.execute({}, handler)
assert order == [1, 2, 0, -2, -1]
class TestAgentMiddlewareIntegration:
"""Test middleware integration with Agent class."""
def test_agent_accepts_hooks_parameter(self):
"""Agent should accept hooks parameter."""
from praisonaiagents import Agent
from praisonaiagents.hooks import before_model
@before_model
def my_hook(req):
return req
# Should not raise
agent = Agent(
name="Test",
instructions="Test agent",
hooks=[my_hook]
)
assert agent is not None
def test_agent_hooks_empty_by_default(self):
"""Agent should have empty hooks by default."""
from praisonaiagents import Agent
agent = Agent(name="Test", instructions="Test")
# Should have no hooks or empty hooks list
hooks = getattr(agent, '_hooks', None) or getattr(agent, 'hooks', [])
assert hooks is None or len(hooks) == 0
class TestMiddlewareZeroOverhead:
"""Test that middleware has zero overhead when not used."""
def test_no_hooks_fast_path(self):
"""When no hooks registered, should take fast path."""
from praisonaiagents.hooks.middleware import MiddlewareChain
# Empty chain should be essentially a no-op
chain = MiddlewareChain([])
call_count = [0]
def handler(req):
call_count[0] += 1
return "result"
result = chain.execute({}, handler)
assert result == "result"
assert call_count[0] == 1
def test_middleware_chain_is_none_check(self):
"""Should handle None middleware list gracefully."""
from praisonaiagents.hooks.middleware import MiddlewareChain
chain = MiddlewareChain(None)
result = chain.execute({}, lambda r: "ok")
assert result == "ok"
@@ -0,0 +1,260 @@
"""
TDD Tests for Rate Limiter.
These tests are written FIRST before implementation.
"""
import pytest
import time
from unittest.mock import Mock, patch
class TestRateLimiterBasic:
"""Test basic rate limiter functionality."""
def test_rate_limiter_exists(self):
"""RateLimiter class should exist in praisonaiagents.llm."""
from praisonaiagents.llm import RateLimiter
limiter = RateLimiter(requests_per_minute=60)
assert limiter is not None
def test_rate_limiter_rpm_parameter(self):
"""RateLimiter should accept requests_per_minute."""
from praisonaiagents.llm import RateLimiter
limiter = RateLimiter(requests_per_minute=60)
assert limiter.requests_per_minute == 60
def test_rate_limiter_acquire_sync(self):
"""RateLimiter.acquire() should block if rate exceeded."""
from praisonaiagents.llm import RateLimiter
# 1 request per second = 60 per minute
limiter = RateLimiter(requests_per_minute=60)
# First acquire should be instant
start = time.monotonic()
limiter.acquire()
elapsed = time.monotonic() - start
assert elapsed < 0.1 # Should be nearly instant
def test_rate_limiter_blocks_when_exceeded(self):
"""RateLimiter should block when rate is exceeded."""
from praisonaiagents.llm import RateLimiter
# Very low rate: 6 per minute = 1 per 10 seconds
# But we'll use a mock clock to test without waiting
limiter = RateLimiter(requests_per_minute=6)
# Inject a mock clock for testing
mock_time = [0.0]
limiter._get_time = lambda: mock_time[0]
limiter._sleep = lambda s: mock_time.__setitem__(0, mock_time[0] + s)
# First request should pass
limiter.acquire()
# Second request should need to wait ~10 seconds
# (but our mock sleep just advances time)
limiter.acquire()
# Time should have advanced
assert mock_time[0] >= 10.0
def test_rate_limiter_token_bucket(self):
"""RateLimiter should use token bucket algorithm."""
from praisonaiagents.llm import RateLimiter
limiter = RateLimiter(requests_per_minute=60, burst=5)
# Reset internal state with mock time
mock_time = [0.0]
limiter._get_time = lambda: mock_time[0]
limiter._sleep = lambda s: mock_time.__setitem__(0, mock_time[0] + s)
limiter._tokens = 5.0 # Reset tokens to burst size
limiter._last_update = 0.0 # Reset last update
for _ in range(5):
limiter.acquire()
# Should not have waited much (burst allows 5 instant requests)
assert mock_time[0] < 1.0
class TestRateLimiterAsync:
"""Test async rate limiter functionality."""
@pytest.mark.asyncio
async def test_rate_limiter_acquire_async(self):
"""RateLimiter.acquire_async() should work with async."""
from praisonaiagents.llm import RateLimiter
limiter = RateLimiter(requests_per_minute=60)
# Should have async acquire method
await limiter.acquire_async()
@pytest.mark.asyncio
async def test_rate_limiter_async_blocks(self):
"""Async acquire should also respect rate limits."""
from praisonaiagents.llm import RateLimiter
import asyncio
limiter = RateLimiter(requests_per_minute=6)
# Mock async sleep
sleep_times = []
original_sleep = asyncio.sleep
async def mock_sleep(s):
sleep_times.append(s)
# Don't actually sleep in tests
with patch('asyncio.sleep', mock_sleep):
await limiter.acquire_async()
await limiter.acquire_async()
# Should have tried to sleep
class TestRateLimiterAgentIntegration:
"""Test rate limiter integration with Agent."""
def test_agent_accepts_rate_limiter(self):
"""Agent should accept rate_limiter parameter."""
from praisonaiagents import Agent
from praisonaiagents.llm import RateLimiter
limiter = RateLimiter(requests_per_minute=60)
agent = Agent(
name="Test",
instructions="Test agent",
rate_limiter=limiter
)
assert agent is not None
assert getattr(agent, '_rate_limiter', None) is limiter or \
getattr(agent, 'rate_limiter', None) is limiter
def test_agent_rate_limiter_applied_to_llm_calls(self):
"""Rate limiter should be applied before LLM calls."""
from praisonaiagents import Agent
from praisonaiagents.llm import RateLimiter
limiter = RateLimiter(requests_per_minute=60)
acquire_count = [0]
original_acquire = limiter.acquire
def counting_acquire():
acquire_count[0] += 1
return original_acquire()
limiter.acquire = counting_acquire
agent = Agent(
name="Test",
instructions="Test agent",
rate_limiter=limiter
)
with patch.object(agent, '_chat_completion', return_value="mocked"):
agent.chat("Hello")
# Rate limiter should have been called
assert acquire_count[0] >= 1
class TestRateLimiterZeroOverhead:
"""Test that rate limiter has zero overhead when not used."""
def test_no_rate_limiter_no_overhead(self):
"""When no rate limiter, should not add any overhead."""
from praisonaiagents import Agent
agent = Agent(
name="Test",
instructions="Test agent"
# No rate_limiter
)
# Should not have rate limiter
assert getattr(agent, '_rate_limiter', None) is None
def test_rate_limiter_none_check_fast(self):
"""Checking for None rate limiter should be O(1)."""
from praisonaiagents.llm import RateLimiter
# This is more of a design requirement than a test
# The implementation should do: if self._rate_limiter: self._rate_limiter.acquire()
class TestRateLimiterConfiguration:
"""Test rate limiter configuration options."""
def test_rate_limiter_default_burst(self):
"""RateLimiter should have sensible default burst."""
from praisonaiagents.llm import RateLimiter
limiter = RateLimiter(requests_per_minute=60)
# Default burst should be 1 or based on rpm
assert hasattr(limiter, 'burst') or hasattr(limiter, '_burst')
def test_rate_limiter_custom_burst(self):
"""RateLimiter should accept custom burst parameter."""
from praisonaiagents.llm import RateLimiter
limiter = RateLimiter(requests_per_minute=60, burst=10)
burst = getattr(limiter, 'burst', None) or getattr(limiter, '_burst', None)
assert burst == 10
def test_rate_limiter_tpm_optional(self):
"""RateLimiter should optionally accept tokens_per_minute."""
from praisonaiagents.llm import RateLimiter
# Should not raise
limiter = RateLimiter(
requests_per_minute=60,
tokens_per_minute=100000
)
assert limiter is not None
class TestRateLimiterDeterministic:
"""Test that rate limiter is deterministic for testing."""
def test_injectable_clock(self):
"""RateLimiter should allow injecting clock for testing."""
from praisonaiagents.llm import RateLimiter
mock_time = [0.0]
limiter = RateLimiter(requests_per_minute=60)
limiter._get_time = lambda: mock_time[0]
# Should use our mock time
t1 = limiter._get_time()
mock_time[0] = 100.0
t2 = limiter._get_time()
assert t1 == 0.0
assert t2 == 100.0
def test_injectable_sleep(self):
"""RateLimiter should allow injecting sleep for testing."""
from praisonaiagents.llm import RateLimiter
sleep_calls = []
limiter = RateLimiter(requests_per_minute=60)
limiter._sleep = lambda s: sleep_calls.append(s)
# Force a wait scenario
limiter._get_time = lambda: 0.0
limiter._tokens = 0 # No tokens available
limiter._last_update = 0.0
limiter.acquire()
# Should have called our mock sleep
# (actual behavior depends on implementation)
+1 -1
View File
@@ -2422,7 +2422,7 @@ wheels = [
[[package]]
name = "praisonaiagents"
version = "0.2.0"
version = "0.3.0"
source = { editable = "." }
dependencies = [
{ name = "aiohttp" },
+291 -1
View File
@@ -261,6 +261,16 @@ npm install praisonai
| ↳ 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
| ↳ Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
| ↳ Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
| ↳ Middleware System | [Example](examples/middleware/basic_middleware.py) | [📖](https://docs.praison.ai/features/middleware) |
| ↳ Configurable Model | [Example](examples/middleware/configurable_model.py) | [📖](https://docs.praison.ai/features/configurable-model) |
| ↳ Rate Limiter | [Example](examples/middleware/rate_limiter.py) | [📖](https://docs.praison.ai/features/rate-limiter) |
| ↳ Injected Tool State | [Example](examples/middleware/injected_state.py) | [📖](https://docs.praison.ai/features/injected-state) |
| ↳ Shadow Git Checkpoints | [Example](#10-shadow-git-checkpoints) | [📖](https://docs.praison.ai/features/checkpoints) |
| ↳ Background Tasks | [Example](examples/background/basic_background.py) | [📖](https://docs.praison.ai/features/background-tasks) |
| ↳ Policy Engine | [Example](examples/policy/basic_policy.py) | [📖](https://docs.praison.ai/features/policy-engine) |
| ↳ Thinking Budgets | [Example](examples/thinking/basic_thinking.py) | [📖](https://docs.praison.ai/features/thinking-budgets) |
| ↳ Output Styles | [Example](examples/output/basic_output.py) | [📖](https://docs.praison.ai/features/output-styles) |
| ↳ Context Compaction | [Example](examples/compaction/basic_compaction.py) | [📖](https://docs.praison.ai/features/context-compaction) |
| **📊 Monitoring & Management** | | |
| ↳ Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
| ↳ Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
@@ -930,8 +940,191 @@ praisonai checkpoint diff # Show changes
praisonai checkpoint restore abc123 # Restore to checkpoint
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/checkpoints)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/checkpoint)
- [💻 Example](examples/checkpoints/basic_checkpoints.py)
### 11. Field Names Reference (A-I-G-S)
---
### 11. Background Tasks
Run agent tasks asynchronously without blocking:
```python
import asyncio
from praisonaiagents.background import BackgroundRunner, BackgroundConfig
async def main():
config = BackgroundConfig(max_concurrent_tasks=3)
runner = BackgroundRunner(config=config)
async def my_task(name: str) -> str:
await asyncio.sleep(2)
return f"Task {name} completed"
task = await runner.submit(my_task, args=("example",), name="my_task")
await task.wait(timeout=10.0)
print(task.result)
asyncio.run(main())
```
**CLI Commands:**
```bash
praisonai background list # List running tasks
praisonai background status <id> # Check task status
praisonai background cancel <id> # Cancel a task
praisonai background clear # Clear completed tasks
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/background-tasks)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/background)
- [💻 Example](examples/background/basic_background.py)
---
### 12. Policy Engine
Control what agents can and cannot do with policy-based execution:
```python
from praisonaiagents.policy import (
PolicyEngine, Policy, PolicyRule, PolicyAction
)
engine = PolicyEngine()
policy = Policy(
name="no_delete",
rules=[
PolicyRule(
action=PolicyAction.DENY,
resource="tool:delete_*",
reason="Delete operations blocked"
)
]
)
engine.add_policy(policy)
result = engine.check("tool:delete_file", {})
print(f"Allowed: {result.allowed}")
```
**CLI Commands:**
```bash
praisonai policy list # List policies
praisonai policy check "tool:name" # Check if allowed
praisonai policy init # Create template
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/policy-engine)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/policy)
- [💻 Example](examples/policy/basic_policy.py)
---
### 13. Thinking Budgets
Configure token budgets for extended thinking:
```python
from praisonaiagents.thinking import ThinkingBudget, ThinkingTracker
# Use predefined levels
budget = ThinkingBudget.high() # 16,000 tokens
# Track usage
tracker = ThinkingTracker()
session = tracker.start_session(budget_tokens=16000)
tracker.end_session(session, tokens_used=12000)
summary = tracker.get_summary()
print(f"Utilization: {summary['average_utilization']:.1%}")
```
**CLI Commands:**
```bash
praisonai thinking status # Show current budget
praisonai thinking set high # Set budget level
praisonai thinking stats # Show usage statistics
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/thinking-budgets)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/thinking)
- [💻 Example](examples/thinking/basic_thinking.py)
---
### 14. Output Styles
Configure how agents format their responses:
```python
from praisonaiagents.output import OutputStyle, OutputFormatter
# Use preset styles
style = OutputStyle.concise()
formatter = OutputFormatter(style)
# Format output
text = "# Hello\n\nThis is **bold** text."
plain = formatter.format(text)
print(plain)
```
**CLI Commands:**
```bash
praisonai output status # Show current style
praisonai output set concise # Set output style
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/output-styles)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/output-style)
- [💻 Example](examples/output/basic_output.py)
---
### 15. Context Compaction
Automatically manage context window size:
```python
from praisonaiagents.compaction import (
ContextCompactor, CompactionStrategy
)
compactor = ContextCompactor(
max_tokens=4000,
strategy=CompactionStrategy.SLIDING,
preserve_recent=3
)
messages = [...] # Your conversation history
compacted, result = compactor.compact(messages)
print(f"Compression: {result.compression_ratio:.1%}")
```
**CLI Commands:**
```bash
praisonai compaction status # Show settings
praisonai compaction set sliding # Set strategy
praisonai compaction stats # Show statistics
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/context-compaction)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/compaction)
- [💻 Example](examples/compaction/basic_compaction.py)
---
### 16. Field Names Reference (A-I-G-S)
PraisonAI accepts both old (agents.yaml) and new (workflow.yaml) field names. Use the **canonical names** for new projects:
@@ -2535,6 +2728,103 @@ PraisonAI provides zero-dependency persistent memory for agents. For detailed ex
---
## 💾 Persistence (Databases)
Enable automatic conversation persistence with 2 lines of code:
```python
from praisonaiagents import Agent, db
agent = Agent(
name="Assistant",
db=db(database_url="postgresql://localhost/mydb"), # db(...) shortcut
session_id="my-session" # Optional: defaults to per-hour ID (YYYYMMDDHH)
)
agent.chat("Hello!") # Auto-persists messages, runs, traces
```
### Persistence CLI Commands
| Command | Description |
|---------|-------------|
| `praisonai persistence doctor` | Validate DB connectivity |
| `praisonai persistence run` | Run agent with persistence |
| `praisonai persistence resume` | Resume existing session |
| `praisonai persistence export` | Export session to JSONL |
| `praisonai persistence import` | Import session from JSONL |
| `praisonai persistence migrate` | Apply schema migrations |
| `praisonai persistence status` | Show schema status |
### Databases Table
| Database | Store Type | Install | Example | Docs |
|----------|------------|---------|---------|------|
| PostgreSQL | Conversation | `pip install "praisonai[tools]"` | [simple_db_agent.py](examples/persistence/simple_db_agent.py) | [docs](https://docs.praison.ai/docs/databases/postgres) |
| MySQL | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| SQLite | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| SingleStore | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Supabase | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| SurrealDB | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Qdrant | Knowledge | `pip install "praisonai[tools]"` | [knowledge_qdrant.py](examples/persistence/knowledge_qdrant.py) | [docs](https://docs.praison.ai/docs/databases/qdrant) |
| ChromaDB | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Pinecone | Knowledge | `pip install pinecone` | [pinecone_wow.py](examples/vector/pinecone_wow.py) | [docs](https://docs.praison.ai/docs/databases/pinecone) |
| Weaviate | Knowledge | `pip install weaviate-client` | [weaviate_wow.py](examples/vector/weaviate_wow.py) | [docs](https://docs.praison.ai/docs/databases/weaviate) |
| LanceDB | Knowledge | `pip install lancedb` | [lancedb_real_wow.py](examples/vector/lancedb_real_wow.py) | [docs](https://docs.praison.ai/docs/databases/lancedb) |
| Milvus | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| PGVector | Knowledge | `pip install psycopg2-binary` | [pgvector_real_wow.py](examples/vector/pgvector_real_wow.py) | [docs](https://docs.praison.ai/docs/databases/pgvector) |
| Redis Vector | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Cassandra | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| ClickHouse | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Redis | State | `pip install "praisonai[tools]"` | [state_redis.py](examples/persistence/state_redis.py) | [docs](https://docs.praison.ai/docs/databases/redis) |
| MongoDB | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| DynamoDB | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Firestore | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Upstash | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Memory | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
---
## 🔧 Tools Table
Install all tools with: `pip install "praisonai[tools]"`
| Tool | Category | Import | Docs |
|------|----------|--------|------|
| Tavily | Web Search | `from praisonai_tools import TavilyTool` | [docs](https://docs.praison.ai/docs/tools/external/tavily) |
| DuckDuckGo | Web Search | `from praisonai_tools import DuckDuckGoTool` | [docs](https://docs.praison.ai/docs/tools/external/duckduckgo) |
| Exa | Web Search | `from praisonai_tools import ExaTool` | [docs](https://docs.praison.ai/docs/tools/external/exa) |
| Serper | Web Search | `from praisonai_tools import SerperTool` | [docs](https://docs.praison.ai/docs/tools/external/serper) |
| Jina | Web Reader | `from praisonai_tools import JinaTool` | [docs](https://docs.praison.ai/docs/tools/external/jina) |
| Firecrawl | Web Scraping | `from praisonai_tools import FirecrawlTool` | [docs](https://docs.praison.ai/docs/tools/external/firecrawl) |
| Crawl4AI | Web Scraping | `from praisonai_tools import Crawl4AITool` | [docs](https://docs.praison.ai/docs/tools/external/crawl4ai) |
| Wikipedia | Knowledge | `from praisonai_tools import WikipediaTool` | [docs](https://docs.praison.ai/docs/tools/external/wikipedia) |
| ArXiv | Research | `from praisonai_tools import ArxivTool` | [docs](https://docs.praison.ai/docs/tools/external/arxiv) |
| HackerNews | News | `from praisonai_tools import HackerNewsTool` | [docs](https://docs.praison.ai/docs/tools/external/hackernews) |
| YouTube | Media | `from praisonai_tools import YouTubeTool` | [docs](https://docs.praison.ai/docs/tools/external/youtube) |
| Weather | Data | `from praisonai_tools import WeatherTool` | [docs](https://docs.praison.ai/docs/tools/external/weather) |
| PostgreSQL | Database | `from praisonai_tools import PostgresTool` | [docs](https://docs.praison.ai/docs/tools/external/postgres) |
| MySQL | Database | `from praisonai_tools import MySQLTool` | [docs](https://docs.praison.ai/docs/tools/external/mysql) |
| SQLite | Database | `from praisonai_tools import SQLiteTool` | [docs](https://docs.praison.ai/docs/tools/external/sqlite) |
| MongoDB | Database | `from praisonai_tools import MongoDBTool` | [docs](https://docs.praison.ai/docs/tools/external/mongodb) |
| Redis | Database | `from praisonai_tools import RedisTool` | [docs](https://docs.praison.ai/docs/tools/external/redis) |
| Qdrant | Vector DB | `from praisonai_tools import QdrantTool` | [docs](https://docs.praison.ai/docs/tools/external/qdrant) |
| GitHub | DevOps | `from praisonai_tools import GitHubTool` | [docs](https://docs.praison.ai/docs/tools/external/github) |
| Slack | Communication | `from praisonai_tools import SlackTool` | [docs](https://docs.praison.ai/docs/tools/external/slack) |
| Discord | Communication | `from praisonai_tools import DiscordTool` | [docs](https://docs.praison.ai/docs/tools/external/discord) |
| Telegram | Communication | `from praisonai_tools import TelegramTool` | [docs](https://docs.praison.ai/docs/tools/external/telegram) |
| Email | Communication | `from praisonai_tools import EmailTool` | [docs](https://docs.praison.ai/docs/tools/external/email) |
| Notion | Productivity | `from praisonai_tools import NotionTool` | [docs](https://docs.praison.ai/docs/tools/external/notion) |
| File | File System | `from praisonai_tools import FileTool` | [docs](https://docs.praison.ai/docs/tools/external/file) |
| Shell | System | `from praisonai_tools import ShellTool` | [docs](https://docs.praison.ai/docs/tools/external/shell) |
| Python | Code | `from praisonai_tools import PythonTool` | [docs](https://docs.praison.ai/docs/tools/external/python) |
| JSON | Data | `from praisonai_tools import JSONTool` | [docs](https://docs.praison.ai/docs/tools/external/json) |
| CSV | Data | `from praisonai_tools import CSVTool` | [docs](https://docs.praison.ai/docs/tools/external/csv) |
| Calculator | Math | `from praisonai_tools import CalculatorTool` | [docs](https://docs.praison.ai/docs/tools/external/calculator) |
> See [full tools documentation](https://docs.praison.ai/docs/tools/tools) for all 100+ available tools.
---
## 🎓 Video Tutorials
Learn PraisonAI through our comprehensive video series:
+2 -2
View File
@@ -3,8 +3,8 @@ class Praisonai < Formula
desc "AI tools for various AI applications"
homepage "https://github.com/MervinPraison/PraisonAI"
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.4.0.tar.gz"
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.4.0.tar.gz | shasum -a 256`.split.first
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.5.0.tar.gz"
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.5.0.tar.gz | shasum -a 256`.split.first
license "MIT"
depends_on "[email protected]"
@@ -0,0 +1,293 @@
"""
Observability CLI Handler
CLI commands for managing observability providers.
Usage: praisonai obs <command>
"""
import os
from typing import Any, Dict, List, Optional, Tuple
from .base import FlagHandler
class ObservabilityHandler(FlagHandler):
"""
Handler for observability CLI commands.
Commands:
praisonai obs list - List available providers
praisonai obs doctor - Check provider connectivity
praisonai obs init <name> - Initialize a provider
"""
@property
def feature_name(self) -> str:
return "observability"
@property
def flag_name(self) -> str:
return "obs"
@property
def flag_help(self) -> str:
return "Observability commands (list, doctor, init)"
def check_dependencies(self) -> Tuple[bool, str]:
"""Check if observability module is available."""
try:
from praisonai_tools.observability import obs
return True, ""
except ImportError:
return False, "Observability requires praisonai-tools. Install with: pip install praisonai-tools"
def list_providers(self) -> Dict[str, Any]:
"""List all available observability providers."""
result = {
"registered": [],
"available": [],
"configured": [],
}
try:
from praisonai_tools.observability.manager import ObservabilityManager
from praisonai_tools.observability.config import PROVIDER_ENV_KEYS
mgr = ObservabilityManager()
# Get all registered providers
result["registered"] = mgr.list_providers()
# Check which have env vars configured
for provider, keys in PROVIDER_ENV_KEYS.items():
if all(os.getenv(key) for key in keys):
result["configured"].append(provider)
# Check which SDKs are available
for provider in result["registered"]:
try:
mgr._load_provider(provider)
if provider in mgr._providers:
provider_class = mgr._providers[provider]
instance = provider_class()
if instance.is_available():
result["available"].append(provider)
except Exception:
pass
except ImportError:
pass
return result
def doctor(self, provider: Optional[str] = None) -> Dict[str, Any]:
"""Run diagnostics on observability setup."""
try:
from praisonai_tools.observability import obs
if provider:
obs.init(provider=provider)
return obs.doctor()
except ImportError:
return {
"error": "praisonai-tools not installed",
"enabled": False,
}
def init_provider(self, provider: str, **kwargs) -> bool:
"""Initialize a specific provider."""
try:
from praisonai_tools.observability import obs
return obs.init(provider=provider, **kwargs)
except ImportError:
return False
def execute(self, action: str = "list", **kwargs) -> Any:
"""
Execute observability command.
Args:
action: Command to execute (list, doctor, init)
**kwargs: Additional arguments
Returns:
Command result
"""
if action == "list":
return self._execute_list()
elif action == "doctor":
return self._execute_doctor(kwargs.get("provider"))
elif action == "init":
return self._execute_init(kwargs.get("provider"), **kwargs)
else:
self.print_status(f"Unknown action: {action}", "error")
return None
def _execute_list(self) -> Dict[str, Any]:
"""Execute list command."""
result = self.list_providers()
self.print_status("\n📊 Observability Providers", "info")
self.print_status("=" * 40, "info")
# Provider status table
all_providers = [
"agentops", "langfuse", "langsmith", "traceloop",
"arize_phoenix", "openlit", "langtrace", "langwatch",
"datadog", "mlflow", "opik", "portkey",
"braintrust", "maxim", "weave", "neatlogs",
"langdb", "atla", "patronus", "truefoundry",
]
for provider in all_providers:
status_parts = []
if provider in result.get("available", []):
status_parts.append("✅ SDK")
else:
status_parts.append("❌ SDK")
if provider in result.get("configured", []):
status_parts.append("🔑 Keys")
else:
status_parts.append("⚪ Keys")
status = " | ".join(status_parts)
self.print_status(f" {provider:20} {status}", "info")
self.print_status("\n✅ = Available ❌ = Not installed 🔑 = API key configured", "info")
return result
def _execute_doctor(self, provider: Optional[str] = None) -> Dict[str, Any]:
"""Execute doctor command."""
result = self.doctor(provider)
self.print_status("\n🩺 Observability Diagnostics", "info")
self.print_status("=" * 40, "info")
self.print_status(f" Enabled: {result.get('enabled', False)}", "info")
self.print_status(f" Provider: {result.get('provider', 'None')}", "info")
if result.get("connection_status") is not None:
status = "✅" if result["connection_status"] else "❌"
self.print_status(f" Connection: {status} {result.get('connection_message', '')}", "info")
if result.get("available_providers"):
self.print_status(f" Available: {', '.join(result['available_providers'])}", "info")
return result
def _execute_init(self, provider: str, **kwargs) -> bool:
"""Execute init command."""
if not provider:
self.print_status("Provider name required. Usage: praisonai obs init <provider>", "error")
return False
success = self.init_provider(provider, **kwargs)
if success:
self.print_status(f"✅ {provider} initialized successfully", "success")
else:
self.print_status(f"❌ Failed to initialize {provider}", "error")
self.print_status(f" Check that {provider} SDK is installed and API keys are set", "info")
return success
def apply_to_agent_config(self, config: Dict[str, Any], flag_value: Any) -> Dict[str, Any]:
"""Apply observability to agent config."""
if flag_value:
try:
from praisonai_tools.observability import obs
if isinstance(flag_value, str):
obs.init(provider=flag_value)
else:
obs.init()
self.print_status("📊 Observability enabled", "success")
except ImportError:
self.print_status("Observability requires praisonai-tools", "warning")
return config
# Provider environment variable reference
PROVIDER_SETUP_GUIDE = {
"agentops": {
"env_vars": ["AGENTOPS_API_KEY"],
"install": "pip install agentops",
"docs": "https://agentops.ai/",
},
"langfuse": {
"env_vars": ["LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY"],
"install": "pip install opentelemetry-sdk opentelemetry-exporter-otlp",
"docs": "https://langfuse.com/",
},
"langsmith": {
"env_vars": ["LANGSMITH_API_KEY"],
"install": "pip install opentelemetry-sdk opentelemetry-exporter-otlp",
"docs": "https://smith.langchain.com/",
},
"traceloop": {
"env_vars": ["TRACELOOP_API_KEY"],
"install": "pip install traceloop-sdk",
"docs": "https://traceloop.com/",
},
"arize_phoenix": {
"env_vars": ["PHOENIX_API_KEY"],
"install": "pip install arize-phoenix openinference-instrumentation",
"docs": "https://phoenix.arize.com/",
},
"openlit": {
"env_vars": [],
"install": "pip install openlit",
"docs": "https://github.com/openlit/openlit",
},
"langtrace": {
"env_vars": ["LANGTRACE_API_KEY"],
"install": "pip install langtrace-python-sdk",
"docs": "https://langtrace.ai/",
},
"langwatch": {
"env_vars": ["LANGWATCH_API_KEY"],
"install": "pip install langwatch",
"docs": "https://langwatch.ai/",
},
"datadog": {
"env_vars": ["DD_API_KEY"],
"install": "pip install ddtrace",
"docs": "https://www.datadoghq.com/product/llm-observability/",
},
"mlflow": {
"env_vars": ["MLFLOW_TRACKING_URI"],
"install": "pip install mlflow",
"docs": "https://mlflow.org/",
},
"opik": {
"env_vars": ["OPIK_API_KEY"],
"install": "pip install opik",
"docs": "https://www.comet.com/docs/opik/",
},
"portkey": {
"env_vars": ["PORTKEY_API_KEY"],
"install": "pip install portkey-ai",
"docs": "https://portkey.ai/",
},
"braintrust": {
"env_vars": ["BRAINTRUST_API_KEY"],
"install": "pip install braintrust",
"docs": "https://www.braintrust.dev/",
},
"maxim": {
"env_vars": ["MAXIM_API_KEY"],
"install": "pip install maxim-py",
"docs": "https://getmaxim.ai/",
},
"weave": {
"env_vars": ["WANDB_API_KEY"],
"install": "pip install weave",
"docs": "https://weave-docs.wandb.ai/",
},
}
@@ -0,0 +1,636 @@
"""
Persistence CLI feature for PraisonAI.
Provides commands for database persistence configuration and management:
- praisonai persistence doctor - Validate DB connectivity
- praisonai persistence run - Run agent with persistence
- praisonai persistence resume - Resume a session
"""
import os
import sys
import logging
from typing import Optional
logger = logging.getLogger(__name__)
def handle_persistence_command(args: list) -> bool:
"""
Handle persistence subcommands.
Args:
args: Command line arguments after 'persistence'
Returns:
True if command was handled, False otherwise
"""
if not args:
print_persistence_help()
return True
subcommand = args[0].lower()
sub_args = args[1:]
if subcommand == "doctor":
return handle_doctor(sub_args)
elif subcommand == "run":
return handle_run(sub_args)
elif subcommand == "resume":
return handle_resume(sub_args)
elif subcommand == "export":
return handle_export(sub_args)
elif subcommand == "import":
return handle_import(sub_args)
elif subcommand == "migrate":
return handle_migrate(sub_args)
elif subcommand == "status":
return handle_status(sub_args)
elif subcommand in ("--help", "-h", "help"):
print_persistence_help()
return True
else:
print(f"Unknown persistence subcommand: {subcommand}")
print_persistence_help()
return True
def print_persistence_help():
"""Print persistence command help."""
help_text = """
PraisonAI Persistence Commands
==============================
Usage: praisonai persistence <command> [options]
Commands:
doctor Validate database connectivity and configuration
run Run an agent with persistence enabled
resume Resume an existing session
Doctor Command:
praisonai persistence doctor [options]
Options:
--conversation-url URL Conversation store URL (postgres/mysql/sqlite)
--knowledge-url URL Knowledge store URL (qdrant/chroma/pinecone)
--state-url URL State store URL (redis/memory)
--all Test all configured stores
Environment Variables:
PRAISON_CONVERSATION_URL Default conversation store URL
PRAISON_KNOWLEDGE_URL Default knowledge store URL
PRAISON_STATE_URL Default state store URL
Run Command:
praisonai persistence run [options] "prompt"
Options:
--session-id ID Session identifier (auto-generated if not provided)
--user-id ID User identifier (default: "default")
--conversation-url URL Conversation store URL
--knowledge-url URL Knowledge store URL
--state-url URL State store URL
--agent-name NAME Agent name (default: "Assistant")
--agent-instructions TEXT Agent instructions
--dry-run Show configuration without running
Resume Command:
praisonai persistence resume --session-id ID [options]
Options:
--session-id ID Session to resume (required)
--conversation-url URL Conversation store URL
--show-history Display conversation history
--continue "prompt" Continue with a new prompt
Export/Import Commands:
praisonai persistence export --session-id ID [options]
praisonai persistence import --file FILE [options]
Options:
--session-id ID Session to export (required for export)
--conversation-url URL Conversation store URL
--file FILE File path for export/import (JSONL format)
--output FILE Output file path (default: session-{id}.jsonl)
Examples:
# Check PostgreSQL connectivity
praisonai persistence doctor --conversation-url "postgresql://localhost/mydb"
# Run agent with persistence
praisonai persistence run --session-id "my-session" "Hello, remember my name is Alice"
# Resume a session
praisonai persistence resume --session-id "my-session" --show-history
# Continue a session
praisonai persistence resume --session-id "my-session" --continue "What's my name?"
# Export a session
praisonai persistence export --session-id "my-session" --output session.jsonl
# Import a session
praisonai persistence import --file session.jsonl
"""
print(help_text)
def parse_persistence_args(args: list) -> dict:
"""Parse persistence command arguments."""
config = {
"session_id": None,
"user_id": "default",
"conversation_url": os.getenv("PRAISON_CONVERSATION_URL"),
"knowledge_url": os.getenv("PRAISON_KNOWLEDGE_URL"),
"state_url": os.getenv("PRAISON_STATE_URL"),
"agent_name": "Assistant",
"agent_instructions": "You are a helpful assistant.",
"dry_run": False,
"show_history": False,
"all_stores": False,
"prompt": None,
"continue_prompt": None,
"output_file": None,
"input_file": None,
}
i = 0
while i < len(args):
arg = args[i]
if arg == "--session-id" and i + 1 < len(args):
config["session_id"] = args[i + 1]
i += 2
elif arg == "--user-id" and i + 1 < len(args):
config["user_id"] = args[i + 1]
i += 2
elif arg == "--conversation-url" and i + 1 < len(args):
config["conversation_url"] = args[i + 1]
i += 2
elif arg == "--knowledge-url" and i + 1 < len(args):
config["knowledge_url"] = args[i + 1]
i += 2
elif arg == "--state-url" and i + 1 < len(args):
config["state_url"] = args[i + 1]
i += 2
elif arg == "--agent-name" and i + 1 < len(args):
config["agent_name"] = args[i + 1]
i += 2
elif arg == "--agent-instructions" and i + 1 < len(args):
config["agent_instructions"] = args[i + 1]
i += 2
elif arg == "--dry-run":
config["dry_run"] = True
i += 1
elif arg == "--show-history":
config["show_history"] = True
i += 1
elif arg == "--all":
config["all_stores"] = True
i += 1
elif arg == "--continue" and i + 1 < len(args):
config["continue_prompt"] = args[i + 1]
i += 2
elif arg == "--output" and i + 1 < len(args):
config["output_file"] = args[i + 1]
i += 2
elif arg == "--file" and i + 1 < len(args):
config["input_file"] = args[i + 1]
i += 2
elif not arg.startswith("-"):
config["prompt"] = arg
i += 1
else:
i += 1
return config
def handle_doctor(args: list) -> bool:
"""Handle persistence doctor command."""
config = parse_persistence_args(args)
print("=" * 50)
print("PraisonAI Persistence Doctor")
print("=" * 50)
results = []
# Test conversation store
if config["conversation_url"] or config["all_stores"]:
url = config["conversation_url"]
if url:
print(f"\n[Conversation Store] Testing: {_mask_url(url)}")
success, msg = _test_conversation_store(url)
results.append(("Conversation", success, msg))
print(f" {'✅' if success else '❌'} {msg}")
else:
print("\n[Conversation Store] Not configured (set --conversation-url or PRAISON_CONVERSATION_URL)")
# Test knowledge store
if config["knowledge_url"] or config["all_stores"]:
url = config["knowledge_url"]
if url:
print(f"\n[Knowledge Store] Testing: {_mask_url(url)}")
success, msg = _test_knowledge_store(url)
results.append(("Knowledge", success, msg))
print(f" {'✅' if success else '❌'} {msg}")
else:
print("\n[Knowledge Store] Not configured (set --knowledge-url or PRAISON_KNOWLEDGE_URL)")
# Test state store
if config["state_url"] or config["all_stores"]:
url = config["state_url"]
if url:
print(f"\n[State Store] Testing: {_mask_url(url)}")
success, msg = _test_state_store(url)
results.append(("State", success, msg))
print(f" {'✅' if success else '❌'} {msg}")
else:
print("\n[State Store] Not configured (set --state-url or PRAISON_STATE_URL)")
# Summary
print("\n" + "=" * 50)
if results:
passed = sum(1 for _, s, _ in results if s)
total = len(results)
print(f"Results: {passed}/{total} stores connected successfully")
else:
print("No stores configured. Use --conversation-url, --knowledge-url, or --state-url")
print("=" * 50)
return True
def handle_run(args: list) -> bool:
"""Handle persistence run command."""
config = parse_persistence_args(args)
if not config["prompt"]:
print("Error: No prompt provided")
print("Usage: praisonai persistence run [options] \"your prompt\"")
return True
if not config["conversation_url"]:
print("Error: No conversation URL configured")
print("Set --conversation-url or PRAISON_CONVERSATION_URL")
return True
if config["dry_run"]:
print("=" * 50)
print("Dry Run - Configuration")
print("=" * 50)
print(f"Session ID: {config['session_id'] or '(auto-generated)'}")
print(f"User ID: {config['user_id']}")
print(f"Conversation URL: {_mask_url(config['conversation_url'])}")
print(f"Agent Name: {config['agent_name']}")
print(f"Prompt: {config['prompt'][:50]}...")
return True
# Run agent with persistence
try:
from praisonaiagents import Agent, db
db_instance = db(
database_url=config["conversation_url"],
state_url=config["state_url"],
knowledge_url=config["knowledge_url"],
)
agent = Agent(
name=config["agent_name"],
instructions=config["agent_instructions"],
db=db_instance,
session_id=config["session_id"],
verbose=True
)
print(f"\n[Session: {agent.session_id}]")
response = agent.chat(config["prompt"])
print(f"\nAgent: {response}")
db_instance.close()
except Exception as e:
print(f"Error: {e}")
return True
return True
def handle_resume(args: list) -> bool:
"""Handle persistence resume command."""
config = parse_persistence_args(args)
if not config["session_id"]:
print("Error: --session-id is required for resume")
return True
if not config["conversation_url"]:
print("Error: No conversation URL configured")
print("Set --conversation-url or PRAISON_CONVERSATION_URL")
return True
try:
from praisonaiagents import db
db_instance = db(database_url=config["conversation_url"])
# Get session history
history = db_instance.on_agent_start(
agent_name="Resume",
session_id=config["session_id"],
user_id=config["user_id"]
)
print(f"\n[Session: {config['session_id']}]")
print(f"Messages in history: {len(history)}")
if config["show_history"]:
print("\n--- Conversation History ---")
for msg in history:
role = msg.role.upper()
content = msg.content[:100] + "..." if len(msg.content) > 100 else msg.content
print(f"[{role}] {content}")
print("--- End History ---\n")
if config["continue_prompt"]:
from praisonaiagents import Agent
agent = Agent(
name=config["agent_name"],
instructions=config["agent_instructions"],
db=db_instance,
session_id=config["session_id"],
verbose=True
)
response = agent.chat(config["continue_prompt"])
print(f"\nAgent: {response}")
db_instance.close()
except Exception as e:
print(f"Error: {e}")
return True
return True
def handle_export(args: list) -> bool:
"""Handle persistence export command."""
config = parse_persistence_args(args)
if not config["session_id"]:
print("Error: --session-id is required for export")
return True
if not config["conversation_url"]:
print("Error: No conversation URL configured")
return True
try:
from praisonaiagents import db
import json
db_instance = db(database_url=config["conversation_url"])
# Export session
data = db_instance.export_session(config["session_id"])
if not data:
print(f"Error: Session {config['session_id']} not found")
db_instance.close()
return True
# Determine output file
output_file = config.get("output_file") or f"session-{config['session_id']}.jsonl"
with open(output_file, 'w') as f:
f.write(json.dumps(data, default=str) + '\n')
print(f"Exported session to: {output_file}")
print(f"Messages: {len(data.get('messages', []))}")
db_instance.close()
except Exception as e:
print(f"Error: {e}")
return True
return True
def handle_import(args: list) -> bool:
"""Handle persistence import command."""
config = parse_persistence_args(args)
input_file = config.get("input_file")
if not input_file:
print("Error: --file is required for import")
return True
if not config["conversation_url"]:
print("Error: No conversation URL configured")
return True
try:
from praisonaiagents import db
import json
db_instance = db(database_url=config["conversation_url"])
with open(input_file, 'r') as f:
data = json.loads(f.readline())
session_id = db_instance.import_session(data)
print(f"Imported session: {session_id}")
print(f"Messages: {len(data.get('messages', []))}")
db_instance.close()
except Exception as e:
print(f"Error: {e}")
return True
return True
def handle_migrate(args: list) -> bool:
"""Handle persistence migrate command."""
config = parse_persistence_args(args)
if not config["conversation_url"] and not config["state_url"]:
print("Error: No database URL configured")
print("Set --conversation-url or --state-url")
return True
try:
from praisonaiagents import db
from praisonai.persistence.migrations import MigrationManager
db_instance = db(
database_url=config["conversation_url"],
state_url=config["state_url"]
)
manager = MigrationManager(db_instance)
# Check for --up or --down flags
direction = "up"
target = None
for i, arg in enumerate(args):
if arg == "--down":
direction = "down"
elif arg == "--target" and i + 1 < len(args):
target = args[i + 1]
if direction == "up":
print("Applying pending migrations...")
applied = manager.migrate_up(target)
if applied:
print(f"Applied migrations: {', '.join(applied)}")
else:
print("No pending migrations")
else:
if not target:
print("Error: --target required for --down migration")
return True
print(f"Rolling back to version {target}...")
rolled_back = manager.migrate_down(target)
if rolled_back:
print(f"Rolled back: {', '.join(rolled_back)}")
else:
print("No migrations to rollback")
db_instance.close()
except Exception as e:
print(f"Error: {e}")
return True
return True
def handle_status(args: list) -> bool:
"""Handle persistence status command."""
config = parse_persistence_args(args)
if not config["conversation_url"] and not config["state_url"]:
print("Error: No database URL configured")
return True
try:
from praisonaiagents import db
from praisonai.persistence.migrations import MigrationManager
db_instance = db(
database_url=config["conversation_url"],
state_url=config["state_url"]
)
manager = MigrationManager(db_instance)
status = manager.get_migration_status()
print("\n=== Schema Status ===")
print(f"Current version: {status['current_version'] or '(not set)'}")
print(f"Latest version: {status['latest_version']}")
print(f"Pending: {status['pending_count']} migration(s)")
if status['pending_versions']:
print(f"Pending versions: {', '.join(status['pending_versions'])}")
db_instance.close()
except Exception as e:
print(f"Error: {e}")
return True
return True
def _mask_url(url: str) -> str:
"""Mask sensitive parts of URL."""
if not url:
return "(not set)"
# Mask password in URLs
import re
masked = re.sub(r'://([^:]+):([^@]+)@', r'://\1:****@', url)
return masked
def _test_conversation_store(url: str) -> tuple:
"""Test conversation store connectivity."""
try:
from praisonai.persistence.factory import create_conversation_store
# Detect backend
if "postgresql" in url or "postgres" in url:
backend = "postgres"
elif "mysql" in url:
backend = "mysql"
elif url.endswith(".db") or "sqlite" in url:
backend = "sqlite"
else:
backend = "sqlite"
store = create_conversation_store(backend, url=url)
# Try a simple operation
sessions = store.list_sessions(limit=1)
store.close()
return True, f"Connected ({backend})"
except Exception as e:
return False, str(e)
def _test_knowledge_store(url: str) -> tuple:
"""Test knowledge store connectivity."""
try:
from praisonai.persistence.factory import create_knowledge_store
# Detect backend
if "qdrant" in url or ":6333" in url:
backend = "qdrant"
elif "chroma" in url:
backend = "chroma"
else:
backend = "qdrant"
store = create_knowledge_store(backend, url=url)
# Try a simple operation
exists = store.collection_exists("test_doctor")
store.close()
return True, f"Connected ({backend})"
except Exception as e:
return False, str(e)
def _test_state_store(url: str) -> tuple:
"""Test state store connectivity."""
try:
from praisonai.persistence.factory import create_state_store
# Detect backend
if "redis" in url:
backend = "redis"
else:
backend = "memory"
store = create_state_store(backend, url=url)
# Try a simple operation
store.set("_doctor_test", "ok", ttl=5)
value = store.get("_doctor_test")
store.delete("_doctor_test")
store.close()
return True, f"Connected ({backend})"
except Exception as e:
return False, str(e)
+15
View File
@@ -266,6 +266,12 @@ class PraisonAI:
file_input = self.read_file_if_provided(getattr(args, 'file', None))
if args.command:
# Handle persistence command
if args.command == "persistence":
from praisonai.cli.features.persistence import handle_persistence_command
handle_persistence_command(unknown_args)
return None
# Handle schedule command
if args.command == "schedule":
from praisonai.cli.features.agent_scheduler import AgentSchedulerHandler
@@ -807,6 +813,15 @@ class PraisonAI:
parser.add_argument('--max-cost', type=float, help='Maximum total cost budget in USD')
parser.add_argument('--daemon', action='store_true', help=argparse.SUPPRESS) # Hidden flag for daemon mode
# Rate Limiter - control API request rate
parser.add_argument("--rpm", type=int, help="Rate limit: requests per minute for LLM calls")
parser.add_argument("--tpm", type=int, help="Rate limit: tokens per minute for LLM calls (optional)")
# Configurable Model - runtime model switching
parser.add_argument("--configurable-model", action="store_true", help="Enable runtime model switching via config parameter")
parser.add_argument("--temperature", type=float, help="Override temperature for LLM calls")
parser.add_argument("--llm-provider", type=str, help="Override LLM provider (openai, anthropic, google, etc.)")
# If we're in a test environment, parse with empty args to avoid pytest interference
if in_test_env:
args, unknown_args = parser.parse_known_args([])
+1 -1
View File
@@ -57,7 +57,7 @@ class CloudDeployer:
file.write("FROM python:3.11-slim\n")
file.write("WORKDIR /app\n")
file.write("COPY . .\n")
file.write("RUN pip install flask praisonai==2.4.0 gunicorn markdown\n")
file.write("RUN pip install flask praisonai==2.5.0 gunicorn markdown\n")
file.write("EXPOSE 8080\n")
file.write('CMD ["gunicorn", "-b", "0.0.0.0:8080", "api:app"]\n')
@@ -0,0 +1,84 @@
"""
PraisonAI Persistence Layer
Provides database integrations for:
- Conversation history persistence (ConversationStore)
- Knowledge storage and retrieval (KnowledgeStore)
- Session/state management (StateStore)
All integrations use lazy loading to avoid importing unused dependencies.
Supported Backends (22 total):
- ConversationStore (6): postgres, mysql, sqlite, singlestore, supabase, surrealdb
- KnowledgeStore (10): qdrant, pinecone, chroma, weaviate, lancedb, milvus, pgvector, redis, cassandra, clickhouse
- StateStore (6): redis, dynamodb, firestore, mongodb, upstash, memory
"""
from typing import TYPE_CHECKING
__all__ = [
# Base interfaces
"ConversationStore",
"ConversationSession",
"ConversationMessage",
"KnowledgeStore",
"KnowledgeDocument",
"StateStore",
# Orchestrator
"PersistenceOrchestrator",
# Factory functions
"create_conversation_store",
"create_knowledge_store",
"create_state_store",
"create_stores_from_config",
# Config
"PersistenceConfig",
# Agent hooks
"wrap_agent_with_persistence",
"PersistentAgent",
"create_persistent_session",
]
# Lazy imports to avoid loading dependencies until needed
def __getattr__(name: str):
if name in ("ConversationStore", "ConversationSession", "ConversationMessage"):
from .conversation.base import ConversationStore, ConversationSession, ConversationMessage
return locals()[name]
if name in ("KnowledgeStore", "KnowledgeDocument"):
from .knowledge.base import KnowledgeStore, KnowledgeDocument
return locals()[name]
if name == "StateStore":
from .state.base import StateStore
return StateStore
if name == "PersistenceOrchestrator":
from .orchestrator import PersistenceOrchestrator
return PersistenceOrchestrator
if name == "PersistenceConfig":
from .config import PersistenceConfig
return PersistenceConfig
if name == "create_conversation_store":
from .factory import create_conversation_store
return create_conversation_store
if name == "create_knowledge_store":
from .factory import create_knowledge_store
return create_knowledge_store
if name == "create_state_store":
from .factory import create_state_store
return create_state_store
if name == "create_stores_from_config":
from .factory import create_stores_from_config
return create_stores_from_config
if name in ("wrap_agent_with_persistence", "PersistentAgent", "create_persistent_session"):
from .hooks.agent_hooks import wrap_agent_with_persistence, PersistentAgent, create_persistent_session
return locals()[name]
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -0,0 +1,238 @@
"""
Configuration handling for PraisonAI persistence layer.
Supports configuration via:
- Environment variables
- Config file (YAML/JSON)
- CLI arguments
- Direct Python API
"""
import os
import logging
from dataclasses import dataclass, field
from typing import Any, Dict, Optional
logger = logging.getLogger(__name__)
# Environment variable mappings
ENV_VARS = {
# Conversation stores
"conversation_store": "PRAISONAI_CONVERSATION_STORE",
"conversation_url": "PRAISONAI_CONVERSATION_URL",
# Knowledge stores
"knowledge_store": "PRAISONAI_KNOWLEDGE_STORE",
"knowledge_url": "PRAISONAI_KNOWLEDGE_URL",
# State stores
"state_store": "PRAISONAI_STATE_STORE",
"state_url": "PRAISONAI_STATE_URL",
# Common
"session_id": "PRAISONAI_SESSION_ID",
"user_id": "PRAISONAI_USER_ID",
# PostgreSQL
"postgres_url": "POSTGRES_URL",
"postgres_host": "POSTGRES_HOST",
"postgres_port": "POSTGRES_PORT",
"postgres_database": "POSTGRES_DATABASE",
"postgres_user": "POSTGRES_USER",
"postgres_password": "POSTGRES_PASSWORD",
# Qdrant
"qdrant_url": "QDRANT_URL",
"qdrant_host": "QDRANT_HOST",
"qdrant_port": "QDRANT_PORT",
"qdrant_api_key": "QDRANT_API_KEY",
# Redis
"redis_url": "REDIS_URL",
"redis_host": "REDIS_HOST",
"redis_port": "REDIS_PORT",
"redis_password": "REDIS_PASSWORD",
# Pinecone
"pinecone_api_key": "PINECONE_API_KEY",
"pinecone_environment": "PINECONE_ENVIRONMENT",
"pinecone_index": "PINECONE_INDEX",
# ChromaDB
"chroma_path": "CHROMA_DB_PATH",
"chroma_host": "CHROMA_HOST",
"chroma_port": "CHROMA_PORT",
# Weaviate
"weaviate_url": "WEAVIATE_URL",
"weaviate_api_key": "WEAVIATE_API_KEY",
# Milvus
"milvus_host": "MILVUS_HOST",
"milvus_port": "MILVUS_PORT",
"milvus_token": "MILVUS_TOKEN",
# MongoDB
"mongodb_url": "MONGODB_URL",
"mongodb_database": "MONGODB_DATABASE",
# DynamoDB
"aws_region": "AWS_REGION",
"dynamodb_table": "DYNAMODB_TABLE",
# Firestore
"google_credentials": "GOOGLE_APPLICATION_CREDENTIALS",
"firestore_project": "FIRESTORE_PROJECT",
# Upstash
"upstash_redis_url": "UPSTASH_REDIS_URL",
"upstash_redis_token": "UPSTASH_REDIS_TOKEN",
"upstash_vector_url": "UPSTASH_VECTOR_URL",
"upstash_vector_token": "UPSTASH_VECTOR_TOKEN",
# Supabase
"supabase_url": "SUPABASE_URL",
"supabase_key": "SUPABASE_KEY",
# SurrealDB
"surrealdb_url": "SURREALDB_URL",
"surrealdb_namespace": "SURREALDB_NS",
"surrealdb_database": "SURREALDB_DB",
# MySQL
"mysql_url": "MYSQL_URL",
"mysql_host": "MYSQL_HOST",
"mysql_port": "MYSQL_PORT",
"mysql_database": "MYSQL_DATABASE",
"mysql_user": "MYSQL_USER",
"mysql_password": "MYSQL_PASSWORD",
# SingleStore
"singlestore_url": "SINGLESTORE_URL",
# ClickHouse
"clickhouse_host": "CLICKHOUSE_HOST",
"clickhouse_port": "CLICKHOUSE_PORT",
"clickhouse_user": "CLICKHOUSE_USER",
"clickhouse_password": "CLICKHOUSE_PASSWORD",
# Cassandra
"cassandra_hosts": "CASSANDRA_HOSTS",
"cassandra_keyspace": "CASSANDRA_KEYSPACE",
# LanceDB
"lancedb_path": "LANCEDB_PATH",
"lancedb_uri": "LANCEDB_URI",
}
# Supported backends
CONVERSATION_BACKENDS = ["postgres", "mysql", "sqlite", "singlestore", "supabase", "surrealdb"]
KNOWLEDGE_BACKENDS = ["qdrant", "pinecone", "chroma", "weaviate", "lancedb", "milvus", "pgvector", "redis", "cassandra", "clickhouse"]
STATE_BACKENDS = ["redis", "dynamodb", "firestore", "mongodb", "upstash", "memory"]
@dataclass
class PersistenceConfig:
"""
Configuration for PraisonAI persistence layer.
Example:
config = PersistenceConfig(
conversation_store="postgres",
conversation_url="postgresql://localhost:5432/praisonai",
knowledge_store="qdrant",
knowledge_url="http://localhost:6333",
state_store="redis",
state_url="redis://localhost:6379",
)
"""
# Store backends
conversation_store: Optional[str] = None
conversation_url: Optional[str] = None
conversation_options: Dict[str, Any] = field(default_factory=dict)
knowledge_store: Optional[str] = None
knowledge_url: Optional[str] = None
knowledge_options: Dict[str, Any] = field(default_factory=dict)
state_store: Optional[str] = None
state_url: Optional[str] = None
state_options: Dict[str, Any] = field(default_factory=dict)
# Session/user context
session_id: Optional[str] = None
user_id: Optional[str] = None
# General options
auto_create_tables: bool = True
lazy_init: bool = True
@classmethod
def from_env(cls) -> "PersistenceConfig":
"""Create config from environment variables."""
return cls(
conversation_store=os.getenv(ENV_VARS["conversation_store"]),
conversation_url=os.getenv(ENV_VARS["conversation_url"]) or os.getenv(ENV_VARS["postgres_url"]),
knowledge_store=os.getenv(ENV_VARS["knowledge_store"]),
knowledge_url=os.getenv(ENV_VARS["knowledge_url"]) or os.getenv(ENV_VARS["qdrant_url"]),
state_store=os.getenv(ENV_VARS["state_store"]),
state_url=os.getenv(ENV_VARS["state_url"]) or os.getenv(ENV_VARS["redis_url"]),
session_id=os.getenv(ENV_VARS["session_id"]),
user_id=os.getenv(ENV_VARS["user_id"]),
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "PersistenceConfig":
"""Create config from dictionary."""
return cls(**{k: v for k, v in data.items() if hasattr(cls, k)})
@classmethod
def from_yaml(cls, path: str) -> "PersistenceConfig":
"""Load config from YAML file."""
try:
import yaml
except ImportError:
raise ImportError("PyYAML required for YAML config. Install with: pip install pyyaml")
with open(path) as f:
data = yaml.safe_load(f)
# Handle nested persistence config
if "persistence" in data:
data = data["persistence"]
return cls.from_dict(data)
def to_dict(self) -> Dict[str, Any]:
"""Convert config to dictionary."""
return {
"conversation_store": self.conversation_store,
"conversation_url": self.conversation_url,
"conversation_options": self.conversation_options,
"knowledge_store": self.knowledge_store,
"knowledge_url": self.knowledge_url,
"knowledge_options": self.knowledge_options,
"state_store": self.state_store,
"state_url": self.state_url,
"state_options": self.state_options,
"session_id": self.session_id,
"user_id": self.user_id,
"auto_create_tables": self.auto_create_tables,
"lazy_init": self.lazy_init,
}
def validate(self) -> bool:
"""Validate configuration."""
valid = True
if self.conversation_store and self.conversation_store not in CONVERSATION_BACKENDS:
logger.error(f"Invalid conversation_store: {self.conversation_store}. "
f"Supported: {CONVERSATION_BACKENDS}")
valid = False
if self.knowledge_store and self.knowledge_store not in KNOWLEDGE_BACKENDS:
logger.error(f"Invalid knowledge_store: {self.knowledge_store}. "
f"Supported: {KNOWLEDGE_BACKENDS}")
valid = False
if self.state_store and self.state_store not in STATE_BACKENDS:
logger.error(f"Invalid state_store: {self.state_store}. "
f"Supported: {STATE_BACKENDS}")
valid = False
return valid
def get_env_var(key: str, default: Optional[str] = None) -> Optional[str]:
"""Get environment variable by config key name."""
env_name = ENV_VARS.get(key, key.upper())
return os.getenv(env_name, default)
def list_available_backends() -> Dict[str, list]:
"""List all available backends by store type."""
return {
"conversation": CONVERSATION_BACKENDS,
"knowledge": KNOWLEDGE_BACKENDS,
"state": STATE_BACKENDS,
}
@@ -0,0 +1,25 @@
"""
ConversationStore implementations for session and message persistence.
Supported backends:
- PostgreSQL (postgres)
- MySQL (mysql)
- SQLite (sqlite)
- SingleStore (singlestore)
- Supabase (supabase)
- SurrealDB (surrealdb)
"""
from typing import TYPE_CHECKING
__all__ = [
"ConversationStore",
"ConversationSession",
"ConversationMessage",
]
def __getattr__(name: str):
if name in ("ConversationStore", "ConversationSession", "ConversationMessage"):
from .base import ConversationStore, ConversationSession, ConversationMessage
return locals()[name]
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -0,0 +1,427 @@
"""
Async MySQL implementation of ConversationStore.
Requires: aiomysql
Install: pip install aiomysql
"""
import asyncio
import json
import logging
import time
from typing import List, Optional
from .base import ConversationStore, ConversationSession, ConversationMessage
logger = logging.getLogger(__name__)
class AsyncMySQLConversationStore(ConversationStore):
"""
Async MySQL conversation store using aiomysql.
Provides high-performance async database operations.
Example:
store = AsyncMySQLConversationStore(
host="localhost",
user="root",
password="password",
database="praisonai"
)
await store.init()
"""
def __init__(
self,
url: Optional[str] = None,
host: str = "localhost",
port: int = 3306,
database: str = "praisonai",
user: str = "root",
password: str = "",
table_prefix: str = "praisonai_",
pool_size: int = 10,
):
"""
Initialize async MySQL store.
Args:
url: MySQL connection URL (takes precedence)
host: Database host
port: Database port
database: Database name
user: Database user
password: Database password
table_prefix: Prefix for table names
pool_size: Connection pool size
"""
self.url = url
self.host = host
self.port = port
self.database = database
self.user = user
self.password = password
self.table_prefix = table_prefix
self.pool_size = pool_size
self._pool = None
self._initialized = False
if url:
self._parse_url(url)
def _parse_url(self, url: str):
"""Parse MySQL URL into components."""
from urllib.parse import urlparse
parsed = urlparse(url)
self.host = parsed.hostname or "localhost"
self.port = parsed.port or 3306
self.database = parsed.path.lstrip('/') or "praisonai"
self.user = parsed.username or "root"
self.password = parsed.password or ""
async def init(self):
"""Initialize connection pool and create tables."""
if self._initialized:
return
try:
import aiomysql
except ImportError:
raise ImportError(
"aiomysql is required for async MySQL support. "
"Install with: pip install aiomysql"
)
self._pool = await aiomysql.create_pool(
host=self.host,
port=self.port,
user=self.user,
password=self.password,
db=self.database,
minsize=1,
maxsize=self.pool_size,
autocommit=True
)
await self._create_tables()
self._initialized = True
async def _create_tables(self):
"""Create required tables if they don't exist."""
sessions_table = f"{self.table_prefix}sessions"
messages_table = f"{self.table_prefix}messages"
async with self._pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(f"""
CREATE TABLE IF NOT EXISTS {sessions_table} (
session_id VARCHAR(255) PRIMARY KEY,
user_id VARCHAR(255),
agent_id VARCHAR(255),
name VARCHAR(255),
metadata JSON,
created_at DOUBLE,
updated_at DOUBLE,
INDEX idx_user_id (user_id),
INDEX idx_agent_id (agent_id)
)
""")
await cur.execute(f"""
CREATE TABLE IF NOT EXISTS {messages_table} (
id VARCHAR(255) PRIMARY KEY,
session_id VARCHAR(255),
role VARCHAR(50),
content TEXT,
metadata JSON,
created_at DOUBLE,
INDEX idx_session_id (session_id),
FOREIGN KEY (session_id) REFERENCES {sessions_table}(session_id) ON DELETE CASCADE
)
""")
async def async_create_session(self, session: ConversationSession) -> ConversationSession:
"""Create a new session asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
async with self._pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(f"""
INSERT INTO {table} (session_id, user_id, agent_id, name, metadata, created_at, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, %s)
""", (session.session_id, session.user_id, session.agent_id, session.name,
json.dumps(session.metadata) if session.metadata else None,
session.created_at, session.updated_at))
return session
def create_session(self, session: ConversationSession) -> ConversationSession:
"""Sync wrapper for create_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_create_session(session)
)
async def async_get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Get a session by ID asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
async with self._pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(f"""
SELECT session_id, user_id, agent_id, name, metadata, created_at, updated_at
FROM {table} WHERE session_id = %s
""", (session_id,))
row = await cur.fetchone()
if row:
return ConversationSession(
session_id=row[0],
user_id=row[1],
agent_id=row[2],
name=row[3],
metadata=json.loads(row[4]) if row[4] else None,
created_at=row[5],
updated_at=row[6]
)
return None
def get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Sync wrapper for get_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_get_session(session_id)
)
async def async_update_session(self, session: ConversationSession) -> ConversationSession:
"""Update an existing session asynchronously."""
if not self._initialized:
await self.init()
session.updated_at = time.time()
table = f"{self.table_prefix}sessions"
async with self._pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(f"""
UPDATE {table}
SET name = %s, metadata = %s, updated_at = %s
WHERE session_id = %s
""", (session.name, json.dumps(session.metadata) if session.metadata else None,
session.updated_at, session.session_id))
return session
def update_session(self, session: ConversationSession) -> ConversationSession:
"""Sync wrapper for update_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_update_session(session)
)
async def async_delete_session(self, session_id: str) -> bool:
"""Delete a session asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
async with self._pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(f"""
DELETE FROM {table} WHERE session_id = %s
""", (session_id,))
return cur.rowcount > 0
def delete_session(self, session_id: str) -> bool:
"""Sync wrapper for delete_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_delete_session(session_id)
)
async def async_list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""List sessions asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
conditions = []
params = []
if user_id:
conditions.append("user_id = %s")
params.append(user_id)
if agent_id:
conditions.append("agent_id = %s")
params.append(agent_id)
where_clause = f"WHERE {' AND '.join(conditions)}" if conditions else ""
params.extend([limit, offset])
async with self._pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(f"""
SELECT session_id, user_id, agent_id, name, metadata, created_at, updated_at
FROM {table} {where_clause}
ORDER BY updated_at DESC
LIMIT %s OFFSET %s
""", params)
rows = await cur.fetchall()
return [
ConversationSession(
session_id=row[0],
user_id=row[1],
agent_id=row[2],
name=row[3],
metadata=json.loads(row[4]) if row[4] else None,
created_at=row[5],
updated_at=row[6]
)
for row in rows
]
def list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""Sync wrapper for list_sessions."""
return asyncio.get_event_loop().run_until_complete(
self.async_list_sessions(user_id, agent_id, limit, offset)
)
async def async_add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Add a message asynchronously."""
if not self._initialized:
await self.init()
message.session_id = session_id
table = f"{self.table_prefix}messages"
async with self._pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(f"""
INSERT INTO {table} (id, session_id, role, content, metadata, created_at)
VALUES (%s, %s, %s, %s, %s, %s)
""", (message.id, session_id, message.role, message.content,
json.dumps(message.metadata) if message.metadata else None,
message.created_at))
return message
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Sync wrapper for add_message."""
return asyncio.get_event_loop().run_until_complete(
self.async_add_message(session_id, message)
)
async def async_get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Get messages asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}messages"
conditions = ["session_id = %s"]
params = [session_id]
if before:
conditions.append("created_at < %s")
params.append(before)
if after:
conditions.append("created_at > %s")
params.append(after)
where_clause = f"WHERE {' AND '.join(conditions)}"
limit_clause = "LIMIT %s" if limit else ""
if limit:
params.append(limit)
async with self._pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(f"""
SELECT id, session_id, role, content, metadata, created_at
FROM {table} {where_clause}
ORDER BY created_at ASC {limit_clause}
""", params)
rows = await cur.fetchall()
return [
ConversationMessage(
id=row[0],
session_id=row[1],
role=row[2],
content=row[3],
metadata=json.loads(row[4]) if row[4] else None,
created_at=row[5]
)
for row in rows
]
def get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Sync wrapper for get_messages."""
return asyncio.get_event_loop().run_until_complete(
self.async_get_messages(session_id, limit, before, after)
)
async def async_delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Delete messages asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}messages"
async with self._pool.acquire() as conn:
async with conn.cursor() as cur:
if message_ids is None:
await cur.execute(f"""
DELETE FROM {table} WHERE session_id = %s
""", (session_id,))
else:
placeholders = ','.join(['%s'] * len(message_ids))
await cur.execute(f"""
DELETE FROM {table} WHERE session_id = %s AND id IN ({placeholders})
""", [session_id] + message_ids)
return cur.rowcount
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Sync wrapper for delete_messages."""
return asyncio.get_event_loop().run_until_complete(
self.async_delete_messages(session_id, message_ids)
)
async def async_close(self) -> None:
"""Close the connection pool."""
if self._pool:
self._pool.close()
await self._pool.wait_closed()
self._pool = None
self._initialized = False
def close(self) -> None:
"""Sync wrapper for close."""
if self._pool:
asyncio.get_event_loop().run_until_complete(self.async_close())
@@ -0,0 +1,410 @@
"""
Async PostgreSQL implementation of ConversationStore.
Requires: asyncpg
Install: pip install asyncpg
"""
import asyncio
import json
import logging
import time
from typing import List, Optional
from .base import ConversationStore, ConversationSession, ConversationMessage
logger = logging.getLogger(__name__)
class AsyncPostgresConversationStore(ConversationStore):
"""
Async PostgreSQL conversation store using asyncpg.
Provides high-performance async database operations.
Example:
store = AsyncPostgresConversationStore(
url="postgresql://user:pass@localhost:5432/praisonai"
)
await store.init()
"""
def __init__(
self,
url: Optional[str] = None,
host: str = "localhost",
port: int = 5432,
database: str = "praisonai",
user: str = "postgres",
password: str = "",
table_prefix: str = "praisonai_",
pool_size: int = 10,
):
"""
Initialize async PostgreSQL store.
Args:
url: PostgreSQL connection URL (takes precedence)
host: Database host
port: Database port
database: Database name
user: Database user
password: Database password
table_prefix: Prefix for table names
pool_size: Connection pool size
"""
self.url = url
self.host = host
self.port = port
self.database = database
self.user = user
self.password = password
self.table_prefix = table_prefix
self.pool_size = pool_size
self._pool = None
self._initialized = False
async def init(self):
"""Initialize connection pool and create tables."""
if self._initialized:
return
try:
import asyncpg
except ImportError:
raise ImportError(
"asyncpg is required for async PostgreSQL support. "
"Install with: pip install asyncpg"
)
if self.url:
self._pool = await asyncpg.create_pool(self.url, min_size=1, max_size=self.pool_size)
else:
self._pool = await asyncpg.create_pool(
host=self.host,
port=self.port,
database=self.database,
user=self.user,
password=self.password,
min_size=1,
max_size=self.pool_size
)
await self._create_tables()
self._initialized = True
async def _create_tables(self):
"""Create required tables if they don't exist."""
sessions_table = f"{self.table_prefix}sessions"
messages_table = f"{self.table_prefix}messages"
async with self._pool.acquire() as conn:
await conn.execute(f"""
CREATE TABLE IF NOT EXISTS {sessions_table} (
session_id VARCHAR(255) PRIMARY KEY,
user_id VARCHAR(255),
agent_id VARCHAR(255),
name VARCHAR(255),
metadata JSONB,
created_at DOUBLE PRECISION,
updated_at DOUBLE PRECISION
)
""")
await conn.execute(f"""
CREATE TABLE IF NOT EXISTS {messages_table} (
id VARCHAR(255) PRIMARY KEY,
session_id VARCHAR(255) REFERENCES {sessions_table}(session_id) ON DELETE CASCADE,
role VARCHAR(50),
content TEXT,
metadata JSONB,
created_at DOUBLE PRECISION
)
""")
await conn.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{messages_table}_session
ON {messages_table}(session_id)
""")
async def async_create_session(self, session: ConversationSession) -> ConversationSession:
"""Create a new session asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
async with self._pool.acquire() as conn:
await conn.execute(f"""
INSERT INTO {table} (session_id, user_id, agent_id, name, metadata, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7)
""", session.session_id, session.user_id, session.agent_id, session.name,
json.dumps(session.metadata) if session.metadata else None,
session.created_at, session.updated_at)
return session
def create_session(self, session: ConversationSession) -> ConversationSession:
"""Sync wrapper for create_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_create_session(session)
)
async def async_get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Get a session by ID asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
async with self._pool.acquire() as conn:
row = await conn.fetchrow(f"""
SELECT * FROM {table} WHERE session_id = $1
""", session_id)
if row:
return ConversationSession(
session_id=row['session_id'],
user_id=row['user_id'],
agent_id=row['agent_id'],
name=row['name'],
metadata=json.loads(row['metadata']) if row['metadata'] else None,
created_at=row['created_at'],
updated_at=row['updated_at']
)
return None
def get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Sync wrapper for get_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_get_session(session_id)
)
async def async_update_session(self, session: ConversationSession) -> ConversationSession:
"""Update an existing session asynchronously."""
if not self._initialized:
await self.init()
session.updated_at = time.time()
table = f"{self.table_prefix}sessions"
async with self._pool.acquire() as conn:
await conn.execute(f"""
UPDATE {table}
SET name = $2, metadata = $3, updated_at = $4
WHERE session_id = $1
""", session.session_id, session.name,
json.dumps(session.metadata) if session.metadata else None,
session.updated_at)
return session
def update_session(self, session: ConversationSession) -> ConversationSession:
"""Sync wrapper for update_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_update_session(session)
)
async def async_delete_session(self, session_id: str) -> bool:
"""Delete a session asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
async with self._pool.acquire() as conn:
result = await conn.execute(f"""
DELETE FROM {table} WHERE session_id = $1
""", session_id)
return "DELETE 1" in result
def delete_session(self, session_id: str) -> bool:
"""Sync wrapper for delete_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_delete_session(session_id)
)
async def async_list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""List sessions asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
conditions = []
params = []
param_idx = 1
if user_id:
conditions.append(f"user_id = ${param_idx}")
params.append(user_id)
param_idx += 1
if agent_id:
conditions.append(f"agent_id = ${param_idx}")
params.append(agent_id)
param_idx += 1
where_clause = f"WHERE {' AND '.join(conditions)}" if conditions else ""
params.extend([limit, offset])
async with self._pool.acquire() as conn:
rows = await conn.fetch(f"""
SELECT * FROM {table} {where_clause}
ORDER BY updated_at DESC
LIMIT ${param_idx} OFFSET ${param_idx + 1}
""", *params)
return [
ConversationSession(
session_id=row['session_id'],
user_id=row['user_id'],
agent_id=row['agent_id'],
name=row['name'],
metadata=json.loads(row['metadata']) if row['metadata'] else None,
created_at=row['created_at'],
updated_at=row['updated_at']
)
for row in rows
]
def list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""Sync wrapper for list_sessions."""
return asyncio.get_event_loop().run_until_complete(
self.async_list_sessions(user_id, agent_id, limit, offset)
)
async def async_add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Add a message asynchronously."""
if not self._initialized:
await self.init()
message.session_id = session_id
table = f"{self.table_prefix}messages"
async with self._pool.acquire() as conn:
await conn.execute(f"""
INSERT INTO {table} (id, session_id, role, content, metadata, created_at)
VALUES ($1, $2, $3, $4, $5, $6)
""", message.id, session_id, message.role, message.content,
json.dumps(message.metadata) if message.metadata else None,
message.created_at)
return message
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Sync wrapper for add_message."""
return asyncio.get_event_loop().run_until_complete(
self.async_add_message(session_id, message)
)
async def async_get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Get messages asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}messages"
conditions = ["session_id = $1"]
params = [session_id]
param_idx = 2
if before:
conditions.append(f"created_at < ${param_idx}")
params.append(before)
param_idx += 1
if after:
conditions.append(f"created_at > ${param_idx}")
params.append(after)
param_idx += 1
where_clause = f"WHERE {' AND '.join(conditions)}"
limit_clause = f"LIMIT ${param_idx}" if limit else ""
if limit:
params.append(limit)
async with self._pool.acquire() as conn:
rows = await conn.fetch(f"""
SELECT * FROM {table} {where_clause}
ORDER BY created_at ASC {limit_clause}
""", *params)
return [
ConversationMessage(
id=row['id'],
session_id=row['session_id'],
role=row['role'],
content=row['content'],
metadata=json.loads(row['metadata']) if row['metadata'] else None,
created_at=row['created_at']
)
for row in rows
]
def get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Sync wrapper for get_messages."""
return asyncio.get_event_loop().run_until_complete(
self.async_get_messages(session_id, limit, before, after)
)
async def async_delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Delete messages asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}messages"
async with self._pool.acquire() as conn:
if message_ids is None:
result = await conn.execute(f"""
DELETE FROM {table} WHERE session_id = $1
""", session_id)
else:
result = await conn.execute(f"""
DELETE FROM {table} WHERE session_id = $1 AND id = ANY($2)
""", session_id, message_ids)
# Parse count from result like "DELETE 5"
try:
return int(result.split()[-1])
except (ValueError, IndexError):
return 0
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Sync wrapper for delete_messages."""
return asyncio.get_event_loop().run_until_complete(
self.async_delete_messages(session_id, message_ids)
)
async def async_close(self) -> None:
"""Close the connection pool."""
if self._pool:
await self._pool.close()
self._pool = None
self._initialized = False
def close(self) -> None:
"""Sync wrapper for close."""
if self._pool:
asyncio.get_event_loop().run_until_complete(self.async_close())
@@ -0,0 +1,371 @@
"""
Async SQLite implementation of ConversationStore.
Requires: aiosqlite
Install: pip install aiosqlite
"""
import asyncio
import json
import logging
import time
from typing import List, Optional
from .base import ConversationStore, ConversationSession, ConversationMessage
logger = logging.getLogger(__name__)
class AsyncSQLiteConversationStore(ConversationStore):
"""
Async SQLite conversation store using aiosqlite.
Provides async database operations for SQLite.
Example:
store = AsyncSQLiteConversationStore(path="./conversations.db")
await store.init()
"""
def __init__(
self,
path: str = "praisonai_conversations.db",
table_prefix: str = "praisonai_",
):
"""
Initialize async SQLite store.
Args:
path: Path to SQLite database file
table_prefix: Prefix for table names
"""
self.path = path
self.table_prefix = table_prefix
self._conn = None
self._initialized = False
async def init(self):
"""Initialize connection and create tables."""
if self._initialized:
return
try:
import aiosqlite
except ImportError:
raise ImportError(
"aiosqlite is required for async SQLite support. "
"Install with: pip install aiosqlite"
)
self._aiosqlite = aiosqlite
self._conn = await aiosqlite.connect(self.path)
self._conn.row_factory = aiosqlite.Row
await self._create_tables()
self._initialized = True
async def _create_tables(self):
"""Create required tables if they don't exist."""
sessions_table = f"{self.table_prefix}sessions"
messages_table = f"{self.table_prefix}messages"
await self._conn.execute(f"""
CREATE TABLE IF NOT EXISTS {sessions_table} (
session_id TEXT PRIMARY KEY,
user_id TEXT,
agent_id TEXT,
name TEXT,
metadata TEXT,
created_at REAL,
updated_at REAL
)
""")
await self._conn.execute(f"""
CREATE TABLE IF NOT EXISTS {messages_table} (
id TEXT PRIMARY KEY,
session_id TEXT REFERENCES {sessions_table}(session_id) ON DELETE CASCADE,
role TEXT,
content TEXT,
metadata TEXT,
created_at REAL
)
""")
await self._conn.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{messages_table}_session
ON {messages_table}(session_id)
""")
await self._conn.commit()
async def async_create_session(self, session: ConversationSession) -> ConversationSession:
"""Create a new session asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
await self._conn.execute(f"""
INSERT INTO {table} (session_id, user_id, agent_id, name, metadata, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?)
""", (session.session_id, session.user_id, session.agent_id, session.name,
json.dumps(session.metadata) if session.metadata else None,
session.created_at, session.updated_at))
await self._conn.commit()
return session
def create_session(self, session: ConversationSession) -> ConversationSession:
"""Sync wrapper for create_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_create_session(session)
)
async def async_get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Get a session by ID asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
async with self._conn.execute(f"""
SELECT * FROM {table} WHERE session_id = ?
""", (session_id,)) as cursor:
row = await cursor.fetchone()
if row:
return ConversationSession(
session_id=row['session_id'],
user_id=row['user_id'],
agent_id=row['agent_id'],
name=row['name'],
metadata=json.loads(row['metadata']) if row['metadata'] else None,
created_at=row['created_at'],
updated_at=row['updated_at']
)
return None
def get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Sync wrapper for get_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_get_session(session_id)
)
async def async_update_session(self, session: ConversationSession) -> ConversationSession:
"""Update an existing session asynchronously."""
if not self._initialized:
await self.init()
session.updated_at = time.time()
table = f"{self.table_prefix}sessions"
await self._conn.execute(f"""
UPDATE {table}
SET name = ?, metadata = ?, updated_at = ?
WHERE session_id = ?
""", (session.name, json.dumps(session.metadata) if session.metadata else None,
session.updated_at, session.session_id))
await self._conn.commit()
return session
def update_session(self, session: ConversationSession) -> ConversationSession:
"""Sync wrapper for update_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_update_session(session)
)
async def async_delete_session(self, session_id: str) -> bool:
"""Delete a session asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
cursor = await self._conn.execute(f"""
DELETE FROM {table} WHERE session_id = ?
""", (session_id,))
await self._conn.commit()
return cursor.rowcount > 0
def delete_session(self, session_id: str) -> bool:
"""Sync wrapper for delete_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_delete_session(session_id)
)
async def async_list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""List sessions asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
conditions = []
params = []
if user_id:
conditions.append("user_id = ?")
params.append(user_id)
if agent_id:
conditions.append("agent_id = ?")
params.append(agent_id)
where_clause = f"WHERE {' AND '.join(conditions)}" if conditions else ""
params.extend([limit, offset])
async with self._conn.execute(f"""
SELECT * FROM {table} {where_clause}
ORDER BY updated_at DESC
LIMIT ? OFFSET ?
""", params) as cursor:
rows = await cursor.fetchall()
return [
ConversationSession(
session_id=row['session_id'],
user_id=row['user_id'],
agent_id=row['agent_id'],
name=row['name'],
metadata=json.loads(row['metadata']) if row['metadata'] else None,
created_at=row['created_at'],
updated_at=row['updated_at']
)
for row in rows
]
def list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""Sync wrapper for list_sessions."""
return asyncio.get_event_loop().run_until_complete(
self.async_list_sessions(user_id, agent_id, limit, offset)
)
async def async_add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Add a message asynchronously."""
if not self._initialized:
await self.init()
message.session_id = session_id
table = f"{self.table_prefix}messages"
await self._conn.execute(f"""
INSERT INTO {table} (id, session_id, role, content, metadata, created_at)
VALUES (?, ?, ?, ?, ?, ?)
""", (message.id, session_id, message.role, message.content,
json.dumps(message.metadata) if message.metadata else None,
message.created_at))
await self._conn.commit()
return message
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Sync wrapper for add_message."""
return asyncio.get_event_loop().run_until_complete(
self.async_add_message(session_id, message)
)
async def async_get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Get messages asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}messages"
conditions = ["session_id = ?"]
params = [session_id]
if before:
conditions.append("created_at < ?")
params.append(before)
if after:
conditions.append("created_at > ?")
params.append(after)
where_clause = f"WHERE {' AND '.join(conditions)}"
limit_clause = f"LIMIT ?" if limit else ""
if limit:
params.append(limit)
async with self._conn.execute(f"""
SELECT * FROM {table} {where_clause}
ORDER BY created_at ASC {limit_clause}
""", params) as cursor:
rows = await cursor.fetchall()
return [
ConversationMessage(
id=row['id'],
session_id=row['session_id'],
role=row['role'],
content=row['content'],
metadata=json.loads(row['metadata']) if row['metadata'] else None,
created_at=row['created_at']
)
for row in rows
]
def get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Sync wrapper for get_messages."""
return asyncio.get_event_loop().run_until_complete(
self.async_get_messages(session_id, limit, before, after)
)
async def async_delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Delete messages asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}messages"
if message_ids is None:
cursor = await self._conn.execute(f"""
DELETE FROM {table} WHERE session_id = ?
""", (session_id,))
else:
placeholders = ','.join(['?' for _ in message_ids])
cursor = await self._conn.execute(f"""
DELETE FROM {table} WHERE session_id = ? AND id IN ({placeholders})
""", [session_id] + message_ids)
await self._conn.commit()
return cursor.rowcount
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Sync wrapper for delete_messages."""
return asyncio.get_event_loop().run_until_complete(
self.async_delete_messages(session_id, message_ids)
)
async def async_close(self) -> None:
"""Close the connection."""
if self._conn:
await self._conn.close()
self._conn = None
self._initialized = False
def close(self) -> None:
"""Sync wrapper for close."""
if self._conn:
asyncio.get_event_loop().run_until_complete(self.async_close())
@@ -0,0 +1,151 @@
"""
Base interfaces for ConversationStore.
ConversationStore handles session and message persistence for conversation history.
"""
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional
import time
import uuid
@dataclass
class ConversationMessage:
"""A single message in a conversation."""
id: str = field(default_factory=lambda: str(uuid.uuid4()))
session_id: str = ""
role: str = "" # user, assistant, system, tool
content: str = ""
tool_calls: Optional[List[Dict[str, Any]]] = None
tool_call_id: Optional[str] = None
metadata: Optional[Dict[str, Any]] = None
created_at: float = field(default_factory=time.time)
def to_dict(self) -> Dict[str, Any]:
return {
"id": self.id,
"session_id": self.session_id,
"role": self.role,
"content": self.content,
"tool_calls": self.tool_calls,
"tool_call_id": self.tool_call_id,
"metadata": self.metadata,
"created_at": self.created_at,
}
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "ConversationMessage":
return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__})
@dataclass
class ConversationSession:
"""A conversation session containing messages."""
session_id: str = field(default_factory=lambda: str(uuid.uuid4()))
user_id: Optional[str] = None
agent_id: Optional[str] = None
name: Optional[str] = None
state: Optional[Dict[str, Any]] = None
metadata: Optional[Dict[str, Any]] = None
created_at: float = field(default_factory=time.time)
updated_at: float = field(default_factory=time.time)
def to_dict(self) -> Dict[str, Any]:
return {
"session_id": self.session_id,
"user_id": self.user_id,
"agent_id": self.agent_id,
"name": self.name,
"state": self.state,
"metadata": self.metadata,
"created_at": self.created_at,
"updated_at": self.updated_at,
}
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "ConversationSession":
return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__})
class ConversationStore(ABC):
"""
Abstract base class for conversation persistence.
Implementations handle session and message storage for different backends:
- PostgreSQL, MySQL, SQLite (relational)
- SingleStore, Supabase, SurrealDB (hybrid)
"""
@abstractmethod
def create_session(self, session: ConversationSession) -> ConversationSession:
"""Create a new session."""
raise NotImplementedError
@abstractmethod
def get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Get a session by ID."""
raise NotImplementedError
@abstractmethod
def update_session(self, session: ConversationSession) -> ConversationSession:
"""Update an existing session."""
raise NotImplementedError
@abstractmethod
def delete_session(self, session_id: str) -> bool:
"""Delete a session and all its messages."""
raise NotImplementedError
@abstractmethod
def list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""List sessions, optionally filtered by user or agent."""
raise NotImplementedError
@abstractmethod
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Add a message to a session."""
raise NotImplementedError
@abstractmethod
def get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Get messages from a session."""
raise NotImplementedError
@abstractmethod
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Delete messages. If message_ids is None, delete all messages in session."""
raise NotImplementedError
def upsert_session(self, session: ConversationSession) -> ConversationSession:
"""Create or update a session."""
existing = self.get_session(session.session_id)
if existing:
session.updated_at = time.time()
return self.update_session(session)
return self.create_session(session)
@abstractmethod
def close(self) -> None:
"""Close the store and release resources."""
raise NotImplementedError
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
return False
@@ -0,0 +1,250 @@
"""
JSON file-based implementation of ConversationStore.
Simple file-based storage for development and testing.
No external dependencies required.
"""
import json
import logging
import time
from pathlib import Path
from typing import Dict, List, Optional
import threading
from .base import ConversationStore, ConversationSession, ConversationMessage
logger = logging.getLogger(__name__)
class JSONConversationStore(ConversationStore):
"""
JSON file-based conversation store.
Stores sessions and messages in JSON files.
Suitable for development, testing, and small-scale deployments.
Example:
store = JSONConversationStore(path="./data/conversations")
"""
def __init__(
self,
path: str = "./praisonai_conversations",
pretty: bool = True,
):
"""
Initialize JSON conversation store.
Args:
path: Directory path for JSON files
pretty: Use pretty-printed JSON (default: True)
"""
self.path = Path(path)
self.pretty = pretty
self._lock = threading.Lock()
# Create directory if it doesn't exist
self.path.mkdir(parents=True, exist_ok=True)
# Sessions index file
self._index_file = self.path / "_sessions_index.json"
self._load_index()
def _load_index(self):
"""Load or create sessions index."""
if self._index_file.exists():
with open(self._index_file, 'r') as f:
self._index = json.load(f)
else:
self._index = {"sessions": {}}
self._save_index()
def _save_index(self):
"""Save sessions index."""
with open(self._index_file, 'w') as f:
if self.pretty:
json.dump(self._index, f, indent=2, default=str)
else:
json.dump(self._index, f, default=str)
def _session_file(self, session_id: str) -> Path:
"""Get path to session file."""
return self.path / f"{session_id}.json"
def _load_session_data(self, session_id: str) -> Optional[Dict]:
"""Load session data from file."""
file_path = self._session_file(session_id)
if file_path.exists():
with open(file_path, 'r') as f:
return json.load(f)
return None
def _save_session_data(self, session_id: str, data: Dict):
"""Save session data to file."""
file_path = self._session_file(session_id)
with open(file_path, 'w') as f:
if self.pretty:
json.dump(data, f, indent=2, default=str)
else:
json.dump(data, f, default=str)
def create_session(self, session: ConversationSession) -> ConversationSession:
"""Create a new session."""
with self._lock:
data = {
"session": session.to_dict(),
"messages": []
}
self._save_session_data(session.session_id, data)
# Update index
self._index["sessions"][session.session_id] = {
"user_id": session.user_id,
"agent_id": session.agent_id,
"name": session.name,
"created_at": session.created_at,
"updated_at": session.updated_at,
}
self._save_index()
return session
def get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Get a session by ID."""
data = self._load_session_data(session_id)
if data and "session" in data:
return ConversationSession.from_dict(data["session"])
return None
def update_session(self, session: ConversationSession) -> ConversationSession:
"""Update an existing session."""
with self._lock:
data = self._load_session_data(session.session_id)
if not data:
raise ValueError(f"Session {session.session_id} not found")
session.updated_at = time.time()
data["session"] = session.to_dict()
self._save_session_data(session.session_id, data)
# Update index
self._index["sessions"][session.session_id].update({
"name": session.name,
"updated_at": session.updated_at,
})
self._save_index()
return session
def delete_session(self, session_id: str) -> bool:
"""Delete a session and all its messages."""
with self._lock:
file_path = self._session_file(session_id)
if file_path.exists():
file_path.unlink()
# Update index
if session_id in self._index["sessions"]:
del self._index["sessions"][session_id]
self._save_index()
return True
return False
def list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""List sessions, optionally filtered by user or agent."""
sessions = []
for sid, info in self._index["sessions"].items():
if user_id and info.get("user_id") != user_id:
continue
if agent_id and info.get("agent_id") != agent_id:
continue
session = self.get_session(sid)
if session:
sessions.append(session)
# Sort by updated_at descending
sessions.sort(key=lambda s: s.updated_at, reverse=True)
return sessions[offset:offset + limit]
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Add a message to a session."""
with self._lock:
data = self._load_session_data(session_id)
if not data:
raise ValueError(f"Session {session_id} not found")
message.session_id = session_id
data["messages"].append(message.to_dict())
# Update session timestamp
data["session"]["updated_at"] = time.time()
self._save_session_data(session_id, data)
# Update index
if session_id in self._index["sessions"]:
self._index["sessions"][session_id]["updated_at"] = time.time()
self._save_index()
return message
def get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Get messages from a session."""
data = self._load_session_data(session_id)
if not data:
return []
messages = [ConversationMessage.from_dict(m) for m in data.get("messages", [])]
# Apply filters
if before:
messages = [m for m in messages if m.created_at < before]
if after:
messages = [m for m in messages if m.created_at > after]
# Sort by created_at
messages.sort(key=lambda m: m.created_at)
if limit:
messages = messages[-limit:]
return messages
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Delete messages. If message_ids is None, delete all messages in session."""
with self._lock:
data = self._load_session_data(session_id)
if not data:
return 0
if message_ids is None:
count = len(data["messages"])
data["messages"] = []
else:
original_count = len(data["messages"])
data["messages"] = [m for m in data["messages"] if m.get("id") not in message_ids]
count = original_count - len(data["messages"])
self._save_session_data(session_id, data)
return count
def close(self) -> None:
"""Close the store."""
pass # No resources to release
@@ -0,0 +1,387 @@
"""
MySQL implementation of ConversationStore.
Requires: mysql-connector-python or pymysql
Install: pip install mysql-connector-python
"""
import json
import logging
from typing import Any, List, Optional
from .base import ConversationStore, ConversationSession, ConversationMessage
logger = logging.getLogger(__name__)
class MySQLConversationStore(ConversationStore):
"""
MySQL-based conversation store.
Example:
store = MySQLConversationStore(
url="mysql://user:password@localhost:3306/praisonai"
)
"""
SCHEMA_VERSION = "1.0.0"
def __init__(
self,
url: Optional[str] = None,
host: str = "localhost",
port: int = 3306,
database: str = "praisonai",
user: str = "root",
password: str = "",
table_prefix: str = "praison_",
auto_create_tables: bool = True,
pool_size: int = 5,
):
"""
Initialize MySQL conversation store.
Args:
url: Full connection URL (overrides individual params)
host: Database host
port: Database port
database: Database name
user: Database user
password: Database password
table_prefix: Prefix for table names
auto_create_tables: Create tables if they don't exist
pool_size: Connection pool size
"""
try:
import mysql.connector
from mysql.connector import pooling
except ImportError:
raise ImportError(
"mysql-connector-python is required for MySQL support. "
"Install with: pip install mysql-connector-python"
)
self._mysql = mysql.connector
self.table_prefix = table_prefix
self.sessions_table = f"{table_prefix}sessions"
self.messages_table = f"{table_prefix}messages"
# Parse URL if provided
if url:
# Parse mysql://user:pass@host:port/database
from urllib.parse import urlparse
parsed = urlparse(url)
host = parsed.hostname or host
port = parsed.port or port
database = parsed.path.lstrip("/") or database
user = parsed.username or user
password = parsed.password or password
# Create connection pool
self._pool = pooling.MySQLConnectionPool(
pool_name="praison_mysql_pool",
pool_size=pool_size,
host=host,
port=port,
database=database,
user=user,
password=password,
)
if auto_create_tables:
self._create_tables()
def _get_conn(self):
"""Get a connection from the pool."""
return self._pool.get_connection()
def _create_tables(self) -> None:
"""Create tables if they don't exist."""
conn = self._get_conn()
try:
cur = conn.cursor()
# Sessions table
cur.execute(f"""
CREATE TABLE IF NOT EXISTS {self.sessions_table} (
session_id VARCHAR(255) PRIMARY KEY,
user_id VARCHAR(255),
agent_id VARCHAR(255),
name VARCHAR(255),
state JSON,
metadata JSON,
created_at DOUBLE,
updated_at DOUBLE,
INDEX idx_user (user_id),
INDEX idx_agent (agent_id),
INDEX idx_updated (updated_at DESC)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
""")
# Messages table
cur.execute(f"""
CREATE TABLE IF NOT EXISTS {self.messages_table} (
id VARCHAR(255) PRIMARY KEY,
session_id VARCHAR(255) NOT NULL,
role VARCHAR(50) NOT NULL,
content TEXT,
tool_calls JSON,
tool_call_id VARCHAR(255),
metadata JSON,
created_at DOUBLE,
INDEX idx_session (session_id),
INDEX idx_created (session_id, created_at),
FOREIGN KEY (session_id) REFERENCES {self.sessions_table}(session_id) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
""")
conn.commit()
logger.info(f"MySQL tables created: {self.sessions_table}, {self.messages_table}")
finally:
cur.close()
conn.close()
def create_session(self, session: ConversationSession) -> ConversationSession:
"""Create a new session."""
conn = self._get_conn()
try:
cur = conn.cursor()
cur.execute(f"""
INSERT INTO {self.sessions_table}
(session_id, user_id, agent_id, name, state, metadata, created_at, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
""", (
session.session_id,
session.user_id,
session.agent_id,
session.name,
json.dumps(session.state) if session.state else None,
json.dumps(session.metadata) if session.metadata else None,
session.created_at,
session.updated_at,
))
conn.commit()
return session
finally:
cur.close()
conn.close()
def get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Get a session by ID."""
conn = self._get_conn()
try:
cur = conn.cursor(dictionary=True)
cur.execute(f"""
SELECT * FROM {self.sessions_table} WHERE session_id = %s
""", (session_id,))
row = cur.fetchone()
if not row:
return None
return ConversationSession(
session_id=row["session_id"],
user_id=row["user_id"],
agent_id=row["agent_id"],
name=row["name"],
state=row["state"] if isinstance(row["state"], dict) else (json.loads(row["state"]) if row["state"] else None),
metadata=row["metadata"] if isinstance(row["metadata"], dict) else (json.loads(row["metadata"]) if row["metadata"] else None),
created_at=row["created_at"],
updated_at=row["updated_at"],
)
finally:
cur.close()
conn.close()
def update_session(self, session: ConversationSession) -> ConversationSession:
"""Update an existing session."""
conn = self._get_conn()
try:
cur = conn.cursor()
cur.execute(f"""
UPDATE {self.sessions_table}
SET user_id = %s, agent_id = %s, name = %s,
state = %s, metadata = %s, updated_at = %s
WHERE session_id = %s
""", (
session.user_id,
session.agent_id,
session.name,
json.dumps(session.state) if session.state else None,
json.dumps(session.metadata) if session.metadata else None,
session.updated_at,
session.session_id,
))
conn.commit()
return session
finally:
cur.close()
conn.close()
def delete_session(self, session_id: str) -> bool:
"""Delete a session and all its messages."""
conn = self._get_conn()
try:
cur = conn.cursor()
cur.execute(f"""
DELETE FROM {self.sessions_table} WHERE session_id = %s
""", (session_id,))
deleted = cur.rowcount > 0
conn.commit()
return deleted
finally:
cur.close()
conn.close()
def list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""List sessions."""
conn = self._get_conn()
try:
cur = conn.cursor(dictionary=True)
conditions = []
params: List[Any] = []
if user_id:
conditions.append("user_id = %s")
params.append(user_id)
if agent_id:
conditions.append("agent_id = %s")
params.append(agent_id)
where_clause = ""
if conditions:
where_clause = "WHERE " + " AND ".join(conditions)
params.extend([limit, offset])
cur.execute(f"""
SELECT * FROM {self.sessions_table}
{where_clause}
ORDER BY updated_at DESC
LIMIT %s OFFSET %s
""", params)
return [
ConversationSession(
session_id=row["session_id"],
user_id=row["user_id"],
agent_id=row["agent_id"],
name=row["name"],
state=row["state"] if isinstance(row["state"], dict) else (json.loads(row["state"]) if row["state"] else None),
metadata=row["metadata"] if isinstance(row["metadata"], dict) else (json.loads(row["metadata"]) if row["metadata"] else None),
created_at=row["created_at"],
updated_at=row["updated_at"],
)
for row in cur.fetchall()
]
finally:
cur.close()
conn.close()
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Add a message to a session."""
message.session_id = session_id
conn = self._get_conn()
try:
cur = conn.cursor()
cur.execute(f"""
INSERT INTO {self.messages_table}
(id, session_id, role, content, tool_calls, tool_call_id, metadata, created_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
""", (
message.id,
session_id,
message.role,
message.content,
json.dumps(message.tool_calls) if message.tool_calls else None,
message.tool_call_id,
json.dumps(message.metadata) if message.metadata else None,
message.created_at,
))
conn.commit()
return message
finally:
cur.close()
conn.close()
def get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Get messages from a session."""
conn = self._get_conn()
try:
cur = conn.cursor(dictionary=True)
conditions = ["session_id = %s"]
params: List[Any] = [session_id]
if before:
conditions.append("created_at < %s")
params.append(before)
if after:
conditions.append("created_at > %s")
params.append(after)
where_clause = "WHERE " + " AND ".join(conditions)
limit_clause = f"LIMIT {limit}" if limit else ""
cur.execute(f"""
SELECT * FROM {self.messages_table}
{where_clause}
ORDER BY created_at ASC
{limit_clause}
""", params)
return [
ConversationMessage(
id=row["id"],
session_id=row["session_id"],
role=row["role"],
content=row["content"],
tool_calls=row["tool_calls"] if isinstance(row["tool_calls"], list) else (json.loads(row["tool_calls"]) if row["tool_calls"] else None),
tool_call_id=row["tool_call_id"],
metadata=row["metadata"] if isinstance(row["metadata"], dict) else (json.loads(row["metadata"]) if row["metadata"] else None),
created_at=row["created_at"],
)
for row in cur.fetchall()
]
finally:
cur.close()
conn.close()
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Delete messages."""
conn = self._get_conn()
try:
cur = conn.cursor()
if message_ids:
placeholders = ",".join(["%s"] * len(message_ids))
cur.execute(f"""
DELETE FROM {self.messages_table}
WHERE session_id = %s AND id IN ({placeholders})
""", [session_id] + message_ids)
else:
cur.execute(f"""
DELETE FROM {self.messages_table} WHERE session_id = %s
""", (session_id,))
deleted = cur.rowcount
conn.commit()
return deleted
finally:
cur.close()
conn.close()
def close(self) -> None:
"""Close the store."""
pass # Pool handles cleanup
@@ -0,0 +1,401 @@
"""
PostgreSQL implementation of ConversationStore.
Requires: psycopg2-binary or psycopg2
Install: pip install psycopg2-binary
"""
import json
import logging
from typing import Any, Dict, List, Optional
from .base import ConversationStore, ConversationSession, ConversationMessage
logger = logging.getLogger(__name__)
class PostgresConversationStore(ConversationStore):
"""
PostgreSQL-based conversation store.
Connection URL format: postgresql://user:password@host:port/database
Example:
store = PostgresConversationStore(
url="postgresql://postgres:password@localhost:5432/praisonai"
)
"""
SCHEMA_VERSION = "1.0.0"
def __init__(
self,
url: Optional[str] = None,
host: str = "localhost",
port: int = 5432,
database: str = "praisonai",
user: str = "postgres",
password: str = "",
schema: str = "public",
table_prefix: str = "praison_",
auto_create_tables: bool = True,
pool_size: int = 5,
):
"""
Initialize PostgreSQL conversation store.
Args:
url: Full connection URL (overrides individual params)
host: Database host
port: Database port
database: Database name
user: Database user
password: Database password
schema: PostgreSQL schema
table_prefix: Prefix for table names
auto_create_tables: Create tables if they don't exist
pool_size: Connection pool size
"""
try:
import psycopg2
from psycopg2 import pool as pg_pool
from psycopg2.extras import RealDictCursor
except ImportError:
raise ImportError(
"psycopg2 is required for PostgreSQL support. "
"Install with: pip install psycopg2-binary"
)
self._psycopg2 = psycopg2
self._RealDictCursor = RealDictCursor
self.schema = schema
self.table_prefix = table_prefix
self.sessions_table = f"{schema}.{table_prefix}sessions"
self.messages_table = f"{schema}.{table_prefix}messages"
# Build connection params
if url:
self._pool = pg_pool.ThreadedConnectionPool(1, pool_size, url)
else:
self._pool = pg_pool.ThreadedConnectionPool(
1, pool_size,
host=host,
port=port,
database=database,
user=user,
password=password,
)
if auto_create_tables:
self._create_tables()
def _get_conn(self):
"""Get a connection from the pool."""
return self._pool.getconn()
def _put_conn(self, conn):
"""Return a connection to the pool."""
self._pool.putconn(conn)
def _create_tables(self) -> None:
"""Create tables if they don't exist."""
conn = self._get_conn()
try:
with conn.cursor() as cur:
# Create schema if not exists
cur.execute(f"CREATE SCHEMA IF NOT EXISTS {self.schema}")
# Sessions table
cur.execute(f"""
CREATE TABLE IF NOT EXISTS {self.sessions_table} (
session_id VARCHAR(255) PRIMARY KEY,
user_id VARCHAR(255),
agent_id VARCHAR(255),
name VARCHAR(255),
state JSONB,
metadata JSONB,
created_at DOUBLE PRECISION,
updated_at DOUBLE PRECISION
)
""")
# Sessions indexes
cur.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}sessions_user
ON {self.sessions_table}(user_id)
""")
cur.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}sessions_agent
ON {self.sessions_table}(agent_id)
""")
cur.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}sessions_updated
ON {self.sessions_table}(updated_at DESC)
""")
# Messages table
cur.execute(f"""
CREATE TABLE IF NOT EXISTS {self.messages_table} (
id VARCHAR(255) PRIMARY KEY,
session_id VARCHAR(255) NOT NULL
REFERENCES {self.sessions_table}(session_id) ON DELETE CASCADE,
role VARCHAR(50) NOT NULL,
content TEXT,
tool_calls JSONB,
tool_call_id VARCHAR(255),
metadata JSONB,
created_at DOUBLE PRECISION DEFAULT EXTRACT(EPOCH FROM NOW())
)
""")
# Messages indexes
cur.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}messages_session
ON {self.messages_table}(session_id)
""")
cur.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}messages_created
ON {self.messages_table}(session_id, created_at)
""")
conn.commit()
logger.info(f"PostgreSQL tables created: {self.sessions_table}, {self.messages_table}")
finally:
self._put_conn(conn)
def create_session(self, session: ConversationSession) -> ConversationSession:
"""Create a new session."""
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute(f"""
INSERT INTO {self.sessions_table}
(session_id, user_id, agent_id, name, state, metadata, created_at, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
RETURNING session_id
""", (
session.session_id,
session.user_id,
session.agent_id,
session.name,
json.dumps(session.state) if session.state else None,
json.dumps(session.metadata) if session.metadata else None,
session.created_at,
session.updated_at,
))
conn.commit()
return session
finally:
self._put_conn(conn)
def get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Get a session by ID."""
conn = self._get_conn()
try:
with conn.cursor(cursor_factory=self._RealDictCursor) as cur:
cur.execute(f"""
SELECT * FROM {self.sessions_table} WHERE session_id = %s
""", (session_id,))
row = cur.fetchone()
if not row:
return None
return ConversationSession(
session_id=row["session_id"],
user_id=row["user_id"],
agent_id=row["agent_id"],
name=row["name"],
state=row["state"],
metadata=row["metadata"],
created_at=row["created_at"],
updated_at=row["updated_at"],
)
finally:
self._put_conn(conn)
def update_session(self, session: ConversationSession) -> ConversationSession:
"""Update an existing session."""
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute(f"""
UPDATE {self.sessions_table}
SET user_id = %s, agent_id = %s, name = %s,
state = %s, metadata = %s, updated_at = %s
WHERE session_id = %s
""", (
session.user_id,
session.agent_id,
session.name,
json.dumps(session.state) if session.state else None,
json.dumps(session.metadata) if session.metadata else None,
session.updated_at,
session.session_id,
))
conn.commit()
return session
finally:
self._put_conn(conn)
def delete_session(self, session_id: str) -> bool:
"""Delete a session and all its messages."""
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute(f"""
DELETE FROM {self.sessions_table} WHERE session_id = %s
""", (session_id,))
deleted = cur.rowcount > 0
conn.commit()
return deleted
finally:
self._put_conn(conn)
def list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""List sessions, optionally filtered by user or agent."""
conn = self._get_conn()
try:
with conn.cursor(cursor_factory=self._RealDictCursor) as cur:
conditions = []
params = []
if user_id:
conditions.append("user_id = %s")
params.append(user_id)
if agent_id:
conditions.append("agent_id = %s")
params.append(agent_id)
where_clause = ""
if conditions:
where_clause = "WHERE " + " AND ".join(conditions)
params.extend([limit, offset])
cur.execute(f"""
SELECT * FROM {self.sessions_table}
{where_clause}
ORDER BY updated_at DESC
LIMIT %s OFFSET %s
""", params)
return [
ConversationSession(
session_id=row["session_id"],
user_id=row["user_id"],
agent_id=row["agent_id"],
name=row["name"],
state=row["state"],
metadata=row["metadata"],
created_at=row["created_at"],
updated_at=row["updated_at"],
)
for row in cur.fetchall()
]
finally:
self._put_conn(conn)
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Add a message to a session."""
message.session_id = session_id
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute(f"""
INSERT INTO {self.messages_table}
(id, session_id, role, content, tool_calls, tool_call_id, metadata, created_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
""", (
message.id,
session_id,
message.role,
message.content,
json.dumps(message.tool_calls) if message.tool_calls else None,
message.tool_call_id,
json.dumps(message.metadata) if message.metadata else None,
message.created_at,
))
conn.commit()
return message
finally:
self._put_conn(conn)
def get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Get messages from a session."""
conn = self._get_conn()
try:
with conn.cursor(cursor_factory=self._RealDictCursor) as cur:
conditions = ["session_id = %s"]
params: List[Any] = [session_id]
if before:
conditions.append("created_at < %s")
params.append(before)
if after:
conditions.append("created_at > %s")
params.append(after)
where_clause = "WHERE " + " AND ".join(conditions)
limit_clause = f"LIMIT {limit}" if limit else ""
cur.execute(f"""
SELECT * FROM {self.messages_table}
{where_clause}
ORDER BY created_at ASC
{limit_clause}
""", params)
return [
ConversationMessage(
id=row["id"],
session_id=row["session_id"],
role=row["role"],
content=row["content"],
tool_calls=row["tool_calls"],
tool_call_id=row["tool_call_id"],
metadata=row["metadata"],
created_at=row["created_at"],
)
for row in cur.fetchall()
]
finally:
self._put_conn(conn)
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Delete messages. If message_ids is None, delete all messages in session."""
conn = self._get_conn()
try:
with conn.cursor() as cur:
if message_ids:
placeholders = ",".join(["%s"] * len(message_ids))
cur.execute(f"""
DELETE FROM {self.messages_table}
WHERE session_id = %s AND id IN ({placeholders})
""", [session_id] + message_ids)
else:
cur.execute(f"""
DELETE FROM {self.messages_table} WHERE session_id = %s
""", (session_id,))
deleted = cur.rowcount
conn.commit()
return deleted
finally:
self._put_conn(conn)
def close(self) -> None:
"""Close the store and release resources."""
if self._pool:
self._pool.closeall()
self._pool = None
@@ -0,0 +1,240 @@
"""
SingleStore implementation of ConversationStore.
Requires: singlestoredb
Install: pip install singlestoredb
"""
import json
import logging
from typing import Any, List, Optional
from .base import ConversationStore, ConversationSession, ConversationMessage
logger = logging.getLogger(__name__)
class SingleStoreConversationStore(ConversationStore):
"""
SingleStore-based conversation store.
SingleStore is MySQL-compatible with additional vector capabilities.
Example:
store = SingleStoreConversationStore(
url="singlestoredb://user:password@host:3306/database"
)
"""
SCHEMA_VERSION = "1.0.0"
def __init__(
self,
url: Optional[str] = None,
host: str = "localhost",
port: int = 3306,
database: str = "praisonai",
user: str = "root",
password: str = "",
table_prefix: str = "praison_",
auto_create_tables: bool = True,
):
try:
import singlestoredb as s2
except ImportError:
raise ImportError(
"singlestoredb is required for SingleStore support. "
"Install with: pip install singlestoredb"
)
self._s2 = s2
self.table_prefix = table_prefix
self.sessions_table = f"{table_prefix}sessions"
self.messages_table = f"{table_prefix}messages"
if url:
self._conn = s2.connect(url)
else:
self._conn = s2.connect(
host=host,
port=port,
database=database,
user=user,
password=password,
)
if auto_create_tables:
self._create_tables()
def _create_tables(self) -> None:
"""Create tables if they don't exist."""
cur = self._conn.cursor()
cur.execute(f"""
CREATE TABLE IF NOT EXISTS {self.sessions_table} (
session_id VARCHAR(255) PRIMARY KEY,
user_id VARCHAR(255),
agent_id VARCHAR(255),
name VARCHAR(255),
state JSON,
metadata JSON,
created_at DOUBLE,
updated_at DOUBLE,
KEY idx_user (user_id),
KEY idx_agent (agent_id)
)
""")
cur.execute(f"""
CREATE TABLE IF NOT EXISTS {self.messages_table} (
id VARCHAR(255) PRIMARY KEY,
session_id VARCHAR(255) NOT NULL,
role VARCHAR(50) NOT NULL,
content TEXT,
tool_calls JSON,
tool_call_id VARCHAR(255),
metadata JSON,
created_at DOUBLE,
KEY idx_session (session_id),
KEY idx_created (session_id, created_at)
)
""")
self._conn.commit()
logger.info(f"SingleStore tables created")
def create_session(self, session: ConversationSession) -> ConversationSession:
cur = self._conn.cursor()
cur.execute(f"""
INSERT INTO {self.sessions_table}
(session_id, user_id, agent_id, name, state, metadata, created_at, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
""", (
session.session_id, session.user_id, session.agent_id, session.name,
json.dumps(session.state) if session.state else None,
json.dumps(session.metadata) if session.metadata else None,
session.created_at, session.updated_at,
))
self._conn.commit()
return session
def get_session(self, session_id: str) -> Optional[ConversationSession]:
cur = self._conn.cursor()
cur.execute(f"SELECT * FROM {self.sessions_table} WHERE session_id = %s", (session_id,))
row = cur.fetchone()
if not row:
return None
cols = [d[0] for d in cur.description]
data = dict(zip(cols, row))
return ConversationSession(
session_id=data["session_id"], user_id=data["user_id"],
agent_id=data["agent_id"], name=data["name"],
state=json.loads(data["state"]) if data["state"] else None,
metadata=json.loads(data["metadata"]) if data["metadata"] else None,
created_at=data["created_at"], updated_at=data["updated_at"],
)
def update_session(self, session: ConversationSession) -> ConversationSession:
cur = self._conn.cursor()
cur.execute(f"""
UPDATE {self.sessions_table}
SET user_id=%s, agent_id=%s, name=%s, state=%s, metadata=%s, updated_at=%s
WHERE session_id=%s
""", (
session.user_id, session.agent_id, session.name,
json.dumps(session.state) if session.state else None,
json.dumps(session.metadata) if session.metadata else None,
session.updated_at, session.session_id,
))
self._conn.commit()
return session
def delete_session(self, session_id: str) -> bool:
cur = self._conn.cursor()
cur.execute(f"DELETE FROM {self.messages_table} WHERE session_id = %s", (session_id,))
cur.execute(f"DELETE FROM {self.sessions_table} WHERE session_id = %s", (session_id,))
self._conn.commit()
return cur.rowcount > 0
def list_sessions(self, user_id: Optional[str] = None, agent_id: Optional[str] = None,
limit: int = 100, offset: int = 0) -> List[ConversationSession]:
cur = self._conn.cursor()
conditions, params = [], []
if user_id:
conditions.append("user_id = %s")
params.append(user_id)
if agent_id:
conditions.append("agent_id = %s")
params.append(agent_id)
where = "WHERE " + " AND ".join(conditions) if conditions else ""
params.extend([limit, offset])
cur.execute(f"SELECT * FROM {self.sessions_table} {where} ORDER BY updated_at DESC LIMIT %s OFFSET %s", params)
cols = [d[0] for d in cur.description]
return [ConversationSession(
session_id=dict(zip(cols, r))["session_id"],
user_id=dict(zip(cols, r))["user_id"],
agent_id=dict(zip(cols, r))["agent_id"],
name=dict(zip(cols, r))["name"],
state=json.loads(dict(zip(cols, r))["state"]) if dict(zip(cols, r))["state"] else None,
metadata=json.loads(dict(zip(cols, r))["metadata"]) if dict(zip(cols, r))["metadata"] else None,
created_at=dict(zip(cols, r))["created_at"],
updated_at=dict(zip(cols, r))["updated_at"],
) for r in cur.fetchall()]
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
message.session_id = session_id
cur = self._conn.cursor()
cur.execute(f"""
INSERT INTO {self.messages_table}
(id, session_id, role, content, tool_calls, tool_call_id, metadata, created_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
""", (
message.id, session_id, message.role, message.content,
json.dumps(message.tool_calls) if message.tool_calls else None,
message.tool_call_id,
json.dumps(message.metadata) if message.metadata else None,
message.created_at,
))
self._conn.commit()
return message
def get_messages(self, session_id: str, limit: Optional[int] = None,
before: Optional[float] = None, after: Optional[float] = None) -> List[ConversationMessage]:
cur = self._conn.cursor()
conditions, params = ["session_id = %s"], [session_id]
if before:
conditions.append("created_at < %s")
params.append(before)
if after:
conditions.append("created_at > %s")
params.append(after)
where = "WHERE " + " AND ".join(conditions)
limit_clause = f"LIMIT {limit}" if limit else ""
cur.execute(f"SELECT * FROM {self.messages_table} {where} ORDER BY created_at ASC {limit_clause}", params)
cols = [d[0] for d in cur.description]
return [ConversationMessage(
id=dict(zip(cols, r))["id"],
session_id=dict(zip(cols, r))["session_id"],
role=dict(zip(cols, r))["role"],
content=dict(zip(cols, r))["content"],
tool_calls=json.loads(dict(zip(cols, r))["tool_calls"]) if dict(zip(cols, r))["tool_calls"] else None,
tool_call_id=dict(zip(cols, r))["tool_call_id"],
metadata=json.loads(dict(zip(cols, r))["metadata"]) if dict(zip(cols, r))["metadata"] else None,
created_at=dict(zip(cols, r))["created_at"],
) for r in cur.fetchall()]
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
cur = self._conn.cursor()
if message_ids:
placeholders = ",".join(["%s"] * len(message_ids))
cur.execute(f"DELETE FROM {self.messages_table} WHERE session_id = %s AND id IN ({placeholders})",
[session_id] + message_ids)
else:
cur.execute(f"DELETE FROM {self.messages_table} WHERE session_id = %s", (session_id,))
self._conn.commit()
return cur.rowcount
def close(self) -> None:
if self._conn:
self._conn.close()
self._conn = None
@@ -0,0 +1,341 @@
"""
SQLite implementation of ConversationStore.
Zero external dependencies - uses built-in sqlite3 module.
"""
import json
import logging
import sqlite3
import threading
from typing import Any, List, Optional
from .base import ConversationStore, ConversationSession, ConversationMessage
logger = logging.getLogger(__name__)
class SQLiteConversationStore(ConversationStore):
"""
SQLite-based conversation store.
Zero external dependencies - uses Python's built-in sqlite3.
Example:
store = SQLiteConversationStore(
path="./praisonai.db"
)
"""
SCHEMA_VERSION = "1.0.0"
def __init__(
self,
path: str = "praisonai_conversations.db",
table_prefix: str = "praison_",
auto_create_tables: bool = True,
check_same_thread: bool = False,
):
"""
Initialize SQLite conversation store.
Args:
path: Path to SQLite database file
table_prefix: Prefix for table names
auto_create_tables: Create tables if they don't exist
check_same_thread: SQLite check_same_thread parameter
"""
self.path = path
self.table_prefix = table_prefix
self.sessions_table = f"{table_prefix}sessions"
self.messages_table = f"{table_prefix}messages"
self._check_same_thread = check_same_thread
self._local = threading.local()
if auto_create_tables:
self._create_tables()
def _get_conn(self) -> sqlite3.Connection:
"""Get thread-local connection."""
if not hasattr(self._local, "conn") or self._local.conn is None:
self._local.conn = sqlite3.connect(
self.path,
check_same_thread=self._check_same_thread
)
self._local.conn.row_factory = sqlite3.Row
return self._local.conn
def _create_tables(self) -> None:
"""Create tables if they don't exist."""
conn = self._get_conn()
cur = conn.cursor()
# Sessions table
cur.execute(f"""
CREATE TABLE IF NOT EXISTS {self.sessions_table} (
session_id TEXT PRIMARY KEY,
user_id TEXT,
agent_id TEXT,
name TEXT,
state TEXT,
metadata TEXT,
created_at REAL,
updated_at REAL
)
""")
# Sessions indexes
cur.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}sessions_user
ON {self.sessions_table}(user_id)
""")
cur.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}sessions_agent
ON {self.sessions_table}(agent_id)
""")
# Messages table
cur.execute(f"""
CREATE TABLE IF NOT EXISTS {self.messages_table} (
id TEXT PRIMARY KEY,
session_id TEXT NOT NULL,
role TEXT NOT NULL,
content TEXT,
tool_calls TEXT,
tool_call_id TEXT,
metadata TEXT,
created_at REAL,
FOREIGN KEY (session_id) REFERENCES {self.sessions_table}(session_id) ON DELETE CASCADE
)
""")
# Messages indexes
cur.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}messages_session
ON {self.messages_table}(session_id)
""")
cur.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}messages_created
ON {self.messages_table}(session_id, created_at)
""")
# Enable foreign keys
cur.execute("PRAGMA foreign_keys = ON")
conn.commit()
logger.info(f"SQLite tables created: {self.sessions_table}, {self.messages_table}")
def create_session(self, session: ConversationSession) -> ConversationSession:
"""Create a new session."""
conn = self._get_conn()
cur = conn.cursor()
cur.execute(f"""
INSERT INTO {self.sessions_table}
(session_id, user_id, agent_id, name, state, metadata, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""", (
session.session_id,
session.user_id,
session.agent_id,
session.name,
json.dumps(session.state) if session.state else None,
json.dumps(session.metadata) if session.metadata else None,
session.created_at,
session.updated_at,
))
conn.commit()
return session
def get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Get a session by ID."""
conn = self._get_conn()
cur = conn.cursor()
cur.execute(f"""
SELECT * FROM {self.sessions_table} WHERE session_id = ?
""", (session_id,))
row = cur.fetchone()
if not row:
return None
return ConversationSession(
session_id=row["session_id"],
user_id=row["user_id"],
agent_id=row["agent_id"],
name=row["name"],
state=json.loads(row["state"]) if row["state"] else None,
metadata=json.loads(row["metadata"]) if row["metadata"] else None,
created_at=row["created_at"],
updated_at=row["updated_at"],
)
def update_session(self, session: ConversationSession) -> ConversationSession:
"""Update an existing session."""
conn = self._get_conn()
cur = conn.cursor()
cur.execute(f"""
UPDATE {self.sessions_table}
SET user_id = ?, agent_id = ?, name = ?,
state = ?, metadata = ?, updated_at = ?
WHERE session_id = ?
""", (
session.user_id,
session.agent_id,
session.name,
json.dumps(session.state) if session.state else None,
json.dumps(session.metadata) if session.metadata else None,
session.updated_at,
session.session_id,
))
conn.commit()
return session
def delete_session(self, session_id: str) -> bool:
"""Delete a session and all its messages."""
conn = self._get_conn()
cur = conn.cursor()
# Delete messages first (foreign key)
cur.execute(f"DELETE FROM {self.messages_table} WHERE session_id = ?", (session_id,))
cur.execute(f"DELETE FROM {self.sessions_table} WHERE session_id = ?", (session_id,))
deleted = cur.rowcount > 0
conn.commit()
return deleted
def list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""List sessions, optionally filtered by user or agent."""
conn = self._get_conn()
cur = conn.cursor()
conditions = []
params: List[Any] = []
if user_id:
conditions.append("user_id = ?")
params.append(user_id)
if agent_id:
conditions.append("agent_id = ?")
params.append(agent_id)
where_clause = ""
if conditions:
where_clause = "WHERE " + " AND ".join(conditions)
params.extend([limit, offset])
cur.execute(f"""
SELECT * FROM {self.sessions_table}
{where_clause}
ORDER BY updated_at DESC
LIMIT ? OFFSET ?
""", params)
return [
ConversationSession(
session_id=row["session_id"],
user_id=row["user_id"],
agent_id=row["agent_id"],
name=row["name"],
state=json.loads(row["state"]) if row["state"] else None,
metadata=json.loads(row["metadata"]) if row["metadata"] else None,
created_at=row["created_at"],
updated_at=row["updated_at"],
)
for row in cur.fetchall()
]
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Add a message to a session."""
message.session_id = session_id
conn = self._get_conn()
cur = conn.cursor()
cur.execute(f"""
INSERT INTO {self.messages_table}
(id, session_id, role, content, tool_calls, tool_call_id, metadata, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""", (
message.id,
session_id,
message.role,
message.content,
json.dumps(message.tool_calls) if message.tool_calls else None,
message.tool_call_id,
json.dumps(message.metadata) if message.metadata else None,
message.created_at,
))
conn.commit()
return message
def get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Get messages from a session."""
conn = self._get_conn()
cur = conn.cursor()
conditions = ["session_id = ?"]
params: List[Any] = [session_id]
if before:
conditions.append("created_at < ?")
params.append(before)
if after:
conditions.append("created_at > ?")
params.append(after)
where_clause = "WHERE " + " AND ".join(conditions)
limit_clause = f"LIMIT {limit}" if limit else ""
cur.execute(f"""
SELECT * FROM {self.messages_table}
{where_clause}
ORDER BY created_at ASC
{limit_clause}
""", params)
return [
ConversationMessage(
id=row["id"],
session_id=row["session_id"],
role=row["role"],
content=row["content"],
tool_calls=json.loads(row["tool_calls"]) if row["tool_calls"] else None,
tool_call_id=row["tool_call_id"],
metadata=json.loads(row["metadata"]) if row["metadata"] else None,
created_at=row["created_at"],
)
for row in cur.fetchall()
]
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Delete messages."""
conn = self._get_conn()
cur = conn.cursor()
if message_ids:
placeholders = ",".join(["?"] * len(message_ids))
cur.execute(f"""
DELETE FROM {self.messages_table}
WHERE session_id = ? AND id IN ({placeholders})
""", [session_id] + message_ids)
else:
cur.execute(f"""
DELETE FROM {self.messages_table} WHERE session_id = ?
""", (session_id,))
deleted = cur.rowcount
conn.commit()
return deleted
def close(self) -> None:
"""Close the store."""
if hasattr(self._local, "conn") and self._local.conn:
self._local.conn.close()
self._local.conn = None
@@ -0,0 +1,203 @@
"""
Supabase implementation of ConversationStore.
Requires: supabase
Install: pip install supabase
"""
import json
import logging
import time
from typing import Any, Dict, List, Optional
from .base import ConversationStore, ConversationSession, ConversationMessage
logger = logging.getLogger(__name__)
class SupabaseConversationStore(ConversationStore):
"""
Supabase-based conversation store.
Uses Supabase's PostgreSQL backend via REST API.
Example:
store = SupabaseConversationStore(
url="https://xxx.supabase.co",
key="your-anon-key"
)
"""
SCHEMA_VERSION = "1.0.0"
def __init__(
self,
url: Optional[str] = None,
key: Optional[str] = None,
table_prefix: str = "praison_",
auto_create_tables: bool = False, # Tables should be created via Supabase dashboard
):
"""
Initialize Supabase conversation store.
Args:
url: Supabase project URL
key: Supabase anon/service key
table_prefix: Prefix for table names
auto_create_tables: Not supported - use Supabase dashboard
"""
try:
from supabase import create_client, Client
except ImportError:
raise ImportError(
"supabase is required for Supabase support. "
"Install with: pip install supabase"
)
import os
url = url or os.getenv("SUPABASE_URL")
key = key or os.getenv("SUPABASE_KEY")
if not url or not key:
raise ValueError("Supabase URL and key are required")
self._client: Client = create_client(url, key)
self.table_prefix = table_prefix
self.sessions_table = f"{table_prefix}sessions"
self.messages_table = f"{table_prefix}messages"
if auto_create_tables:
logger.warning("auto_create_tables not supported for Supabase. Create tables via dashboard.")
def create_session(self, session: ConversationSession) -> ConversationSession:
data = {
"session_id": session.session_id,
"user_id": session.user_id,
"agent_id": session.agent_id,
"name": session.name,
"state": session.state,
"metadata": session.metadata,
"created_at": session.created_at,
"updated_at": session.updated_at,
}
self._client.table(self.sessions_table).insert(data).execute()
return session
def get_session(self, session_id: str) -> Optional[ConversationSession]:
result = self._client.table(self.sessions_table).select("*").eq("session_id", session_id).execute()
if not result.data:
return None
row = result.data[0]
return ConversationSession(
session_id=row["session_id"],
user_id=row.get("user_id"),
agent_id=row.get("agent_id"),
name=row.get("name"),
state=row.get("state"),
metadata=row.get("metadata"),
created_at=row.get("created_at"),
updated_at=row.get("updated_at"),
)
def update_session(self, session: ConversationSession) -> ConversationSession:
data = {
"user_id": session.user_id,
"agent_id": session.agent_id,
"name": session.name,
"state": session.state,
"metadata": session.metadata,
"updated_at": session.updated_at,
}
self._client.table(self.sessions_table).update(data).eq("session_id", session.session_id).execute()
return session
def delete_session(self, session_id: str) -> bool:
# Delete messages first
self._client.table(self.messages_table).delete().eq("session_id", session_id).execute()
result = self._client.table(self.sessions_table).delete().eq("session_id", session_id).execute()
return len(result.data) > 0 if result.data else False
def list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
query = self._client.table(self.sessions_table).select("*")
if user_id:
query = query.eq("user_id", user_id)
if agent_id:
query = query.eq("agent_id", agent_id)
query = query.order("updated_at", desc=True).range(offset, offset + limit - 1)
result = query.execute()
return [
ConversationSession(
session_id=row["session_id"],
user_id=row.get("user_id"),
agent_id=row.get("agent_id"),
name=row.get("name"),
state=row.get("state"),
metadata=row.get("metadata"),
created_at=row.get("created_at"),
updated_at=row.get("updated_at"),
)
for row in (result.data or [])
]
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
message.session_id = session_id
data = {
"id": message.id,
"session_id": session_id,
"role": message.role,
"content": message.content,
"tool_calls": message.tool_calls,
"tool_call_id": message.tool_call_id,
"metadata": message.metadata,
"created_at": message.created_at,
}
self._client.table(self.messages_table).insert(data).execute()
return message
def get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
query = self._client.table(self.messages_table).select("*").eq("session_id", session_id)
if before:
query = query.lt("created_at", before)
if after:
query = query.gt("created_at", after)
query = query.order("created_at", desc=False)
if limit:
query = query.limit(limit)
result = query.execute()
return [
ConversationMessage(
id=row["id"],
session_id=row["session_id"],
role=row["role"],
content=row.get("content", ""),
tool_calls=row.get("tool_calls"),
tool_call_id=row.get("tool_call_id"),
metadata=row.get("metadata"),
created_at=row.get("created_at"),
)
for row in (result.data or [])
]
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
if message_ids:
result = self._client.table(self.messages_table).delete().eq("session_id", session_id).in_("id", message_ids).execute()
else:
result = self._client.table(self.messages_table).delete().eq("session_id", session_id).execute()
return len(result.data) if result.data else 0
def close(self) -> None:
pass # Supabase client doesn't need explicit close
@@ -0,0 +1,287 @@
"""
SurrealDB implementation of ConversationStore.
Requires: surrealdb
Install: pip install surrealdb
"""
import json
import logging
from typing import List, Optional
from .base import ConversationStore, ConversationSession, ConversationMessage
logger = logging.getLogger(__name__)
class SurrealDBConversationStore(ConversationStore):
"""
SurrealDB-based conversation store.
SurrealDB is a multi-model database supporting SQL-like queries.
Example:
store = SurrealDBConversationStore(
url="ws://localhost:8000/rpc",
namespace="praisonai",
database="conversations"
)
"""
SCHEMA_VERSION = "1.0.0"
def __init__(
self,
url: str = "ws://localhost:8000/rpc",
namespace: str = "praisonai",
database: str = "conversations",
username: Optional[str] = None,
password: Optional[str] = None,
table_prefix: str = "praison_",
auto_create_tables: bool = True,
):
try:
from surrealdb import Surreal
except ImportError:
raise ImportError(
"surrealdb is required for SurrealDB support. "
"Install with: pip install surrealdb"
)
self._Surreal = Surreal
self.url = url
self.namespace = namespace
self.database = database
self.username = username
self.password = password
self.table_prefix = table_prefix
self.sessions_table = f"{table_prefix}sessions"
self.messages_table = f"{table_prefix}messages"
self._client = None
# Note: SurrealDB Python client is async, we'll use sync wrapper
self._init_sync()
def _init_sync(self):
"""Initialize synchronously using asyncio."""
import asyncio
async def _init():
self._client = self._Surreal(self.url)
await self._client.connect()
if self.username and self.password:
await self._client.signin({"user": self.username, "pass": self.password})
await self._client.use(self.namespace, self.database)
try:
loop = asyncio.get_event_loop()
except RuntimeError:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(_init())
def _run_sync(self, coro):
"""Run async coroutine synchronously."""
import asyncio
try:
loop = asyncio.get_event_loop()
except RuntimeError:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
return loop.run_until_complete(coro)
def create_session(self, session: ConversationSession) -> ConversationSession:
data = {
"session_id": session.session_id,
"user_id": session.user_id,
"agent_id": session.agent_id,
"name": session.name,
"state": session.state,
"metadata": session.metadata,
"created_at": session.created_at,
"updated_at": session.updated_at,
}
self._run_sync(self._client.create(self.sessions_table, data))
return session
def get_session(self, session_id: str) -> Optional[ConversationSession]:
result = self._run_sync(
self._client.query(
f"SELECT * FROM {self.sessions_table} WHERE session_id = $sid",
{"sid": session_id}
)
)
if not result or not result[0].get("result"):
return None
row = result[0]["result"][0]
return ConversationSession(
session_id=row["session_id"],
user_id=row.get("user_id"),
agent_id=row.get("agent_id"),
name=row.get("name"),
state=row.get("state"),
metadata=row.get("metadata"),
created_at=row.get("created_at"),
updated_at=row.get("updated_at"),
)
def update_session(self, session: ConversationSession) -> ConversationSession:
self._run_sync(
self._client.query(
f"""UPDATE {self.sessions_table} SET
user_id = $user_id, agent_id = $agent_id, name = $name,
state = $state, metadata = $metadata, updated_at = $updated_at
WHERE session_id = $session_id""",
{
"session_id": session.session_id,
"user_id": session.user_id,
"agent_id": session.agent_id,
"name": session.name,
"state": session.state,
"metadata": session.metadata,
"updated_at": session.updated_at,
}
)
)
return session
def delete_session(self, session_id: str) -> bool:
self._run_sync(
self._client.query(
f"DELETE FROM {self.messages_table} WHERE session_id = $sid",
{"sid": session_id}
)
)
result = self._run_sync(
self._client.query(
f"DELETE FROM {self.sessions_table} WHERE session_id = $sid",
{"sid": session_id}
)
)
return True
def list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
conditions = []
params = {"limit": limit, "offset": offset}
if user_id:
conditions.append("user_id = $user_id")
params["user_id"] = user_id
if agent_id:
conditions.append("agent_id = $agent_id")
params["agent_id"] = agent_id
where = "WHERE " + " AND ".join(conditions) if conditions else ""
result = self._run_sync(
self._client.query(
f"SELECT * FROM {self.sessions_table} {where} ORDER BY updated_at DESC LIMIT $limit START $offset",
params
)
)
if not result or not result[0].get("result"):
return []
return [
ConversationSession(
session_id=row["session_id"],
user_id=row.get("user_id"),
agent_id=row.get("agent_id"),
name=row.get("name"),
state=row.get("state"),
metadata=row.get("metadata"),
created_at=row.get("created_at"),
updated_at=row.get("updated_at"),
)
for row in result[0]["result"]
]
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
message.session_id = session_id
data = {
"id": message.id,
"session_id": session_id,
"role": message.role,
"content": message.content,
"tool_calls": message.tool_calls,
"tool_call_id": message.tool_call_id,
"metadata": message.metadata,
"created_at": message.created_at,
}
self._run_sync(self._client.create(self.messages_table, data))
return message
def get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
conditions = ["session_id = $session_id"]
params = {"session_id": session_id}
if before:
conditions.append("created_at < $before")
params["before"] = before
if after:
conditions.append("created_at > $after")
params["after"] = after
where = "WHERE " + " AND ".join(conditions)
limit_clause = f"LIMIT {limit}" if limit else ""
result = self._run_sync(
self._client.query(
f"SELECT * FROM {self.messages_table} {where} ORDER BY created_at ASC {limit_clause}",
params
)
)
if not result or not result[0].get("result"):
return []
return [
ConversationMessage(
id=row["id"],
session_id=row["session_id"],
role=row["role"],
content=row.get("content", ""),
tool_calls=row.get("tool_calls"),
tool_call_id=row.get("tool_call_id"),
metadata=row.get("metadata"),
created_at=row.get("created_at"),
)
for row in result[0]["result"]
]
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
if message_ids:
self._run_sync(
self._client.query(
f"DELETE FROM {self.messages_table} WHERE session_id = $sid AND id IN $ids",
{"sid": session_id, "ids": message_ids}
)
)
return len(message_ids)
else:
self._run_sync(
self._client.query(
f"DELETE FROM {self.messages_table} WHERE session_id = $sid",
{"sid": session_id}
)
)
return -1
def close(self) -> None:
if self._client:
self._run_sync(self._client.close())
self._client = None
@@ -0,0 +1,301 @@
"""
Factory functions for creating store instances.
Provides lazy loading of backend implementations to avoid importing
unused dependencies.
"""
import logging
from typing import Any, Dict, Optional
from .config import PersistenceConfig
logger = logging.getLogger(__name__)
def create_conversation_store(
backend: str,
url: Optional[str] = None,
**options: Any
):
"""
Create a ConversationStore instance.
Args:
backend: Backend type (postgres, mysql, sqlite, singlestore, supabase, surrealdb)
url: Connection URL
**options: Backend-specific options
Returns:
ConversationStore instance
Example:
store = create_conversation_store(
"postgres",
url="postgresql://localhost:5432/praisonai"
)
"""
backend = backend.lower()
if backend == "postgres":
from .conversation.postgres import PostgresConversationStore
return PostgresConversationStore(url=url, **options)
elif backend == "mysql":
from .conversation.mysql import MySQLConversationStore
return MySQLConversationStore(url=url, **options)
elif backend == "sqlite":
from .conversation.sqlite import SQLiteConversationStore
path = url or options.pop("path", None)
return SQLiteConversationStore(path=path, **options)
elif backend == "singlestore":
from .conversation.singlestore import SingleStoreConversationStore
return SingleStoreConversationStore(url=url, **options)
elif backend == "supabase":
from .conversation.supabase import SupabaseConversationStore
return SupabaseConversationStore(url=url, **options)
elif backend == "surrealdb":
from .conversation.surrealdb import SurrealDBConversationStore
return SurrealDBConversationStore(url=url, **options)
elif backend == "json":
from .conversation.json_store import JSONConversationStore
path = url or options.pop("path", None) or "./praisonai_conversations"
return JSONConversationStore(path=path, **options)
elif backend in ("async_postgres", "asyncpg", "postgres_async"):
from .conversation.async_postgres import AsyncPostgresConversationStore
return AsyncPostgresConversationStore(url=url, **options)
elif backend in ("async_sqlite", "aiosqlite", "sqlite_async"):
from .conversation.async_sqlite import AsyncSQLiteConversationStore
path = url or options.pop("path", None)
return AsyncSQLiteConversationStore(path=path, **options)
elif backend in ("async_mysql", "aiomysql", "mysql_async"):
from .conversation.async_mysql import AsyncMySQLConversationStore
return AsyncMySQLConversationStore(url=url, **options)
else:
raise ValueError(
f"Unknown conversation store backend: {backend}. "
f"Supported: postgres, mysql, sqlite, json, singlestore, supabase, surrealdb, "
f"async_postgres, async_sqlite, async_mysql"
)
def create_knowledge_store(
backend: str,
url: Optional[str] = None,
**options: Any
):
"""
Create a KnowledgeStore instance.
Args:
backend: Backend type (qdrant, pinecone, chroma, weaviate, lancedb, milvus, pgvector, redis, cassandra, clickhouse)
url: Connection URL
**options: Backend-specific options
Returns:
KnowledgeStore instance
Example:
store = create_knowledge_store(
"qdrant",
url="http://localhost:6333"
)
"""
backend = backend.lower()
if backend == "qdrant":
from .knowledge.qdrant import QdrantKnowledgeStore
return QdrantKnowledgeStore(url=url, **options)
elif backend == "pinecone":
from .knowledge.pinecone import PineconeKnowledgeStore
return PineconeKnowledgeStore(**options)
elif backend == "chroma":
from .knowledge.chroma import ChromaKnowledgeStore
path = url or options.pop("path", None)
return ChromaKnowledgeStore(path=path, **options)
elif backend == "weaviate":
from .knowledge.weaviate import WeaviateKnowledgeStore
return WeaviateKnowledgeStore(url=url, **options)
elif backend == "lancedb":
from .knowledge.lancedb import LanceDBKnowledgeStore
path = url or options.pop("path", None)
return LanceDBKnowledgeStore(path=path, **options)
elif backend == "milvus":
from .knowledge.milvus import MilvusKnowledgeStore
return MilvusKnowledgeStore(url=url, **options)
elif backend == "pgvector":
from .knowledge.pgvector import PGVectorKnowledgeStore
return PGVectorKnowledgeStore(url=url, **options)
elif backend == "redis":
from .knowledge.redis_vector import RedisVectorKnowledgeStore
return RedisVectorKnowledgeStore(url=url, **options)
elif backend == "cassandra":
from .knowledge.cassandra import CassandraKnowledgeStore
return CassandraKnowledgeStore(**options)
elif backend == "clickhouse":
from .knowledge.clickhouse import ClickHouseKnowledgeStore
return ClickHouseKnowledgeStore(**options)
elif backend == "couchbase":
from .knowledge.couchbase import CouchbaseKnowledgeStore
return CouchbaseKnowledgeStore(**options)
elif backend in ("mongodb_vector", "mongodb_atlas", "mongo_vector"):
from .knowledge.mongodb_vector import MongoDBVectorKnowledgeStore
return MongoDBVectorKnowledgeStore(url=url, **options)
elif backend in ("singlestore_vector", "singlestore_v"):
from .knowledge.singlestore_vector import SingleStoreVectorKnowledgeStore
return SingleStoreVectorKnowledgeStore(url=url, **options)
elif backend in ("surrealdb_vector", "surrealdb_v"):
from .knowledge.surrealdb_vector import SurrealDBVectorKnowledgeStore
return SurrealDBVectorKnowledgeStore(url=url, **options)
elif backend in ("upstash_vector", "upstash_v"):
from .knowledge.upstash_vector import UpstashVectorKnowledgeStore
return UpstashVectorKnowledgeStore(url=url, **options)
elif backend == "lightrag":
from .knowledge.lightrag_adapter import LightRAGKnowledgeStore
return LightRAGKnowledgeStore(**options)
elif backend in ("langchain", "langchain_adapter"):
from .knowledge.langchain_adapter import LangChainKnowledgeStore
return LangChainKnowledgeStore(**options)
elif backend in ("llamaindex", "llama_index", "llamaindex_adapter"):
from .knowledge.llamaindex_adapter import LlamaIndexKnowledgeStore
return LlamaIndexKnowledgeStore(**options)
elif backend in ("cosmosdb", "cosmos", "azure_cosmos", "cosmosdb_vector"):
from .knowledge.cosmosdb_vector import CosmosDBVectorKnowledgeStore
return CosmosDBVectorKnowledgeStore(**options)
else:
raise ValueError(
f"Unknown knowledge store backend: {backend}. "
f"Supported: qdrant, pinecone, chroma, weaviate, lancedb, milvus, pgvector, redis, cassandra, clickhouse, "
f"couchbase, mongodb_vector, singlestore_vector, surrealdb_vector, upstash_vector, lightrag, langchain, llamaindex, cosmosdb"
)
def create_state_store(
backend: str,
url: Optional[str] = None,
**options: Any
):
"""
Create a StateStore instance.
Args:
backend: Backend type (redis, dynamodb, firestore, mongodb, upstash, memory)
url: Connection URL
**options: Backend-specific options
Returns:
StateStore instance
Example:
store = create_state_store(
"redis",
url="redis://localhost:6379"
)
"""
backend = backend.lower()
if backend == "redis":
from .state.redis import RedisStateStore
return RedisStateStore(url=url, **options)
elif backend == "dynamodb":
from .state.dynamodb import DynamoDBStateStore
return DynamoDBStateStore(**options)
elif backend == "firestore":
from .state.firestore import FirestoreStateStore
return FirestoreStateStore(**options)
elif backend == "mongodb":
from .state.mongodb import MongoDBStateStore
return MongoDBStateStore(url=url, **options)
elif backend == "upstash":
from .state.upstash import UpstashStateStore
return UpstashStateStore(url=url, **options)
elif backend == "memory":
from .state.memory import MemoryStateStore
return MemoryStateStore(**options)
elif backend == "gcs":
from .state.gcs import GCSStateStore
bucket = options.pop("bucket_name", None) or options.pop("bucket", None)
if not bucket:
raise ValueError("GCS state store requires 'bucket_name' option")
return GCSStateStore(bucket_name=bucket, **options)
elif backend in ("async_mongodb", "motor", "mongodb_async"):
from .state.async_mongodb import AsyncMongoDBStateStore
return AsyncMongoDBStateStore(url=url, **options)
else:
raise ValueError(
f"Unknown state store backend: {backend}. "
f"Supported: redis, dynamodb, firestore, mongodb, upstash, memory, gcs, async_mongodb"
)
def create_stores_from_config(config: PersistenceConfig) -> Dict[str, Any]:
"""
Create all configured stores from a PersistenceConfig.
Returns:
Dict with keys: conversation, knowledge, state (values may be None)
"""
stores = {
"conversation": None,
"knowledge": None,
"state": None,
}
if config.conversation_store:
stores["conversation"] = create_conversation_store(
config.conversation_store,
url=config.conversation_url,
**config.conversation_options
)
if config.knowledge_store:
stores["knowledge"] = create_knowledge_store(
config.knowledge_store,
url=config.knowledge_url,
**config.knowledge_options
)
if config.state_store:
stores["state"] = create_state_store(
config.state_store,
url=config.state_url,
**config.state_options
)
return stores
@@ -0,0 +1,18 @@
"""
Agent hooks for automatic persistence integration.
Provides wrapper functions to add persistence capabilities to PraisonAI agents
without modifying the core SDK.
"""
from .agent_hooks import (
wrap_agent_with_persistence,
PersistentAgent,
create_persistent_session,
)
__all__ = [
"wrap_agent_with_persistence",
"PersistentAgent",
"create_persistent_session",
]
@@ -0,0 +1,297 @@
"""
Agent hooks for automatic persistence integration.
Provides wrapper functions to add persistence capabilities to PraisonAI agents
without modifying the core SDK.
"""
import time
import uuid
import logging
from typing import Any, Dict, List, Optional, Callable
from functools import wraps
from ..orchestrator import PersistenceOrchestrator
from ..conversation.base import ConversationMessage, ConversationSession
logger = logging.getLogger(__name__)
def wrap_agent_with_persistence(
agent,
orchestrator: PersistenceOrchestrator,
session_id: Optional[str] = None,
user_id: Optional[str] = None,
auto_resume: bool = True,
):
"""
Wrap an agent to add automatic persistence hooks.
This function wraps the agent's chat method to automatically:
1. Load/create session on first message
2. Persist each user message
3. Persist each assistant response
4. Update session metadata after each interaction
Args:
agent: PraisonAI Agent instance
orchestrator: PersistenceOrchestrator instance
session_id: Session ID (auto-generated if None)
user_id: User ID for the session
auto_resume: Whether to auto-resume existing session
Returns:
The wrapped agent with persistence capabilities
Example:
from praisonai.persistence import PersistenceOrchestrator
from praisonai.persistence.hooks import wrap_agent_with_persistence
from praisonaiagents import Agent
orchestrator = PersistenceOrchestrator(...)
agent = Agent(name="Assistant", role="Helper")
# Wrap agent with persistence
agent = wrap_agent_with_persistence(agent, orchestrator, session_id="my-session")
# Now all chats are automatically persisted
response = agent.chat("Hello!")
"""
session_id = session_id or f"session-{uuid.uuid4().hex[:8]}"
user_id = user_id or getattr(agent, 'user_id', 'default')
# Store original chat method
original_chat = agent.chat
_session_initialized = [False] # Use list to allow mutation in closure
@wraps(original_chat)
def chat_with_persistence(prompt, *args, **kwargs):
nonlocal _session_initialized
# Initialize session on first call
if not _session_initialized[0]:
history = orchestrator.on_agent_start(
agent,
session_id=session_id,
user_id=user_id,
resume=auto_resume
)
# Inject history into agent's chat_history if resuming
if auto_resume and history:
agent.chat_history = [
{"role": msg.role, "content": msg.content}
for msg in history
]
logger.info(f"Resumed session {session_id} with {len(history)} messages")
_session_initialized[0] = True
# Persist user message
orchestrator.on_message(session_id, "user", prompt if isinstance(prompt, str) else str(prompt))
# Call original chat
response = original_chat(prompt, *args, **kwargs)
# Persist assistant response
if response:
orchestrator.on_message(session_id, "assistant", response)
return response
# Replace chat method
agent.chat = chat_with_persistence
# Add persistence-related methods to agent
agent._persistence_orchestrator = orchestrator
agent._persistence_session_id = session_id
def get_session():
"""Get the current session."""
return orchestrator.get_session(session_id)
def get_messages(limit: Optional[int] = None):
"""Get messages from the current session."""
return orchestrator.get_messages(session_id, limit=limit)
def end_session():
"""End the current session."""
orchestrator.on_agent_end(agent, session_id)
agent.get_session = get_session
agent.get_messages = get_messages
agent.end_session = end_session
return agent
class PersistentAgent:
"""
A wrapper class that provides persistence capabilities for any agent.
This is an alternative to wrap_agent_with_persistence that uses composition
instead of monkey-patching.
Example:
from praisonai.persistence import PersistenceOrchestrator
from praisonai.persistence.hooks import PersistentAgent
from praisonaiagents import Agent
orchestrator = PersistenceOrchestrator(...)
base_agent = Agent(name="Assistant", role="Helper")
agent = PersistentAgent(base_agent, orchestrator, session_id="my-session")
response = agent.chat("Hello!")
"""
def __init__(
self,
agent,
orchestrator: PersistenceOrchestrator,
session_id: Optional[str] = None,
user_id: Optional[str] = None,
auto_resume: bool = True,
):
self._agent = agent
self._orchestrator = orchestrator
self._session_id = session_id or f"session-{uuid.uuid4().hex[:8]}"
self._user_id = user_id or getattr(agent, 'user_id', 'default')
self._auto_resume = auto_resume
self._session_initialized = False
def __getattr__(self, name):
"""Delegate attribute access to wrapped agent."""
return getattr(self._agent, name)
def _ensure_session(self):
"""Ensure session is initialized."""
if not self._session_initialized:
history = self._orchestrator.on_agent_start(
self._agent,
session_id=self._session_id,
user_id=self._user_id,
resume=self._auto_resume
)
if self._auto_resume and history:
self._agent.chat_history = [
{"role": msg.role, "content": msg.content}
for msg in history
]
logger.info(f"Resumed session {self._session_id} with {len(history)} messages")
self._session_initialized = True
def chat(self, prompt, *args, **kwargs):
"""Chat with automatic persistence."""
self._ensure_session()
# Persist user message
self._orchestrator.on_message(
self._session_id, "user",
prompt if isinstance(prompt, str) else str(prompt)
)
# Call agent chat
response = self._agent.chat(prompt, *args, **kwargs)
# Persist assistant response
if response:
self._orchestrator.on_message(self._session_id, "assistant", response)
return response
def get_session(self) -> Optional[ConversationSession]:
"""Get the current session."""
return self._orchestrator.get_session(self._session_id)
def get_messages(self, limit: Optional[int] = None) -> List[ConversationMessage]:
"""Get messages from the current session."""
return self._orchestrator.get_messages(self._session_id, limit=limit)
def end_session(self):
"""End the current session."""
self._orchestrator.on_agent_end(self._agent, self._session_id)
@property
def session_id(self) -> str:
"""Get the session ID."""
return self._session_id
def create_persistent_session(
orchestrator: PersistenceOrchestrator,
session_id: Optional[str] = None,
user_id: Optional[str] = None,
):
"""
Create a context manager for persistent sessions.
Example:
from praisonai.persistence import PersistenceOrchestrator
from praisonai.persistence.hooks import create_persistent_session
from praisonaiagents import Agent
orchestrator = PersistenceOrchestrator(...)
with create_persistent_session(orchestrator, session_id="my-session") as session:
agent = Agent(name="Assistant", role="Helper")
# Messages are persisted within the session context
session.persist_message("user", "Hello!")
response = agent.chat("Hello!")
session.persist_message("assistant", response)
"""
class PersistentSession:
def __init__(self, orchestrator, session_id, user_id):
self.orchestrator = orchestrator
self.session_id = session_id or f"session-{uuid.uuid4().hex[:8]}"
self.user_id = user_id or "default"
self._started = False
def __enter__(self):
self.start()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.end()
return False
def start(self, agent=None):
"""Start the session."""
if not self._started:
class MockAgent:
name = "session-agent"
self.orchestrator.on_agent_start(
agent or MockAgent(),
session_id=self.session_id,
user_id=self.user_id
)
self._started = True
def end(self, agent=None):
"""End the session."""
if self._started:
class MockAgent:
name = "session-agent"
self.orchestrator.on_agent_end(
agent or MockAgent(),
self.session_id
)
def persist_message(self, role: str, content: str):
"""Persist a message to the session."""
self.orchestrator.on_message(self.session_id, role, content)
def get_messages(self, limit: Optional[int] = None):
"""Get messages from the session."""
return self.orchestrator.get_messages(self.session_id, limit=limit)
def get_session(self):
"""Get the session object."""
return self.orchestrator.get_session(self.session_id)
return PersistentSession(orchestrator, session_id, user_id)
@@ -0,0 +1,26 @@
"""
KnowledgeStore implementations for vector embeddings and semantic search.
Supported backends:
- Qdrant (qdrant)
- Pinecone (pinecone)
- ChromaDB (chroma)
- Weaviate (weaviate)
- LanceDB (lancedb)
- Milvus (milvus)
- PGVector (pgvector)
- Redis Vector (redis)
- Cassandra Vector (cassandra)
- ClickHouse Vector (clickhouse)
"""
__all__ = [
"KnowledgeStore",
"KnowledgeDocument",
]
def __getattr__(name: str):
if name in ("KnowledgeStore", "KnowledgeDocument"):
from .base import KnowledgeStore, KnowledgeDocument
return locals()[name]
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -0,0 +1,144 @@
"""
Base interfaces for KnowledgeStore.
KnowledgeStore handles vector embeddings and semantic search for RAG.
"""
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional
import time
import uuid
import hashlib
@dataclass
class KnowledgeDocument:
"""A document with vector embedding for semantic search."""
id: str = field(default_factory=lambda: str(uuid.uuid4()))
content: str = ""
embedding: Optional[List[float]] = None
metadata: Optional[Dict[str, Any]] = None
content_hash: Optional[str] = None
created_at: float = field(default_factory=time.time)
def __post_init__(self):
if self.content and not self.content_hash:
self.content_hash = hashlib.sha256(self.content.encode()).hexdigest()[:16]
def to_dict(self) -> Dict[str, Any]:
return {
"id": self.id,
"content": self.content,
"embedding": self.embedding,
"metadata": self.metadata,
"content_hash": self.content_hash,
"created_at": self.created_at,
}
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "KnowledgeDocument":
return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__})
class KnowledgeStore(ABC):
"""
Abstract base class for vector/knowledge persistence.
Implementations handle document storage and semantic search for different backends:
- Qdrant, Pinecone, ChromaDB, Weaviate (dedicated vector DBs)
- LanceDB, Milvus (embedded/distributed vector DBs)
- PGVector, Redis, Cassandra, ClickHouse (vector extensions)
"""
@abstractmethod
def create_collection(
self,
name: str,
dimension: int,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create a new collection/index for vectors."""
raise NotImplementedError
@abstractmethod
def delete_collection(self, name: str) -> bool:
"""Delete a collection and all its documents."""
raise NotImplementedError
@abstractmethod
def collection_exists(self, name: str) -> bool:
"""Check if a collection exists."""
raise NotImplementedError
@abstractmethod
def list_collections(self) -> List[str]:
"""List all collections."""
raise NotImplementedError
@abstractmethod
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents into a collection. Returns list of IDs."""
raise NotImplementedError
@abstractmethod
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert or update documents. Returns list of IDs."""
raise NotImplementedError
@abstractmethod
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents by embedding."""
raise NotImplementedError
@abstractmethod
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
raise NotImplementedError
@abstractmethod
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents by IDs or filters. Returns count deleted."""
raise NotImplementedError
@abstractmethod
def count(self, collection: str) -> int:
"""Count documents in a collection."""
raise NotImplementedError
@abstractmethod
def close(self) -> None:
"""Close the store and release resources."""
raise NotImplementedError
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
return False
@@ -0,0 +1,232 @@
"""
Cassandra Vector implementation of KnowledgeStore.
Requires: cassandra-driver
Install: pip install cassandra-driver
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class CassandraKnowledgeStore(KnowledgeStore):
"""
Cassandra-based knowledge store using vector search (SAI).
Requires Cassandra 5.0+ or DataStax Astra with vector search.
Example:
store = CassandraKnowledgeStore(
hosts=["localhost"],
keyspace="praisonai"
)
"""
def __init__(
self,
hosts: Optional[List[str]] = None,
keyspace: str = "praisonai",
username: Optional[str] = None,
password: Optional[str] = None,
port: int = 9042,
secure_connect_bundle: Optional[str] = None,
):
try:
from cassandra.cluster import Cluster
from cassandra.auth import PlainTextAuthProvider
except ImportError:
raise ImportError(
"cassandra-driver is required for Cassandra support. "
"Install with: pip install cassandra-driver"
)
self._Cluster = Cluster
self.keyspace = keyspace
auth = None
if username and password:
auth = PlainTextAuthProvider(username=username, password=password)
if secure_connect_bundle:
# Astra DB connection
from cassandra.cluster import Cluster
cloud_config = {"secure_connect_bundle": secure_connect_bundle}
self._cluster = Cluster(cloud=cloud_config, auth_provider=auth)
else:
hosts = hosts or ["localhost"]
self._cluster = Cluster(hosts, port=port, auth_provider=auth)
self._session = self._cluster.connect()
# Create keyspace if not exists
self._session.execute(f"""
CREATE KEYSPACE IF NOT EXISTS {keyspace}
WITH replication = {{'class': 'SimpleStrategy', 'replication_factor': 1}}
""")
self._session.set_keyspace(keyspace)
logger.info(f"Connected to Cassandra keyspace: {keyspace}")
def create_collection(
self,
name: str,
dimension: int,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create a new table with vector column."""
similarity_map = {"cosine": "COSINE", "euclidean": "EUCLIDEAN", "dot": "DOT_PRODUCT"}
self._session.execute(f"""
CREATE TABLE IF NOT EXISTS {name} (
id text PRIMARY KEY,
content text,
content_hash text,
created_at double,
embedding vector<float, {dimension}>
)
""")
# Create SAI index for vector search
self._session.execute(f"""
CREATE CUSTOM INDEX IF NOT EXISTS {name}_embedding_idx ON {name} (embedding)
USING 'StorageAttachedIndex'
WITH OPTIONS = {{'similarity_function': '{similarity_map.get(distance, "COSINE")}'}}
""")
logger.info(f"Created Cassandra table: {name}")
def delete_collection(self, name: str) -> bool:
"""Delete a table."""
try:
self._session.execute(f"DROP TABLE IF EXISTS {name}")
return True
except Exception as e:
logger.warning(f"Failed to delete table {name}: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if a table exists."""
result = self._session.execute("""
SELECT table_name FROM system_schema.tables
WHERE keyspace_name = %s AND table_name = %s
""", (self.keyspace, name))
return len(list(result)) > 0
def list_collections(self) -> List[str]:
"""List all tables."""
result = self._session.execute("""
SELECT table_name FROM system_schema.tables WHERE keyspace_name = %s
""", (self.keyspace,))
return [row.table_name for row in result]
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
ids = []
for doc in documents:
if doc.embedding is None:
raise ValueError(f"Document {doc.id} has no embedding")
self._session.execute(f"""
INSERT INTO {collection} (id, content, content_hash, created_at, embedding)
VALUES (%s, %s, %s, %s, %s)
""", (doc.id, doc.content, doc.content_hash or "", doc.created_at, doc.embedding))
ids.append(doc.id)
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert or update documents."""
return self.insert(collection, documents) # Cassandra INSERT is upsert
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents using ANN."""
result = self._session.execute(f"""
SELECT id, content, content_hash, created_at, similarity_cosine(embedding, %s) as score
FROM {collection}
ORDER BY embedding ANN OF %s
LIMIT %s
""", (query_embedding, query_embedding, limit))
documents = []
for row in result:
if score_threshold and row.score < score_threshold:
continue
doc = KnowledgeDocument(
id=row.id,
content=row.content,
embedding=None,
metadata={},
content_hash=row.content_hash,
created_at=row.created_at,
)
documents.append(doc)
return documents
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
documents = []
for doc_id in ids:
result = self._session.execute(f"""
SELECT id, content, content_hash, created_at, embedding
FROM {collection} WHERE id = %s
""", (doc_id,))
for row in result:
doc = KnowledgeDocument(
id=row.id,
content=row.content,
embedding=list(row.embedding) if row.embedding else None,
metadata={},
content_hash=row.content_hash,
created_at=row.created_at,
)
documents.append(doc)
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
if ids:
for doc_id in ids:
self._session.execute(f"DELETE FROM {collection} WHERE id = %s", (doc_id,))
return len(ids)
return 0
def count(self, collection: str) -> int:
"""Count documents."""
result = self._session.execute(f"SELECT COUNT(*) FROM {collection}")
return result.one()[0]
def close(self) -> None:
"""Close the store."""
if self._cluster:
self._cluster.shutdown()
@@ -0,0 +1,295 @@
"""
ChromaDB implementation of KnowledgeStore.
Requires: chromadb
Install: pip install chromadb
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class ChromaKnowledgeStore(KnowledgeStore):
"""
ChromaDB-based knowledge store for vector search.
Zero-config embedded vector database.
Example:
store = ChromaKnowledgeStore(
path="./chroma_db"
)
"""
def __init__(
self,
path: Optional[str] = None,
host: Optional[str] = None,
port: int = 8000,
persistent: bool = True,
):
"""
Initialize ChromaDB knowledge store.
Args:
path: Path for persistent storage (embedded mode)
host: Chroma server host (client mode)
port: Chroma server port
persistent: Use persistent storage
"""
try:
import chromadb
from chromadb.config import Settings
except ImportError:
raise ImportError(
"chromadb is required for ChromaDB support. "
"Install with: pip install chromadb"
)
self._chromadb = chromadb
if host:
# Client mode
self._client = chromadb.HttpClient(host=host, port=port)
elif path and persistent:
# Persistent embedded mode
self._client = chromadb.PersistentClient(
path=path,
settings=Settings(anonymized_telemetry=False)
)
else:
# In-memory mode
self._client = chromadb.Client(
settings=Settings(anonymized_telemetry=False)
)
self._collections: Dict[str, Any] = {}
logger.info(f"ChromaDB initialized (path={path}, host={host})")
def _get_collection(self, name: str):
"""Get or cache a collection."""
if name not in self._collections:
self._collections[name] = self._client.get_collection(name)
return self._collections[name]
def create_collection(
self,
name: str,
dimension: int,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create a new collection."""
distance_map = {
"cosine": "cosine",
"euclidean": "l2",
"dot": "ip",
}
self._client.create_collection(
name=name,
metadata={"hnsw:space": distance_map.get(distance, "cosine"), **(metadata or {})}
)
logger.info(f"Created ChromaDB collection: {name}")
def delete_collection(self, name: str) -> bool:
"""Delete a collection."""
try:
self._client.delete_collection(name)
if name in self._collections:
del self._collections[name]
return True
except Exception as e:
logger.warning(f"Failed to delete collection {name}: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if a collection exists."""
try:
self._client.get_collection(name)
return True
except Exception:
return False
def list_collections(self) -> List[str]:
"""List all collections."""
return [c.name for c in self._client.list_collections()]
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents into a collection."""
col = self._get_collection(collection)
ids = []
embeddings = []
contents = []
metadatas = []
for doc in documents:
if doc.embedding is None:
raise ValueError(f"Document {doc.id} has no embedding")
ids.append(doc.id)
embeddings.append(doc.embedding)
contents.append(doc.content)
metadatas.append({
"content_hash": doc.content_hash or "",
"created_at": doc.created_at,
**(doc.metadata or {})
})
col.add(
ids=ids,
embeddings=embeddings,
documents=contents,
metadatas=metadatas,
)
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert or update documents."""
col = self._get_collection(collection)
ids = []
embeddings = []
contents = []
metadatas = []
for doc in documents:
if doc.embedding is None:
raise ValueError(f"Document {doc.id} has no embedding")
ids.append(doc.id)
embeddings.append(doc.embedding)
contents.append(doc.content)
metadatas.append({
"content_hash": doc.content_hash or "",
"created_at": doc.created_at,
**(doc.metadata or {})
})
col.upsert(
ids=ids,
embeddings=embeddings,
documents=contents,
metadatas=metadatas,
)
return ids
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents."""
col = self._get_collection(collection)
where = None
if filters:
where = {k: v for k, v in filters.items()}
results = col.query(
query_embeddings=[query_embedding],
n_results=limit,
where=where,
include=["documents", "metadatas", "distances"],
)
documents = []
if results["ids"] and results["ids"][0]:
for i, doc_id in enumerate(results["ids"][0]):
distance = results["distances"][0][i] if results.get("distances") else None
# Convert distance to similarity score (for cosine: 1 - distance)
if score_threshold is not None and distance is not None:
score = 1 - distance
if score < score_threshold:
continue
metadata = results["metadatas"][0][i] if results.get("metadatas") else {}
content_hash = metadata.pop("content_hash", None)
created_at = metadata.pop("created_at", 0)
doc = KnowledgeDocument(
id=doc_id,
content=results["documents"][0][i] if results.get("documents") else "",
embedding=None,
metadata=metadata,
content_hash=content_hash,
created_at=created_at,
)
documents.append(doc)
return documents
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
col = self._get_collection(collection)
results = col.get(
ids=ids,
include=["documents", "metadatas", "embeddings"],
)
documents = []
if results["ids"]:
for i, doc_id in enumerate(results["ids"]):
metadata = results["metadatas"][i] if results.get("metadatas") else {}
content_hash = metadata.pop("content_hash", None)
created_at = metadata.pop("created_at", 0)
doc = KnowledgeDocument(
id=doc_id,
content=results["documents"][i] if results.get("documents") else "",
embedding=results["embeddings"][i] if results.get("embeddings") else None,
metadata=metadata,
content_hash=content_hash,
created_at=created_at,
)
documents.append(doc)
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
col = self._get_collection(collection)
if ids:
col.delete(ids=ids)
return len(ids)
elif filters:
col.delete(where=filters)
return -1 # Unknown count
return 0
def count(self, collection: str) -> int:
"""Count documents in a collection."""
col = self._get_collection(collection)
return col.count()
def close(self) -> None:
"""Close the store."""
self._collections.clear()
@@ -0,0 +1,242 @@
"""
ClickHouse Vector implementation of KnowledgeStore.
Requires: clickhouse-connect
Install: pip install clickhouse-connect
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class ClickHouseKnowledgeStore(KnowledgeStore):
"""
ClickHouse-based knowledge store using vector search.
Example:
store = ClickHouseKnowledgeStore(
host="localhost",
port=8123
)
"""
def __init__(
self,
host: str = "localhost",
port: int = 8123,
username: str = "default",
password: str = "",
database: str = "praisonai",
secure: bool = False,
):
try:
import clickhouse_connect
except ImportError:
raise ImportError(
"clickhouse-connect is required for ClickHouse support. "
"Install with: pip install clickhouse-connect"
)
self._client = clickhouse_connect.get_client(
host=host,
port=port,
username=username,
password=password,
database=database,
secure=secure,
)
self.database = database
# Create database if not exists
self._client.command(f"CREATE DATABASE IF NOT EXISTS {database}")
logger.info(f"Connected to ClickHouse database: {database}")
def _table_name(self, collection: str) -> str:
return f"{self.database}.praison_vec_{collection}"
def create_collection(
self,
name: str,
dimension: int,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create a new table with vector column."""
table = self._table_name(name)
self._client.command(f"""
CREATE TABLE IF NOT EXISTS {table} (
id String,
content String,
content_hash String,
created_at Float64,
embedding Array(Float32)
) ENGINE = MergeTree()
ORDER BY id
""")
logger.info(f"Created ClickHouse table: {table}")
def delete_collection(self, name: str) -> bool:
"""Delete a table."""
table = self._table_name(name)
try:
self._client.command(f"DROP TABLE IF EXISTS {table}")
return True
except Exception as e:
logger.warning(f"Failed to delete table {table}: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if a table exists."""
table = self._table_name(name)
result = self._client.query(f"EXISTS TABLE {table}")
return result.result_rows[0][0] == 1
def list_collections(self) -> List[str]:
"""List all collections."""
result = self._client.query(f"""
SELECT name FROM system.tables
WHERE database = '{self.database}' AND name LIKE 'praison_vec_%'
""")
return [row[0].replace("praison_vec_", "") for row in result.result_rows]
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
table = self._table_name(collection)
data = []
ids = []
for doc in documents:
if doc.embedding is None:
raise ValueError(f"Document {doc.id} has no embedding")
data.append([
doc.id,
doc.content,
doc.content_hash or "",
doc.created_at,
doc.embedding,
])
ids.append(doc.id)
self._client.insert(
table,
data,
column_names=["id", "content", "content_hash", "created_at", "embedding"]
)
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert or update documents."""
# ClickHouse doesn't have native upsert for MergeTree
# Delete then insert
ids = [doc.id for doc in documents]
self.delete(collection, ids)
return self.insert(collection, documents)
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents using cosine distance."""
table = self._table_name(collection)
# Convert embedding to string for query
embedding_str = "[" + ",".join(map(str, query_embedding)) + "]"
query = f"""
SELECT
id, content, content_hash, created_at,
1 - cosineDistance(embedding, {embedding_str}) as score
FROM {table}
"""
if score_threshold:
query += f" WHERE 1 - cosineDistance(embedding, {embedding_str}) >= {score_threshold}"
query += f" ORDER BY score DESC LIMIT {limit}"
result = self._client.query(query)
documents = []
for row in result.result_rows:
doc = KnowledgeDocument(
id=row[0],
content=row[1],
embedding=None,
metadata={},
content_hash=row[2],
created_at=row[3],
)
documents.append(doc)
return documents
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
table = self._table_name(collection)
id_list = ", ".join([f"'{i}'" for i in ids])
result = self._client.query(f"""
SELECT id, content, content_hash, created_at, embedding
FROM {table} WHERE id IN ({id_list})
""")
documents = []
for row in result.result_rows:
doc = KnowledgeDocument(
id=row[0],
content=row[1],
embedding=list(row[4]) if row[4] else None,
metadata={},
content_hash=row[2],
created_at=row[3],
)
documents.append(doc)
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
table = self._table_name(collection)
if ids:
id_list = ", ".join([f"'{i}'" for i in ids])
self._client.command(f"ALTER TABLE {table} DELETE WHERE id IN ({id_list})")
return len(ids)
return 0
def count(self, collection: str) -> int:
"""Count documents."""
table = self._table_name(collection)
result = self._client.query(f"SELECT COUNT(*) FROM {table}")
return result.result_rows[0][0]
def close(self) -> None:
"""Close the store."""
if self._client:
self._client.close()
@@ -0,0 +1,438 @@
"""
Azure Cosmos DB Vector implementation of KnowledgeStore.
Supports both MongoDB API and vCore modes.
Requires: pymongo (for MongoDB API) or azure-cosmos (for SQL API)
Install: pip install pymongo # or pip install azure-cosmos
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class CosmosDBVectorKnowledgeStore(KnowledgeStore):
"""
Azure Cosmos DB vector store for knowledge/RAG.
Supports MongoDB API with vector search capabilities.
Example:
store = CosmosDBVectorKnowledgeStore(
connection_string="mongodb+srv://...",
database="praisonai",
collection="vectors"
)
"""
def __init__(
self,
connection_string: Optional[str] = None,
database: str = "praisonai",
collection: str = "vectors",
index_name: str = "vector_index",
embedding_field: str = "embedding",
text_field: str = "content",
embedding_dim: int = 1536,
api_mode: str = "mongodb", # "mongodb" or "sql"
):
"""
Initialize Cosmos DB Vector store.
Args:
connection_string: Azure Cosmos DB connection string
database: Database name
collection: Collection name
index_name: Vector search index name
embedding_field: Field name for embeddings
text_field: Field name for text content
embedding_dim: Embedding dimension
api_mode: API mode ("mongodb" or "sql")
"""
self.connection_string = connection_string
self.database_name = database
self.collection_name = collection
self.index_name = index_name
self.embedding_field = embedding_field
self.text_field = text_field
self.embedding_dim = embedding_dim
self.api_mode = api_mode
self._client = None
self._db = None
self._collection = None
self._initialized = False
def _init_client(self):
"""Initialize Cosmos DB client lazily."""
if self._initialized:
return
if not self.connection_string:
import os
self.connection_string = os.getenv("COSMOS_CONNECTION_STRING")
if not self.connection_string:
raise ValueError(
"connection_string required. Set COSMOS_CONNECTION_STRING env var."
)
if self.api_mode == "mongodb":
self._init_mongodb_client()
else:
self._init_sql_client()
self._initialized = True
def _init_mongodb_client(self):
"""Initialize MongoDB API client."""
try:
from pymongo import MongoClient
except ImportError:
raise ImportError(
"pymongo is required for Cosmos DB MongoDB API. "
"Install with: pip install pymongo"
)
self._client = MongoClient(self.connection_string)
self._db = self._client[self.database_name]
self._collection = self._db[self.collection_name]
def _init_sql_client(self):
"""Initialize SQL API client."""
try:
from azure.cosmos import CosmosClient
except ImportError:
raise ImportError(
"azure-cosmos is required for Cosmos DB SQL API. "
"Install with: pip install azure-cosmos"
)
self._client = CosmosClient.from_connection_string(self.connection_string)
self._db = self._client.get_database_client(self.database_name)
self._collection = self._db.get_container_client(self.collection_name)
def create_collection(
self,
name: str,
dimension: int = 1536,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create collection with vector index."""
self._init_client()
if self.api_mode == "mongodb":
# Create vector search index via MongoDB command
try:
self._db.command({
"createIndexes": name,
"indexes": [{
"name": self.index_name,
"key": {self.embedding_field: "cosmosSearch"},
"cosmosSearchOptions": {
"kind": "vector-ivf",
"numLists": 100,
"similarity": distance,
"dimensions": dimension
}
}]
})
except Exception as e:
logger.warning(f"Index creation may require manual setup: {e}")
def delete_collection(self, name: str) -> bool:
"""Delete collection."""
self._init_client()
try:
if self.api_mode == "mongodb":
self._db.drop_collection(name)
else:
self._db.delete_container(name)
return True
except Exception as e:
logger.error(f"Failed to delete collection: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if collection exists."""
self._init_client()
if self.api_mode == "mongodb":
return name in self._db.list_collection_names()
return True # SQL API containers are pre-created
def list_collections(self) -> List[str]:
"""List collections."""
self._init_client()
if self.api_mode == "mongodb":
return self._db.list_collection_names()
return [c["id"] for c in self._db.list_containers()]
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
self._init_client()
coll = self._db[collection] if self.api_mode == "mongodb" else self._db.get_container_client(collection)
ids = []
for doc in documents:
item = {
"_id" if self.api_mode == "mongodb" else "id": doc.id,
self.text_field: doc.content,
self.embedding_field: doc.embedding,
"metadata": doc.metadata or {},
"content_hash": doc.content_hash,
"created_at": doc.created_at,
}
try:
if self.api_mode == "mongodb":
coll.insert_one(item)
else:
coll.create_item(item)
ids.append(doc.id)
except Exception as e:
logger.warning(f"Failed to insert {doc.id}: {e}")
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Upsert documents."""
self._init_client()
coll = self._db[collection] if self.api_mode == "mongodb" else self._db.get_container_client(collection)
ids = []
for doc in documents:
item = {
"_id" if self.api_mode == "mongodb" else "id": doc.id,
self.text_field: doc.content,
self.embedding_field: doc.embedding,
"metadata": doc.metadata or {},
"content_hash": doc.content_hash,
"created_at": doc.created_at,
}
try:
if self.api_mode == "mongodb":
coll.replace_one({"_id": doc.id}, item, upsert=True)
else:
coll.upsert_item(item)
ids.append(doc.id)
except Exception as e:
logger.warning(f"Failed to upsert {doc.id}: {e}")
return ids
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents using vector search."""
self._init_client()
if self.api_mode == "mongodb":
return self._search_mongodb(collection, query_embedding, limit, filters, score_threshold)
else:
return self._search_sql(collection, query_embedding, limit, filters, score_threshold)
def _search_mongodb(
self,
collection: str,
query_embedding: List[float],
limit: int,
filters: Optional[Dict[str, Any]],
score_threshold: Optional[float]
) -> List[KnowledgeDocument]:
"""MongoDB API vector search."""
coll = self._db[collection]
pipeline = [
{
"$search": {
"cosmosSearch": {
"vector": query_embedding,
"path": self.embedding_field,
"k": limit
},
"returnStoredSource": True
}
},
{
"$project": {
"similarityScore": {"$meta": "searchScore"},
self.text_field: 1,
"metadata": 1,
"content_hash": 1,
"created_at": 1
}
}
]
if filters:
pipeline.insert(1, {"$match": filters})
try:
results = list(coll.aggregate(pipeline))
documents = []
for doc in results:
score = doc.get("similarityScore", 0)
if score_threshold and score < score_threshold:
continue
documents.append(KnowledgeDocument(
id=str(doc.get("_id", "")),
content=doc.get(self.text_field, ""),
metadata={**(doc.get("metadata") or {}), "score": score},
content_hash=doc.get("content_hash"),
created_at=doc.get("created_at", 0)
))
return documents
except Exception as e:
logger.error(f"Vector search failed: {e}")
return []
def _search_sql(
self,
collection: str,
query_embedding: List[float],
limit: int,
filters: Optional[Dict[str, Any]],
score_threshold: Optional[float]
) -> List[KnowledgeDocument]:
"""SQL API vector search (requires DiskANN index)."""
coll = self._db.get_container_client(collection)
# SQL API uses VectorDistance function
query = f"""
SELECT TOP {limit} c.id, c.{self.text_field}, c.metadata, c.content_hash, c.created_at,
VectorDistance(c.{self.embedding_field}, @embedding) AS score
FROM c
ORDER BY VectorDistance(c.{self.embedding_field}, @embedding)
"""
try:
results = list(coll.query_items(
query=query,
parameters=[{"name": "@embedding", "value": query_embedding}],
enable_cross_partition_query=True
))
documents = []
for doc in results:
score = doc.get("score", 0)
if score_threshold and score < score_threshold:
continue
documents.append(KnowledgeDocument(
id=doc.get("id", ""),
content=doc.get(self.text_field, ""),
metadata={**(doc.get("metadata") or {}), "score": score},
content_hash=doc.get("content_hash"),
created_at=doc.get("created_at", 0)
))
return documents
except Exception as e:
logger.error(f"Vector search failed: {e}")
return []
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
self._init_client()
coll = self._db[collection] if self.api_mode == "mongodb" else self._db.get_container_client(collection)
documents = []
for doc_id in ids:
try:
if self.api_mode == "mongodb":
doc = coll.find_one({"_id": doc_id})
else:
doc = coll.read_item(item=doc_id, partition_key=doc_id)
if doc:
documents.append(KnowledgeDocument(
id=str(doc.get("_id" if self.api_mode == "mongodb" else "id", "")),
content=doc.get(self.text_field, ""),
embedding=doc.get(self.embedding_field),
metadata=doc.get("metadata"),
content_hash=doc.get("content_hash"),
created_at=doc.get("created_at", 0)
))
except Exception as e:
logger.warning(f"Failed to get {doc_id}: {e}")
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
self._init_client()
coll = self._db[collection] if self.api_mode == "mongodb" else self._db.get_container_client(collection)
if ids:
count = 0
for doc_id in ids:
try:
if self.api_mode == "mongodb":
coll.delete_one({"_id": doc_id})
else:
coll.delete_item(item=doc_id, partition_key=doc_id)
count += 1
except Exception as e:
logger.warning(f"Failed to delete {doc_id}: {e}")
return count
return 0
def count(self, collection: str) -> int:
"""Count documents."""
self._init_client()
if self.api_mode == "mongodb":
return self._db[collection].count_documents({})
else:
# SQL API count
coll = self._db.get_container_client(collection)
result = list(coll.query_items(
query="SELECT VALUE COUNT(1) FROM c",
enable_cross_partition_query=True
))
return result[0] if result else 0
def close(self) -> None:
"""Close the connection."""
if self._client:
if self.api_mode == "mongodb":
self._client.close()
self._client = None
self._db = None
self._collection = None
self._initialized = False
@@ -0,0 +1,286 @@
"""
Couchbase implementation of KnowledgeStore.
Requires: couchbase
Install: pip install couchbase
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class CouchbaseKnowledgeStore(KnowledgeStore):
"""
Couchbase vector store for knowledge/RAG.
Uses Couchbase's vector search capabilities.
Example:
store = CouchbaseKnowledgeStore(
connection_string="couchbase://localhost",
username="admin",
password="password",
bucket_name="praisonai"
)
"""
def __init__(
self,
connection_string: str = "couchbase://localhost",
username: str = "Administrator",
password: str = "",
bucket_name: str = "praisonai",
scope_name: str = "_default",
collection_name: str = "vectors",
index_name: str = "vector_index",
vector_field: str = "embedding",
text_field: str = "content",
embedding_dim: int = 1536,
):
"""
Initialize Couchbase knowledge store.
Args:
connection_string: Couchbase connection string
username: Couchbase username
password: Couchbase password
bucket_name: Bucket name
scope_name: Scope name
collection_name: Collection name
index_name: Vector search index name
vector_field: Field name for embeddings
text_field: Field name for text content
embedding_dim: Embedding dimension
"""
self.connection_string = connection_string
self.username = username
self.password = password
self.bucket_name = bucket_name
self.scope_name = scope_name
self.collection_name = collection_name
self.index_name = index_name
self.vector_field = vector_field
self.text_field = text_field
self.embedding_dim = embedding_dim
self._cluster = None
self._bucket = None
self._collection = None
self._initialized = False
def _init_client(self):
"""Initialize Couchbase client lazily."""
if self._initialized:
return
try:
from couchbase.cluster import Cluster
from couchbase.options import ClusterOptions
from couchbase.auth import PasswordAuthenticator
from datetime import timedelta
except ImportError:
raise ImportError(
"couchbase is required for Couchbase support. "
"Install with: pip install couchbase"
)
auth = PasswordAuthenticator(self.username, self.password)
options = ClusterOptions(auth)
self._cluster = Cluster(self.connection_string, options)
self._cluster.wait_until_ready(timedelta(seconds=10))
self._bucket = self._cluster.bucket(self.bucket_name)
scope = self._bucket.scope(self.scope_name)
self._collection = scope.collection(self.collection_name)
self._initialized = True
def insert(
self,
collection_name: str,
documents: List[KnowledgeDocument],
embeddings: Optional[List[List[float]]] = None,
) -> List[str]:
"""Insert documents into the store."""
self._init_client()
ids = []
for i, doc in enumerate(documents):
doc_id = doc.id or f"doc_{i}_{hash(doc.content)}"
data = {
self.text_field: doc.content,
"metadata": doc.metadata or {},
}
if embeddings and i < len(embeddings):
data[self.vector_field] = embeddings[i]
self._collection.upsert(doc_id, data)
ids.append(doc_id)
return ids
def upsert(
self,
collection_name: str,
documents: List[KnowledgeDocument],
embeddings: Optional[List[List[float]]] = None,
) -> List[str]:
"""Upsert documents (same as insert for Couchbase)."""
return self.insert(collection_name, documents, embeddings)
def search(
self,
collection_name: str,
query_embedding: List[float],
limit: int = 10,
filters: Optional[Dict[str, Any]] = None,
) -> List[KnowledgeDocument]:
"""Search for similar documents using vector search."""
self._init_client()
try:
from couchbase.vector_search import VectorQuery, VectorSearch
except ImportError:
logger.warning("Vector search not available in this Couchbase version")
return []
vector_query = VectorQuery(
self.vector_field,
query_embedding,
num_candidates=limit * 2
)
search_req = VectorSearch.from_vector_query(vector_query)
try:
scope = self._bucket.scope(self.scope_name)
result = scope.search(
self.index_name,
search_req,
limit=limit
)
documents = []
for row in result.rows():
doc_id = row.id
try:
doc_data = self._collection.get(doc_id).content_as[dict]
documents.append(KnowledgeDocument(
id=doc_id,
content=doc_data.get(self.text_field, ""),
metadata=doc_data.get("metadata", {}),
score=row.score if hasattr(row, 'score') else None
))
except Exception as e:
logger.warning(f"Failed to fetch document {doc_id}: {e}")
return documents
except Exception as e:
logger.error(f"Vector search failed: {e}")
return []
def delete(
self,
collection_name: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None,
) -> int:
"""Delete documents by ID."""
self._init_client()
if not ids:
return 0
count = 0
for doc_id in ids:
try:
self._collection.remove(doc_id)
count += 1
except Exception as e:
logger.warning(f"Failed to delete {doc_id}: {e}")
return count
def get(
self,
collection_name: str,
ids: List[str],
) -> List[KnowledgeDocument]:
"""Get documents by ID."""
self._init_client()
documents = []
for doc_id in ids:
try:
doc_data = self._collection.get(doc_id).content_as[dict]
documents.append(KnowledgeDocument(
id=doc_id,
content=doc_data.get(self.text_field, ""),
metadata=doc_data.get("metadata", {})
))
except Exception as e:
logger.warning(f"Failed to get {doc_id}: {e}")
return documents
def count(self, collection_name: str) -> int:
"""Count documents in collection."""
self._init_client()
try:
query = f"SELECT COUNT(*) as count FROM `{self.bucket_name}`.`{self.scope_name}`.`{self.collection_name}`"
result = self._cluster.query(query)
for row in result:
return row.get("count", 0)
except Exception as e:
logger.error(f"Count query failed: {e}")
return 0
def create_collection(self, collection_name: str, **kwargs) -> bool:
"""Create collection (Couchbase collections are pre-created)."""
self._init_client()
return True
def delete_collection(self, collection_name: str) -> bool:
"""Delete all documents in collection."""
self._init_client()
try:
query = f"DELETE FROM `{self.bucket_name}`.`{self.scope_name}`.`{self.collection_name}`"
self._cluster.query(query)
return True
except Exception as e:
logger.error(f"Delete collection failed: {e}")
return False
def list_collections(self) -> List[str]:
"""List available collections."""
self._init_client()
try:
collections = self._bucket.collections().get_all_scopes()
result = []
for scope in collections:
for coll in scope.collections:
result.append(f"{scope.name}.{coll.name}")
return result
except Exception as e:
logger.error(f"List collections failed: {e}")
return []
def close(self) -> None:
"""Close the connection."""
if self._cluster:
self._cluster.close()
self._cluster = None
self._bucket = None
self._collection = None
self._initialized = False
@@ -0,0 +1,216 @@
"""
LanceDB implementation of KnowledgeStore.
Requires: lancedb
Install: pip install lancedb
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class LanceDBKnowledgeStore(KnowledgeStore):
"""
LanceDB-based knowledge store for vector search.
Embedded serverless vector database.
Example:
store = LanceDBKnowledgeStore(
path="./lancedb"
)
"""
def __init__(
self,
path: str = "./lancedb",
uri: Optional[str] = None,
):
try:
import lancedb
except ImportError:
raise ImportError(
"lancedb is required for LanceDB support. "
"Install with: pip install lancedb"
)
self._lancedb = lancedb
self._db = lancedb.connect(uri or path)
self._tables: Dict[str, Any] = {}
logger.info(f"Connected to LanceDB at {uri or path}")
def create_collection(
self,
name: str,
dimension: int,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create a new table."""
import pyarrow as pa
schema = pa.schema([
pa.field("id", pa.string()),
pa.field("content", pa.string()),
pa.field("content_hash", pa.string()),
pa.field("created_at", pa.float64()),
pa.field("vector", pa.list_(pa.float32(), dimension)),
])
self._db.create_table(name, schema=schema)
logger.info(f"Created LanceDB table: {name}")
def delete_collection(self, name: str) -> bool:
"""Delete a table."""
try:
self._db.drop_table(name)
if name in self._tables:
del self._tables[name]
return True
except Exception as e:
logger.warning(f"Failed to delete table {name}: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if a table exists."""
return name in self._db.table_names()
def list_collections(self) -> List[str]:
"""List all tables."""
return self._db.table_names()
def _get_table(self, name: str):
"""Get or cache a table."""
if name not in self._tables:
self._tables[name] = self._db.open_table(name)
return self._tables[name]
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
table = self._get_table(collection)
data = []
ids = []
for doc in documents:
if doc.embedding is None:
raise ValueError(f"Document {doc.id} has no embedding")
data.append({
"id": doc.id,
"content": doc.content,
"content_hash": doc.content_hash or "",
"created_at": doc.created_at,
"vector": doc.embedding,
})
ids.append(doc.id)
table.add(data)
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert or update documents."""
# LanceDB doesn't have native upsert, delete then insert
ids = [doc.id for doc in documents]
self.delete(collection, ids)
return self.insert(collection, documents)
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents."""
table = self._get_table(collection)
query = table.search(query_embedding).limit(limit)
if filters:
where_clauses = [f"{k} = '{v}'" for k, v in filters.items()]
query = query.where(" AND ".join(where_clauses))
results = query.to_pandas()
documents = []
for _, row in results.iterrows():
if score_threshold and row.get("_distance", 1) > (1 - score_threshold):
continue
doc = KnowledgeDocument(
id=row["id"],
content=row["content"],
embedding=None,
metadata={},
content_hash=row.get("content_hash"),
created_at=row.get("created_at", 0),
)
documents.append(doc)
return documents
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
table = self._get_table(collection)
id_list = ", ".join([f"'{i}'" for i in ids])
results = table.search().where(f"id IN ({id_list})").to_pandas()
documents = []
for _, row in results.iterrows():
doc = KnowledgeDocument(
id=row["id"],
content=row["content"],
embedding=list(row["vector"]) if "vector" in row else None,
metadata={},
content_hash=row.get("content_hash"),
created_at=row.get("created_at", 0),
)
documents.append(doc)
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
table = self._get_table(collection)
if ids:
id_list = ", ".join([f"'{i}'" for i in ids])
table.delete(f"id IN ({id_list})")
return len(ids)
elif filters:
where_clauses = [f"{k} = '{v}'" for k, v in filters.items()]
table.delete(" AND ".join(where_clauses))
return -1
return 0
def count(self, collection: str) -> int:
"""Count documents."""
table = self._get_table(collection)
return table.count_rows()
def close(self) -> None:
"""Close the store."""
self._tables.clear()
@@ -0,0 +1,291 @@
"""
LangChain VectorStore adapter implementation of KnowledgeStore.
Wraps any LangChain VectorStore for use with PraisonAI.
Requires: langchain-core
Install: pip install langchain-core
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class LangChainKnowledgeStore(KnowledgeStore):
"""
LangChain VectorStore adapter for knowledge/RAG.
Wraps any LangChain VectorStore for use with PraisonAI's interface.
Example:
from langchain_chroma import Chroma
lc_store = Chroma(persist_directory="./chroma_db")
store = LangChainKnowledgeStore(vectorstore=lc_store)
"""
def __init__(
self,
vectorstore: Any = None,
vectorstore_cls: Optional[str] = None,
vectorstore_kwargs: Optional[Dict[str, Any]] = None,
):
"""
Initialize LangChain adapter.
Args:
vectorstore: Pre-initialized LangChain VectorStore instance
vectorstore_cls: Class name to import (e.g., "langchain_chroma.Chroma")
vectorstore_kwargs: Kwargs for vectorstore initialization
"""
self._vectorstore = vectorstore
self._vectorstore_cls = vectorstore_cls
self._vectorstore_kwargs = vectorstore_kwargs or {}
self._initialized = vectorstore is not None
def _init_client(self):
"""Initialize vectorstore lazily."""
if self._initialized:
return
if not self._vectorstore_cls:
raise ValueError(
"Either vectorstore or vectorstore_cls must be provided"
)
# Dynamic import of vectorstore class
parts = self._vectorstore_cls.rsplit(".", 1)
if len(parts) == 2:
module_name, class_name = parts
import importlib
module = importlib.import_module(module_name)
cls = getattr(module, class_name)
self._vectorstore = cls(**self._vectorstore_kwargs)
else:
raise ValueError(f"Invalid vectorstore_cls: {self._vectorstore_cls}")
self._initialized = True
def create_collection(
self,
name: str,
dimension: int = 1536,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create collection (handled by underlying vectorstore)."""
self._init_client()
logger.info(f"Collection creation handled by LangChain vectorstore")
def delete_collection(self, name: str) -> bool:
"""Delete collection."""
self._init_client()
try:
if hasattr(self._vectorstore, 'delete_collection'):
self._vectorstore.delete_collection()
return True
logger.warning("Underlying vectorstore doesn't support delete_collection")
return False
except Exception as e:
logger.error(f"Failed to delete collection: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if collection exists."""
self._init_client()
return self._vectorstore is not None
def list_collections(self) -> List[str]:
"""List collections."""
self._init_client()
if hasattr(self._vectorstore, 'list_collections'):
return self._vectorstore.list_collections()
return ["default"]
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
self._init_client()
try:
from langchain_core.documents import Document as LCDocument
except ImportError:
raise ImportError(
"langchain-core is required. Install with: pip install langchain-core"
)
lc_docs = []
ids = []
for doc in documents:
lc_doc = LCDocument(
page_content=doc.content,
metadata={
**(doc.metadata or {}),
"id": doc.id,
"content_hash": doc.content_hash,
"created_at": doc.created_at,
}
)
lc_docs.append(lc_doc)
ids.append(doc.id)
if hasattr(self._vectorstore, 'add_documents'):
self._vectorstore.add_documents(lc_docs, ids=ids)
else:
logger.warning("Vectorstore doesn't support add_documents")
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Upsert documents."""
# Most LangChain vectorstores handle upsert via add_documents
return self.insert(collection, documents)
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents."""
self._init_client()
try:
if hasattr(self._vectorstore, 'similarity_search_by_vector'):
results = self._vectorstore.similarity_search_by_vector(
query_embedding,
k=limit,
filter=filters
)
else:
logger.warning("Vectorstore doesn't support vector search")
return []
documents = []
for lc_doc in results:
metadata = lc_doc.metadata or {}
documents.append(KnowledgeDocument(
id=metadata.get("id", ""),
content=lc_doc.page_content,
metadata=metadata,
content_hash=metadata.get("content_hash"),
created_at=metadata.get("created_at", 0)
))
return documents
except Exception as e:
logger.error(f"Search failed: {e}")
return []
def similarity_search(
self,
query_text: str,
limit: int = 5,
filters: Optional[Dict[str, Any]] = None
) -> List[KnowledgeDocument]:
"""
Text-based similarity search (native LangChain method).
Args:
query_text: Query string
limit: Maximum results
filters: Metadata filters
Returns:
List of matching documents
"""
self._init_client()
try:
results = self._vectorstore.similarity_search(
query_text,
k=limit,
filter=filters
)
documents = []
for lc_doc in results:
metadata = lc_doc.metadata or {}
documents.append(KnowledgeDocument(
id=metadata.get("id", ""),
content=lc_doc.page_content,
metadata=metadata,
content_hash=metadata.get("content_hash"),
created_at=metadata.get("created_at", 0)
))
return documents
except Exception as e:
logger.error(f"Similarity search failed: {e}")
return []
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
self._init_client()
if hasattr(self._vectorstore, 'get'):
try:
results = self._vectorstore.get(ids)
documents = []
for lc_doc in results:
metadata = lc_doc.metadata or {}
documents.append(KnowledgeDocument(
id=metadata.get("id", ""),
content=lc_doc.page_content,
metadata=metadata
))
return documents
except Exception as e:
logger.warning(f"Get by ID failed: {e}")
return []
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
self._init_client()
if ids and hasattr(self._vectorstore, 'delete'):
try:
self._vectorstore.delete(ids)
return len(ids)
except Exception as e:
logger.error(f"Delete failed: {e}")
return 0
def count(self, collection: str) -> int:
"""Count documents."""
self._init_client()
if hasattr(self._vectorstore, '__len__'):
return len(self._vectorstore)
logger.warning("Vectorstore doesn't support counting")
return -1
def close(self) -> None:
"""Close the client."""
self._vectorstore = None
self._initialized = False
@@ -0,0 +1,212 @@
"""
LightRAG adapter implementation of KnowledgeStore.
Requires: lightrag
Install: pip install lightrag
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class LightRAGKnowledgeStore(KnowledgeStore):
"""
LightRAG adapter for knowledge/RAG.
Wraps LightRAG for use with PraisonAI's KnowledgeStore interface.
Example:
store = LightRAGKnowledgeStore(
working_dir="./lightrag_data"
)
"""
def __init__(
self,
working_dir: str = "./lightrag_data",
llm_model: str = "gpt-4o-mini",
embedding_model: str = "text-embedding-3-small",
**kwargs
):
"""
Initialize LightRAG adapter.
Args:
working_dir: Directory for LightRAG data
llm_model: LLM model for graph extraction
embedding_model: Embedding model name
**kwargs: Additional LightRAG options
"""
self.working_dir = working_dir
self.llm_model = llm_model
self.embedding_model = embedding_model
self.kwargs = kwargs
self._rag = None
self._initialized = False
def _init_client(self):
"""Initialize LightRAG client lazily."""
if self._initialized:
return
try:
from lightrag import LightRAG, QueryParam
from lightrag.llm import gpt_4o_mini_complete, openai_embedding
except ImportError:
raise ImportError(
"lightrag is required for LightRAG support. "
"Install with: pip install lightrag"
)
self._QueryParam = QueryParam
self._rag = LightRAG(
working_dir=self.working_dir,
llm_model_func=gpt_4o_mini_complete,
embedding_func=openai_embedding,
**self.kwargs
)
self._initialized = True
def create_collection(
self,
name: str,
dimension: int = 1536,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create collection (LightRAG uses working directory)."""
self._init_client()
logger.info(f"LightRAG uses working_dir: {self.working_dir}")
def delete_collection(self, name: str) -> bool:
"""Delete collection data."""
import shutil
import os
try:
if os.path.exists(self.working_dir):
shutil.rmtree(self.working_dir)
return True
except Exception as e:
logger.error(f"Failed to delete collection: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if working directory exists."""
import os
return os.path.exists(self.working_dir)
def list_collections(self) -> List[str]:
"""List collections (single collection per working_dir)."""
import os
if os.path.exists(self.working_dir):
return ["default"]
return []
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents into LightRAG."""
self._init_client()
ids = []
for doc in documents:
try:
self._rag.insert(doc.content)
ids.append(doc.id)
except Exception as e:
logger.warning(f"Failed to insert document {doc.id}: {e}")
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Upsert documents (same as insert for LightRAG)."""
return self.insert(collection, documents)
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search using LightRAG's query method."""
self._init_client()
# LightRAG uses text query, not embeddings directly
# This is a limitation - we need the original query text
logger.warning(
"LightRAG search requires text query, not embeddings. "
"Use query() method directly for best results."
)
return []
def query(
self,
query_text: str,
mode: str = "hybrid",
limit: int = 5
) -> str:
"""
Query LightRAG with text (native method).
Args:
query_text: The query string
mode: Query mode (naive, local, global, hybrid)
limit: Maximum results
Returns:
Query response as string
"""
self._init_client()
try:
result = self._rag.query(
query_text,
param=self._QueryParam(mode=mode, top_k=limit)
)
return result
except Exception as e:
logger.error(f"Query failed: {e}")
return ""
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs (not supported by LightRAG)."""
logger.warning("LightRAG does not support get by ID")
return []
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents (not directly supported)."""
logger.warning("LightRAG does not support individual document deletion")
return 0
def count(self, collection: str) -> int:
"""Count documents (not directly supported)."""
logger.warning("LightRAG does not support document counting")
return -1
def close(self) -> None:
"""Close the client."""
self._rag = None
self._initialized = False
@@ -0,0 +1,256 @@
"""
LlamaIndex VectorStore adapter implementation of KnowledgeStore.
Wraps any LlamaIndex VectorStore for use with PraisonAI.
Requires: llama-index-core
Install: pip install llama-index-core
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class LlamaIndexKnowledgeStore(KnowledgeStore):
"""
LlamaIndex VectorStore adapter for knowledge/RAG.
Wraps any LlamaIndex VectorStore for use with PraisonAI's interface.
Example:
from llama_index.vector_stores.chroma import ChromaVectorStore
li_store = ChromaVectorStore(chroma_collection=collection)
store = LlamaIndexKnowledgeStore(vector_store=li_store)
"""
def __init__(
self,
vector_store: Any = None,
vector_store_cls: Optional[str] = None,
vector_store_kwargs: Optional[Dict[str, Any]] = None,
):
"""
Initialize LlamaIndex adapter.
Args:
vector_store: Pre-initialized LlamaIndex VectorStore instance
vector_store_cls: Class name to import (e.g., "llama_index.vector_stores.chroma.ChromaVectorStore")
vector_store_kwargs: Kwargs for vector_store initialization
"""
self._vector_store = vector_store
self._vector_store_cls = vector_store_cls
self._vector_store_kwargs = vector_store_kwargs or {}
self._initialized = vector_store is not None
def _init_client(self):
"""Initialize vector store lazily."""
if self._initialized:
return
if not self._vector_store_cls:
raise ValueError(
"Either vector_store or vector_store_cls must be provided"
)
# Dynamic import of vector store class
parts = self._vector_store_cls.rsplit(".", 1)
if len(parts) == 2:
module_name, class_name = parts
import importlib
module = importlib.import_module(module_name)
cls = getattr(module, class_name)
self._vector_store = cls(**self._vector_store_kwargs)
else:
raise ValueError(f"Invalid vector_store_cls: {self._vector_store_cls}")
self._initialized = True
def create_collection(
self,
name: str,
dimension: int = 1536,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create collection (handled by underlying vector store)."""
self._init_client()
logger.info("Collection creation handled by LlamaIndex vector store")
def delete_collection(self, name: str) -> bool:
"""Delete collection."""
self._init_client()
try:
if hasattr(self._vector_store, 'delete'):
self._vector_store.delete(delete_all=True)
return True
logger.warning("Underlying vector store doesn't support delete")
return False
except Exception as e:
logger.error(f"Failed to delete collection: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if collection exists."""
self._init_client()
return self._vector_store is not None
def list_collections(self) -> List[str]:
"""List collections."""
self._init_client()
return ["default"]
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
self._init_client()
try:
from llama_index.core.schema import TextNode
except ImportError:
raise ImportError(
"llama-index-core is required. Install with: pip install llama-index-core"
)
nodes = []
ids = []
for doc in documents:
node = TextNode(
text=doc.content,
id_=doc.id,
metadata={
**(doc.metadata or {}),
"content_hash": doc.content_hash,
"created_at": doc.created_at,
},
embedding=doc.embedding
)
nodes.append(node)
ids.append(doc.id)
if hasattr(self._vector_store, 'add'):
self._vector_store.add(nodes)
else:
logger.warning("Vector store doesn't support add")
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Upsert documents."""
return self.insert(collection, documents)
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents."""
self._init_client()
try:
from llama_index.core.vector_stores import VectorStoreQuery
except ImportError:
raise ImportError(
"llama-index-core is required. Install with: pip install llama-index-core"
)
try:
query = VectorStoreQuery(
query_embedding=query_embedding,
similarity_top_k=limit,
)
result = self._vector_store.query(query)
documents = []
for i, node in enumerate(result.nodes or []):
score = result.similarities[i] if result.similarities else None
if score_threshold and score and score < score_threshold:
continue
metadata = node.metadata or {}
if score:
metadata["score"] = score
documents.append(KnowledgeDocument(
id=node.node_id or node.id_,
content=node.text or node.get_content(),
embedding=node.embedding,
metadata=metadata,
content_hash=metadata.get("content_hash"),
created_at=metadata.get("created_at", 0)
))
return documents
except Exception as e:
logger.error(f"Search failed: {e}")
return []
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
self._init_client()
documents = []
for doc_id in ids:
try:
if hasattr(self._vector_store, 'get'):
node = self._vector_store.get(doc_id)
if node:
metadata = node.metadata or {}
documents.append(KnowledgeDocument(
id=node.node_id or node.id_,
content=node.text or node.get_content(),
embedding=node.embedding,
metadata=metadata
))
except Exception as e:
logger.warning(f"Get by ID failed for {doc_id}: {e}")
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
self._init_client()
if ids and hasattr(self._vector_store, 'delete'):
try:
self._vector_store.delete(ref_doc_id=ids[0] if len(ids) == 1 else None)
return len(ids)
except Exception as e:
logger.error(f"Delete failed: {e}")
return 0
def count(self, collection: str) -> int:
"""Count documents."""
self._init_client()
logger.warning("LlamaIndex vector stores don't typically support counting")
return -1
def close(self) -> None:
"""Close the client."""
self._vector_store = None
self._initialized = False
@@ -0,0 +1,277 @@
"""
Milvus implementation of KnowledgeStore.
Requires: pymilvus
Install: pip install pymilvus
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class MilvusKnowledgeStore(KnowledgeStore):
"""
Milvus-based knowledge store for vector search.
Example:
store = MilvusKnowledgeStore(
host="localhost",
port=19530
)
"""
def __init__(
self,
url: Optional[str] = None,
host: str = "localhost",
port: int = 19530,
token: Optional[str] = None,
db_name: str = "default",
):
try:
from pymilvus import connections, Collection, FieldSchema, CollectionSchema, DataType, utility
except ImportError:
raise ImportError(
"pymilvus is required for Milvus support. "
"Install with: pip install pymilvus"
)
self._pymilvus = {
"connections": connections,
"Collection": Collection,
"FieldSchema": FieldSchema,
"CollectionSchema": CollectionSchema,
"DataType": DataType,
"utility": utility,
}
if url:
# Parse milvus://host:port
from urllib.parse import urlparse
parsed = urlparse(url)
host = parsed.hostname or host
port = parsed.port or port
connections.connect(
alias="default",
host=host,
port=port,
token=token,
db_name=db_name,
)
logger.info(f"Connected to Milvus at {host}:{port}")
def create_collection(
self,
name: str,
dimension: int,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create a new collection."""
FieldSchema = self._pymilvus["FieldSchema"]
CollectionSchema = self._pymilvus["CollectionSchema"]
DataType = self._pymilvus["DataType"]
Collection = self._pymilvus["Collection"]
fields = [
FieldSchema(name="id", dtype=DataType.VARCHAR, is_primary=True, max_length=256),
FieldSchema(name="content", dtype=DataType.VARCHAR, max_length=65535),
FieldSchema(name="content_hash", dtype=DataType.VARCHAR, max_length=64),
FieldSchema(name="created_at", dtype=DataType.DOUBLE),
FieldSchema(name="vector", dtype=DataType.FLOAT_VECTOR, dim=dimension),
]
schema = CollectionSchema(fields=fields, description="PraisonAI knowledge store")
collection = Collection(name=name, schema=schema)
# Create index
metric_map = {"cosine": "COSINE", "euclidean": "L2", "dot": "IP"}
index_params = {
"metric_type": metric_map.get(distance, "COSINE"),
"index_type": "HNSW",
"params": {"M": 16, "efConstruction": 256}
}
collection.create_index(field_name="vector", index_params=index_params)
logger.info(f"Created Milvus collection: {name}")
def delete_collection(self, name: str) -> bool:
"""Delete a collection."""
utility = self._pymilvus["utility"]
try:
utility.drop_collection(name)
return True
except Exception as e:
logger.warning(f"Failed to delete collection {name}: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if a collection exists."""
utility = self._pymilvus["utility"]
return utility.has_collection(name)
def list_collections(self) -> List[str]:
"""List all collections."""
utility = self._pymilvus["utility"]
return utility.list_collections()
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
Collection = self._pymilvus["Collection"]
col = Collection(collection)
col.load()
data = [
[doc.id for doc in documents],
[doc.content for doc in documents],
[doc.content_hash or "" for doc in documents],
[doc.created_at for doc in documents],
[doc.embedding for doc in documents],
]
col.insert(data)
col.flush()
return [doc.id for doc in documents]
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert or update documents."""
Collection = self._pymilvus["Collection"]
col = Collection(collection)
col.load()
data = [
[doc.id for doc in documents],
[doc.content for doc in documents],
[doc.content_hash or "" for doc in documents],
[doc.created_at for doc in documents],
[doc.embedding for doc in documents],
]
col.upsert(data)
col.flush()
return [doc.id for doc in documents]
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents."""
Collection = self._pymilvus["Collection"]
col = Collection(collection)
col.load()
search_params = {"metric_type": "COSINE", "params": {"ef": 64}}
expr = None
if filters:
conditions = [f'{k} == "{v}"' for k, v in filters.items()]
expr = " and ".join(conditions)
results = col.search(
data=[query_embedding],
anns_field="vector",
param=search_params,
limit=limit,
expr=expr,
output_fields=["id", "content", "content_hash", "created_at"],
)
documents = []
for hits in results:
for hit in hits:
if score_threshold and hit.score < score_threshold:
continue
doc = KnowledgeDocument(
id=hit.entity.get("id"),
content=hit.entity.get("content", ""),
embedding=None,
metadata={},
content_hash=hit.entity.get("content_hash"),
created_at=hit.entity.get("created_at", 0),
)
documents.append(doc)
return documents
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
Collection = self._pymilvus["Collection"]
col = Collection(collection)
col.load()
id_list = ", ".join([f'"{i}"' for i in ids])
expr = f"id in [{id_list}]"
results = col.query(
expr=expr,
output_fields=["id", "content", "content_hash", "created_at", "vector"],
)
documents = []
for row in results:
doc = KnowledgeDocument(
id=row["id"],
content=row.get("content", ""),
embedding=row.get("vector"),
metadata={},
content_hash=row.get("content_hash"),
created_at=row.get("created_at", 0),
)
documents.append(doc)
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
Collection = self._pymilvus["Collection"]
col = Collection(collection)
col.load()
if ids:
id_list = ", ".join([f'"{i}"' for i in ids])
expr = f"id in [{id_list}]"
col.delete(expr)
return len(ids)
elif filters:
conditions = [f'{k} == "{v}"' for k, v in filters.items()]
expr = " and ".join(conditions)
col.delete(expr)
return -1
return 0
def count(self, collection: str) -> int:
"""Count documents."""
Collection = self._pymilvus["Collection"]
col = Collection(collection)
return col.num_entities
def close(self) -> None:
"""Close the store."""
connections = self._pymilvus["connections"]
connections.disconnect("default")
@@ -0,0 +1,306 @@
"""
MongoDB Vector Search implementation of KnowledgeStore.
Requires: pymongo
Install: pip install pymongo
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class MongoDBVectorKnowledgeStore(KnowledgeStore):
"""
MongoDB Atlas Vector Search store for knowledge/RAG.
Uses MongoDB Atlas Vector Search capabilities.
Example:
store = MongoDBVectorKnowledgeStore(
url="mongodb+srv://...",
database="praisonai",
collection="vectors"
)
"""
def __init__(
self,
url: str = "mongodb://localhost:27017",
database: str = "praisonai",
collection: str = "vectors",
index_name: str = "vector_index",
embedding_field: str = "embedding",
text_field: str = "content",
embedding_dim: int = 1536,
):
"""
Initialize MongoDB Vector store.
Args:
url: MongoDB connection URL
database: Database name
collection: Collection name
index_name: Vector search index name
embedding_field: Field name for embeddings
text_field: Field name for text content
embedding_dim: Embedding dimension
"""
self.url = url
self.database_name = database
self.collection_name = collection
self.index_name = index_name
self.embedding_field = embedding_field
self.text_field = text_field
self.embedding_dim = embedding_dim
self._client = None
self._db = None
self._collection = None
self._initialized = False
def _init_client(self):
"""Initialize MongoDB client lazily."""
if self._initialized:
return
try:
from pymongo import MongoClient
except ImportError:
raise ImportError(
"pymongo is required for MongoDB Vector support. "
"Install with: pip install pymongo"
)
self._client = MongoClient(self.url)
self._db = self._client[self.database_name]
self._collection = self._db[self.collection_name]
self._initialized = True
def create_collection(
self,
name: str,
dimension: int = 1536,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create collection (MongoDB creates on first insert)."""
self._init_client()
# MongoDB creates collections automatically on first insert
# Vector search index must be created via Atlas UI or API
logger.info(
f"Collection '{name}' will be created on first insert. "
f"Note: Vector search index must be created via MongoDB Atlas."
)
def delete_collection(self, name: str) -> bool:
"""Delete a collection."""
self._init_client()
try:
self._db.drop_collection(name)
return True
except Exception as e:
logger.error(f"Failed to delete collection {name}: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if collection exists."""
self._init_client()
return name in self._db.list_collection_names()
def list_collections(self) -> List[str]:
"""List all collections."""
self._init_client()
return self._db.list_collection_names()
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents into collection."""
self._init_client()
coll = self._db[collection]
ids = []
for doc in documents:
mongo_doc = {
"_id": doc.id,
self.text_field: doc.content,
"metadata": doc.metadata or {},
"content_hash": doc.content_hash,
"created_at": doc.created_at,
}
if doc.embedding:
mongo_doc[self.embedding_field] = doc.embedding
try:
coll.insert_one(mongo_doc)
ids.append(doc.id)
except Exception as e:
logger.warning(f"Failed to insert document {doc.id}: {e}")
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Upsert documents into collection."""
self._init_client()
coll = self._db[collection]
ids = []
for doc in documents:
mongo_doc = {
self.text_field: doc.content,
"metadata": doc.metadata or {},
"content_hash": doc.content_hash,
"created_at": doc.created_at,
}
if doc.embedding:
mongo_doc[self.embedding_field] = doc.embedding
try:
coll.replace_one(
{"_id": doc.id},
{"_id": doc.id, **mongo_doc},
upsert=True
)
ids.append(doc.id)
except Exception as e:
logger.warning(f"Failed to upsert document {doc.id}: {e}")
return ids
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents using vector search."""
self._init_client()
coll = self._db[collection]
# Build aggregation pipeline for vector search
pipeline = [
{
"$vectorSearch": {
"index": self.index_name,
"path": self.embedding_field,
"queryVector": query_embedding,
"numCandidates": limit * 10,
"limit": limit
}
},
{
"$project": {
"_id": 1,
self.text_field: 1,
"metadata": 1,
"content_hash": 1,
"created_at": 1,
"score": {"$meta": "vectorSearchScore"}
}
}
]
# Add filter if provided
if filters:
pipeline[0]["$vectorSearch"]["filter"] = filters
try:
results = list(coll.aggregate(pipeline))
documents = []
for doc in results:
score = doc.get("score", 0)
if score_threshold and score < score_threshold:
continue
documents.append(KnowledgeDocument(
id=str(doc["_id"]),
content=doc.get(self.text_field, ""),
metadata={**(doc.get("metadata") or {}), "score": score},
content_hash=doc.get("content_hash"),
created_at=doc.get("created_at", 0)
))
return documents
except Exception as e:
logger.error(f"Vector search failed: {e}")
return []
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
self._init_client()
coll = self._db[collection]
documents = []
for doc_id in ids:
try:
doc = coll.find_one({"_id": doc_id})
if doc:
documents.append(KnowledgeDocument(
id=str(doc["_id"]),
content=doc.get(self.text_field, ""),
embedding=doc.get(self.embedding_field),
metadata=doc.get("metadata"),
content_hash=doc.get("content_hash"),
created_at=doc.get("created_at", 0)
))
except Exception as e:
logger.warning(f"Failed to get document {doc_id}: {e}")
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents by IDs or filters."""
self._init_client()
coll = self._db[collection]
if ids:
result = coll.delete_many({"_id": {"$in": ids}})
return result.deleted_count
elif filters:
result = coll.delete_many(filters)
return result.deleted_count
else:
return 0
def count(self, collection: str) -> int:
"""Count documents in collection."""
self._init_client()
return self._db[collection].count_documents({})
def close(self) -> None:
"""Close the connection."""
if self._client:
self._client.close()
self._client = None
self._db = None
self._collection = None
self._initialized = False
@@ -0,0 +1,335 @@
"""
PGVector implementation of KnowledgeStore.
Requires: pgvector, psycopg2-binary
Install: pip install pgvector psycopg2-binary
"""
import json
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class PGVectorKnowledgeStore(KnowledgeStore):
"""
PGVector-based knowledge store for vector search.
Uses PostgreSQL with pgvector extension.
Example:
store = PGVectorKnowledgeStore(
url="postgresql://localhost:5432/praisonai"
)
"""
def __init__(
self,
url: Optional[str] = None,
host: str = "localhost",
port: int = 5432,
database: str = "praisonai",
user: str = "postgres",
password: str = "",
schema: str = "public",
auto_create_extension: bool = True,
):
try:
import psycopg2
from psycopg2 import pool as pg_pool
from psycopg2.extras import RealDictCursor
except ImportError:
raise ImportError(
"psycopg2 is required for PGVector support. "
"Install with: pip install psycopg2-binary"
)
self._psycopg2 = psycopg2
self._RealDictCursor = RealDictCursor
self.schema = schema
if url:
self._pool = pg_pool.ThreadedConnectionPool(1, 5, url)
else:
self._pool = pg_pool.ThreadedConnectionPool(
1, 5,
host=host, port=port, database=database,
user=user, password=password,
)
if auto_create_extension:
self._create_extension()
def _get_conn(self):
return self._pool.getconn()
def _put_conn(self, conn):
self._pool.putconn(conn)
def _create_extension(self):
"""Create pgvector extension if not exists."""
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute("CREATE EXTENSION IF NOT EXISTS vector")
conn.commit()
finally:
self._put_conn(conn)
def _table_name(self, collection: str) -> str:
return f"{self.schema}.praison_vec_{collection}"
def create_collection(
self,
name: str,
dimension: int,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create a new collection table."""
table = self._table_name(name)
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute(f"""
CREATE TABLE IF NOT EXISTS {table} (
id VARCHAR(255) PRIMARY KEY,
content TEXT,
content_hash VARCHAR(64),
created_at DOUBLE PRECISION,
metadata JSONB,
embedding vector({dimension})
)
""")
# Create index based on distance metric
op_map = {"cosine": "vector_cosine_ops", "euclidean": "vector_l2_ops", "dot": "vector_ip_ops"}
ops = op_map.get(distance, "vector_cosine_ops")
cur.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{name}_embedding
ON {table} USING hnsw (embedding {ops})
""")
conn.commit()
logger.info(f"Created PGVector table: {table}")
finally:
self._put_conn(conn)
def delete_collection(self, name: str) -> bool:
"""Delete a collection table."""
table = self._table_name(name)
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute(f"DROP TABLE IF EXISTS {table}")
conn.commit()
return True
except Exception as e:
logger.warning(f"Failed to delete table {table}: {e}")
return False
finally:
self._put_conn(conn)
def collection_exists(self, name: str) -> bool:
"""Check if a collection exists."""
table = self._table_name(name)
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute("""
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_schema = %s AND table_name = %s
)
""", (self.schema, f"praison_vec_{name}"))
return cur.fetchone()[0]
finally:
self._put_conn(conn)
def list_collections(self) -> List[str]:
"""List all collections."""
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute("""
SELECT table_name FROM information_schema.tables
WHERE table_schema = %s AND table_name LIKE 'praison_vec_%%'
""", (self.schema,))
return [row[0].replace("praison_vec_", "") for row in cur.fetchall()]
finally:
self._put_conn(conn)
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
table = self._table_name(collection)
conn = self._get_conn()
try:
with conn.cursor() as cur:
for doc in documents:
if doc.embedding is None:
raise ValueError(f"Document {doc.id} has no embedding")
cur.execute(f"""
INSERT INTO {table} (id, content, content_hash, created_at, metadata, embedding)
VALUES (%s, %s, %s, %s, %s, %s)
""", (
doc.id, doc.content, doc.content_hash, doc.created_at,
json.dumps(doc.metadata) if doc.metadata else None,
doc.embedding,
))
conn.commit()
return [doc.id for doc in documents]
finally:
self._put_conn(conn)
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert or update documents."""
table = self._table_name(collection)
conn = self._get_conn()
try:
with conn.cursor() as cur:
for doc in documents:
if doc.embedding is None:
raise ValueError(f"Document {doc.id} has no embedding")
cur.execute(f"""
INSERT INTO {table} (id, content, content_hash, created_at, metadata, embedding)
VALUES (%s, %s, %s, %s, %s, %s)
ON CONFLICT (id) DO UPDATE SET
content = EXCLUDED.content,
content_hash = EXCLUDED.content_hash,
created_at = EXCLUDED.created_at,
metadata = EXCLUDED.metadata,
embedding = EXCLUDED.embedding
""", (
doc.id, doc.content, doc.content_hash, doc.created_at,
json.dumps(doc.metadata) if doc.metadata else None,
doc.embedding,
))
conn.commit()
return [doc.id for doc in documents]
finally:
self._put_conn(conn)
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents."""
table = self._table_name(collection)
conn = self._get_conn()
try:
with conn.cursor(cursor_factory=self._RealDictCursor) as cur:
# Use cosine distance operator <=>
query = f"""
SELECT id, content, content_hash, created_at, metadata,
1 - (embedding <=> %s::vector) as score
FROM {table}
"""
params = [query_embedding]
if score_threshold:
query += " WHERE 1 - (embedding <=> %s::vector) >= %s"
params.extend([query_embedding, score_threshold])
query += " ORDER BY embedding <=> %s::vector LIMIT %s"
params.extend([query_embedding, limit])
cur.execute(query, params)
documents = []
for row in cur.fetchall():
doc = KnowledgeDocument(
id=row["id"],
content=row["content"],
embedding=None,
metadata=row["metadata"] or {},
content_hash=row["content_hash"],
created_at=row["created_at"],
)
documents.append(doc)
return documents
finally:
self._put_conn(conn)
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
table = self._table_name(collection)
conn = self._get_conn()
try:
with conn.cursor(cursor_factory=self._RealDictCursor) as cur:
placeholders = ",".join(["%s"] * len(ids))
cur.execute(f"""
SELECT id, content, content_hash, created_at, metadata, embedding::float[]
FROM {table} WHERE id IN ({placeholders})
""", ids)
documents = []
for row in cur.fetchall():
doc = KnowledgeDocument(
id=row["id"],
content=row["content"],
embedding=row["embedding"],
metadata=row["metadata"] or {},
content_hash=row["content_hash"],
created_at=row["created_at"],
)
documents.append(doc)
return documents
finally:
self._put_conn(conn)
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
table = self._table_name(collection)
conn = self._get_conn()
try:
with conn.cursor() as cur:
if ids:
placeholders = ",".join(["%s"] * len(ids))
cur.execute(f"DELETE FROM {table} WHERE id IN ({placeholders})", ids)
deleted = cur.rowcount
else:
deleted = 0
conn.commit()
return deleted
finally:
self._put_conn(conn)
def count(self, collection: str) -> int:
"""Count documents."""
table = self._table_name(collection)
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute(f"SELECT COUNT(*) FROM {table}")
return cur.fetchone()[0]
finally:
self._put_conn(conn)
def close(self) -> None:
"""Close the store."""
if self._pool:
self._pool.closeall()
@@ -0,0 +1,253 @@
"""
Pinecone implementation of KnowledgeStore.
Requires: pinecone-client
Install: pip install pinecone-client
"""
import logging
import os
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class PineconeKnowledgeStore(KnowledgeStore):
"""
Pinecone-based knowledge store for vector search.
Cloud-only managed vector database.
Example:
store = PineconeKnowledgeStore(
api_key="your-api-key",
index_name="praisonai"
)
"""
def __init__(
self,
api_key: Optional[str] = None,
index_name: Optional[str] = None,
environment: Optional[str] = None,
namespace: str = "",
):
try:
from pinecone import Pinecone
except ImportError:
raise ImportError(
"pinecone-client is required for Pinecone support. "
"Install with: pip install pinecone-client"
)
api_key = api_key or os.getenv("PINECONE_API_KEY")
if not api_key:
raise ValueError("Pinecone API key is required")
self._pc = Pinecone(api_key=api_key)
self._index_name = index_name or os.getenv("PINECONE_INDEX", "praisonai")
self._namespace = namespace
self._index = None
# Try to connect to existing index
try:
self._index = self._pc.Index(self._index_name)
logger.info(f"Connected to Pinecone index: {self._index_name}")
except Exception as e:
logger.warning(f"Index {self._index_name} not found: {e}")
def create_collection(
self,
name: str,
dimension: int,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create a new index (Pinecone calls them indexes, not collections)."""
from pinecone import ServerlessSpec
metric_map = {"cosine": "cosine", "euclidean": "euclidean", "dot": "dotproduct"}
self._pc.create_index(
name=name,
dimension=dimension,
metric=metric_map.get(distance, "cosine"),
spec=ServerlessSpec(cloud="aws", region="us-east-1")
)
self._index_name = name
self._index = self._pc.Index(name)
logger.info(f"Created Pinecone index: {name}")
def delete_collection(self, name: str) -> bool:
"""Delete an index."""
try:
self._pc.delete_index(name)
if name == self._index_name:
self._index = None
return True
except Exception as e:
logger.warning(f"Failed to delete index {name}: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if an index exists."""
try:
indexes = self._pc.list_indexes()
return any(idx.name == name for idx in indexes)
except Exception:
return False
def list_collections(self) -> List[str]:
"""List all indexes."""
indexes = self._pc.list_indexes()
return [idx.name for idx in indexes]
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
if collection != self._index_name:
self._index = self._pc.Index(collection)
self._index_name = collection
vectors = []
ids = []
for doc in documents:
if doc.embedding is None:
raise ValueError(f"Document {doc.id} has no embedding")
vectors.append({
"id": doc.id,
"values": doc.embedding,
"metadata": {
"content": doc.content,
"content_hash": doc.content_hash or "",
"created_at": doc.created_at,
**(doc.metadata or {})
}
})
ids.append(doc.id)
self._index.upsert(vectors=vectors, namespace=self._namespace)
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert or update documents."""
return self.insert(collection, documents)
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents."""
if collection != self._index_name:
self._index = self._pc.Index(collection)
self._index_name = collection
results = self._index.query(
vector=query_embedding,
top_k=limit,
filter=filters,
include_metadata=True,
namespace=self._namespace,
)
documents = []
for match in results.matches:
if score_threshold and match.score < score_threshold:
continue
metadata = match.metadata or {}
content = metadata.pop("content", "")
content_hash = metadata.pop("content_hash", None)
created_at = metadata.pop("created_at", 0)
doc = KnowledgeDocument(
id=match.id,
content=content,
embedding=None,
metadata=metadata,
content_hash=content_hash,
created_at=created_at,
)
documents.append(doc)
return documents
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
if collection != self._index_name:
self._index = self._pc.Index(collection)
self._index_name = collection
results = self._index.fetch(ids=ids, namespace=self._namespace)
documents = []
for doc_id, vector in results.vectors.items():
metadata = vector.metadata or {}
content = metadata.pop("content", "")
content_hash = metadata.pop("content_hash", None)
created_at = metadata.pop("created_at", 0)
doc = KnowledgeDocument(
id=doc_id,
content=content,
embedding=list(vector.values),
metadata=metadata,
content_hash=content_hash,
created_at=created_at,
)
documents.append(doc)
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
if collection != self._index_name:
self._index = self._pc.Index(collection)
self._index_name = collection
if ids:
self._index.delete(ids=ids, namespace=self._namespace)
return len(ids)
elif filters:
self._index.delete(filter=filters, namespace=self._namespace)
return -1
return 0
def count(self, collection: str) -> int:
"""Count documents."""
if collection != self._index_name:
self._index = self._pc.Index(collection)
self._index_name = collection
stats = self._index.describe_index_stats()
if self._namespace:
return stats.namespaces.get(self._namespace, {}).get("vector_count", 0)
return stats.total_vector_count
def close(self) -> None:
"""Close the store."""
self._index = None
Loaded 100 of 125 files, more files were not shown because too many files have changed in this diff. Show more