Compare commits

...
6 Commits
108 changed files with 16684 additions and 17 deletions

No files matched your search

+287 -1
View File
@@ -261,6 +261,12 @@ npm install praisonai
| ↳ 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
| ↳ Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
| ↳ Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
| ↳ Shadow Git Checkpoints | [Example](#10-shadow-git-checkpoints) | [📖](https://docs.praison.ai/features/checkpoints) |
| ↳ Background Tasks | [Example](examples/background/basic_background.py) | [📖](https://docs.praison.ai/features/background-tasks) |
| ↳ Policy Engine | [Example](examples/policy/basic_policy.py) | [📖](https://docs.praison.ai/features/policy-engine) |
| ↳ Thinking Budgets | [Example](examples/thinking/basic_thinking.py) | [📖](https://docs.praison.ai/features/thinking-budgets) |
| ↳ Output Styles | [Example](examples/output/basic_output.py) | [📖](https://docs.praison.ai/features/output-styles) |
| ↳ Context Compaction | [Example](examples/compaction/basic_compaction.py) | [📖](https://docs.praison.ai/features/context-compaction) |
| **📊 Monitoring & Management** | | |
| ↳ Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
| ↳ Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
@@ -930,8 +936,191 @@ praisonai checkpoint diff # Show changes
praisonai checkpoint restore abc123 # Restore to checkpoint
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/checkpoints)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/checkpoint)
- [💻 Example](examples/checkpoints/basic_checkpoints.py)
### 11. Field Names Reference (A-I-G-S)
---
### 11. Background Tasks
Run agent tasks asynchronously without blocking:
```python
import asyncio
from praisonaiagents.background import BackgroundRunner, BackgroundConfig
async def main():
config = BackgroundConfig(max_concurrent_tasks=3)
runner = BackgroundRunner(config=config)
async def my_task(name: str) -> str:
await asyncio.sleep(2)
return f"Task {name} completed"
task = await runner.submit(my_task, args=("example",), name="my_task")
await task.wait(timeout=10.0)
print(task.result)
asyncio.run(main())
```
**CLI Commands:**
```bash
praisonai background list # List running tasks
praisonai background status <id> # Check task status
praisonai background cancel <id> # Cancel a task
praisonai background clear # Clear completed tasks
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/background-tasks)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/background)
- [💻 Example](examples/background/basic_background.py)
---
### 12. Policy Engine
Control what agents can and cannot do with policy-based execution:
```python
from praisonaiagents.policy import (
PolicyEngine, Policy, PolicyRule, PolicyAction
)
engine = PolicyEngine()
policy = Policy(
name="no_delete",
rules=[
PolicyRule(
action=PolicyAction.DENY,
resource="tool:delete_*",
reason="Delete operations blocked"
)
]
)
engine.add_policy(policy)
result = engine.check("tool:delete_file", {})
print(f"Allowed: {result.allowed}")
```
**CLI Commands:**
```bash
praisonai policy list # List policies
praisonai policy check "tool:name" # Check if allowed
praisonai policy init # Create template
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/policy-engine)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/policy)
- [💻 Example](examples/policy/basic_policy.py)
---
### 13. Thinking Budgets
Configure token budgets for extended thinking:
```python
from praisonaiagents.thinking import ThinkingBudget, ThinkingTracker
# Use predefined levels
budget = ThinkingBudget.high() # 16,000 tokens
# Track usage
tracker = ThinkingTracker()
session = tracker.start_session(budget_tokens=16000)
tracker.end_session(session, tokens_used=12000)
summary = tracker.get_summary()
print(f"Utilization: {summary['average_utilization']:.1%}")
```
**CLI Commands:**
```bash
praisonai thinking status # Show current budget
praisonai thinking set high # Set budget level
praisonai thinking stats # Show usage statistics
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/thinking-budgets)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/thinking)
- [💻 Example](examples/thinking/basic_thinking.py)
---
### 14. Output Styles
Configure how agents format their responses:
```python
from praisonaiagents.output import OutputStyle, OutputFormatter
# Use preset styles
style = OutputStyle.concise()
formatter = OutputFormatter(style)
# Format output
text = "# Hello\n\nThis is **bold** text."
plain = formatter.format(text)
print(plain)
```
**CLI Commands:**
```bash
praisonai output status # Show current style
praisonai output set concise # Set output style
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/output-styles)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/output-style)
- [💻 Example](examples/output/basic_output.py)
---
### 15. Context Compaction
Automatically manage context window size:
```python
from praisonaiagents.compaction import (
ContextCompactor, CompactionStrategy
)
compactor = ContextCompactor(
max_tokens=4000,
strategy=CompactionStrategy.SLIDING,
preserve_recent=3
)
messages = [...] # Your conversation history
compacted, result = compactor.compact(messages)
print(f"Compression: {result.compression_ratio:.1%}")
```
**CLI Commands:**
```bash
praisonai compaction status # Show settings
praisonai compaction set sliding # Set strategy
praisonai compaction stats # Show statistics
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/context-compaction)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/compaction)
- [💻 Example](examples/compaction/basic_compaction.py)
---
### 16. Field Names Reference (A-I-G-S)
PraisonAI accepts both old (agents.yaml) and new (workflow.yaml) field names. Use the **canonical names** for new projects:
@@ -2535,6 +2724,103 @@ PraisonAI provides zero-dependency persistent memory for agents. For detailed ex
---
## 💾 Persistence (Databases)
Enable automatic conversation persistence with 2 lines of code:
```python
from praisonaiagents import Agent, db
agent = Agent(
name="Assistant",
db=db(database_url="postgresql://localhost/mydb"), # db(...) shortcut
session_id="my-session" # Optional: defaults to per-hour ID (YYYYMMDDHH)
)
agent.chat("Hello!") # Auto-persists messages, runs, traces
```
### Persistence CLI Commands
| Command | Description |
|---------|-------------|
| `praisonai persistence doctor` | Validate DB connectivity |
| `praisonai persistence run` | Run agent with persistence |
| `praisonai persistence resume` | Resume existing session |
| `praisonai persistence export` | Export session to JSONL |
| `praisonai persistence import` | Import session from JSONL |
| `praisonai persistence migrate` | Apply schema migrations |
| `praisonai persistence status` | Show schema status |
### Databases Table
| Database | Store Type | Install | Example | Docs |
|----------|------------|---------|---------|------|
| PostgreSQL | Conversation | `pip install "praisonai[tools]"` | [simple_db_agent.py](examples/persistence/simple_db_agent.py) | [docs](https://docs.praison.ai/docs/databases/postgres) |
| MySQL | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| SQLite | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| SingleStore | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Supabase | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| SurrealDB | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Qdrant | Knowledge | `pip install "praisonai[tools]"` | [knowledge_qdrant.py](examples/persistence/knowledge_qdrant.py) | [docs](https://docs.praison.ai/docs/databases/qdrant) |
| ChromaDB | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Pinecone | Knowledge | `pip install pinecone` | [pinecone_wow.py](examples/vector/pinecone_wow.py) | [docs](https://docs.praison.ai/docs/databases/pinecone) |
| Weaviate | Knowledge | `pip install weaviate-client` | [weaviate_wow.py](examples/vector/weaviate_wow.py) | [docs](https://docs.praison.ai/docs/databases/weaviate) |
| LanceDB | Knowledge | `pip install lancedb` | [lancedb_real_wow.py](examples/vector/lancedb_real_wow.py) | [docs](https://docs.praison.ai/docs/databases/lancedb) |
| Milvus | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| PGVector | Knowledge | `pip install psycopg2-binary` | [pgvector_real_wow.py](examples/vector/pgvector_real_wow.py) | [docs](https://docs.praison.ai/docs/databases/pgvector) |
| Redis Vector | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Cassandra | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| ClickHouse | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Redis | State | `pip install "praisonai[tools]"` | [state_redis.py](examples/persistence/state_redis.py) | [docs](https://docs.praison.ai/docs/databases/redis) |
| MongoDB | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| DynamoDB | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Firestore | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Upstash | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Memory | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
---
## 🔧 Tools Table
Install all tools with: `pip install "praisonai[tools]"`
| Tool | Category | Import | Docs |
|------|----------|--------|------|
| Tavily | Web Search | `from praisonai_tools import TavilyTool` | [docs](https://docs.praison.ai/docs/tools/external/tavily) |
| DuckDuckGo | Web Search | `from praisonai_tools import DuckDuckGoTool` | [docs](https://docs.praison.ai/docs/tools/external/duckduckgo) |
| Exa | Web Search | `from praisonai_tools import ExaTool` | [docs](https://docs.praison.ai/docs/tools/external/exa) |
| Serper | Web Search | `from praisonai_tools import SerperTool` | [docs](https://docs.praison.ai/docs/tools/external/serper) |
| Jina | Web Reader | `from praisonai_tools import JinaTool` | [docs](https://docs.praison.ai/docs/tools/external/jina) |
| Firecrawl | Web Scraping | `from praisonai_tools import FirecrawlTool` | [docs](https://docs.praison.ai/docs/tools/external/firecrawl) |
| Crawl4AI | Web Scraping | `from praisonai_tools import Crawl4AITool` | [docs](https://docs.praison.ai/docs/tools/external/crawl4ai) |
| Wikipedia | Knowledge | `from praisonai_tools import WikipediaTool` | [docs](https://docs.praison.ai/docs/tools/external/wikipedia) |
| ArXiv | Research | `from praisonai_tools import ArxivTool` | [docs](https://docs.praison.ai/docs/tools/external/arxiv) |
| HackerNews | News | `from praisonai_tools import HackerNewsTool` | [docs](https://docs.praison.ai/docs/tools/external/hackernews) |
| YouTube | Media | `from praisonai_tools import YouTubeTool` | [docs](https://docs.praison.ai/docs/tools/external/youtube) |
| Weather | Data | `from praisonai_tools import WeatherTool` | [docs](https://docs.praison.ai/docs/tools/external/weather) |
| PostgreSQL | Database | `from praisonai_tools import PostgresTool` | [docs](https://docs.praison.ai/docs/tools/external/postgres) |
| MySQL | Database | `from praisonai_tools import MySQLTool` | [docs](https://docs.praison.ai/docs/tools/external/mysql) |
| SQLite | Database | `from praisonai_tools import SQLiteTool` | [docs](https://docs.praison.ai/docs/tools/external/sqlite) |
| MongoDB | Database | `from praisonai_tools import MongoDBTool` | [docs](https://docs.praison.ai/docs/tools/external/mongodb) |
| Redis | Database | `from praisonai_tools import RedisTool` | [docs](https://docs.praison.ai/docs/tools/external/redis) |
| Qdrant | Vector DB | `from praisonai_tools import QdrantTool` | [docs](https://docs.praison.ai/docs/tools/external/qdrant) |
| GitHub | DevOps | `from praisonai_tools import GitHubTool` | [docs](https://docs.praison.ai/docs/tools/external/github) |
| Slack | Communication | `from praisonai_tools import SlackTool` | [docs](https://docs.praison.ai/docs/tools/external/slack) |
| Discord | Communication | `from praisonai_tools import DiscordTool` | [docs](https://docs.praison.ai/docs/tools/external/discord) |
| Telegram | Communication | `from praisonai_tools import TelegramTool` | [docs](https://docs.praison.ai/docs/tools/external/telegram) |
| Email | Communication | `from praisonai_tools import EmailTool` | [docs](https://docs.praison.ai/docs/tools/external/email) |
| Notion | Productivity | `from praisonai_tools import NotionTool` | [docs](https://docs.praison.ai/docs/tools/external/notion) |
| File | File System | `from praisonai_tools import FileTool` | [docs](https://docs.praison.ai/docs/tools/external/file) |
| Shell | System | `from praisonai_tools import ShellTool` | [docs](https://docs.praison.ai/docs/tools/external/shell) |
| Python | Code | `from praisonai_tools import PythonTool` | [docs](https://docs.praison.ai/docs/tools/external/python) |
| JSON | Data | `from praisonai_tools import JSONTool` | [docs](https://docs.praison.ai/docs/tools/external/json) |
| CSV | Data | `from praisonai_tools import CSVTool` | [docs](https://docs.praison.ai/docs/tools/external/csv) |
| Calculator | Math | `from praisonai_tools import CalculatorTool` | [docs](https://docs.praison.ai/docs/tools/external/calculator) |
> See [full tools documentation](https://docs.praison.ai/docs/tools/tools) for all 100+ available tools.
---
## 🎓 Video Tutorials
Learn PraisonAI through our comprehensive video series:
+1 -1
View File
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
# Install Python packages (using latest versions)
RUN pip install --no-cache-dir \
praisonai_tools \
"praisonai>=2.4.0" \
"praisonai>=2.4.4" \
"praisonai[chat]" \
"embedchain[github,youtube]"
+1 -1
View File
@@ -20,7 +20,7 @@ RUN mkdir -p /root/.praison
# Install Python packages (using latest versions)
RUN pip install --no-cache-dir \
praisonai_tools \
"praisonai>=2.4.0" \
"praisonai>=2.4.4" \
"praisonai[ui]" \
"praisonai[chat]" \
"praisonai[realtime]" \
+1 -1
View File
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
# Install Python packages (using latest versions)
RUN pip install --no-cache-dir \
praisonai_tools \
"praisonai>=2.4.0" \
"praisonai>=2.4.4" \
"praisonai[ui]" \
"praisonai[crewai]"
+93
View File
@@ -0,0 +1,93 @@
# Persistence CLI Commands
## Quick Reference
```bash
# Check database connectivity
praisonai persistence doctor \
--conversation-url postgresql://localhost:5432/praisonai \
--state-url redis://localhost:6379
# Run agent with persistence
praisonai persistence run \
--conversation-url postgresql://localhost:5432/praisonai \
--session-id my-session
# Resume a session
praisonai persistence resume \
--conversation-url postgresql://localhost:5432/praisonai \
--session-id my-session
# Export session to file
praisonai persistence export \
--conversation-url postgresql://localhost:5432/praisonai \
--session-id my-session \
--output session_backup.jsonl
# Import session from file
praisonai persistence import \
--conversation-url postgresql://localhost:5432/praisonai \
--file session_backup.jsonl
# Check schema status
praisonai persistence status \
--conversation-url postgresql://localhost:5432/praisonai \
--state-url redis://localhost:6379
# Apply migrations
praisonai persistence migrate \
--conversation-url postgresql://localhost:5432/praisonai \
--state-url redis://localhost:6379
```
## Environment Variables
Instead of CLI flags, you can use environment variables:
```bash
export PRAISON_CONVERSATION_URL=postgresql://localhost:5432/praisonai
export PRAISON_STATE_URL=redis://localhost:6379
export PRAISON_KNOWLEDGE_URL=http://localhost:6333
# Then run without URL flags
praisonai persistence doctor
praisonai persistence status
```
## Docker Setup
```bash
# Start all services
docker run -d --name praison-postgres \
-e POSTGRES_PASSWORD=praison123 \
-e POSTGRES_DB=praisonai \
-p 5432:5432 postgres:16
docker run -d --name praison-redis \
-p 6379:6379 redis:7
docker run -d --name praison-qdrant \
-p 6333:6333 -p 6334:6334 qdrant/qdrant
# Verify
docker ps
```
## Example Workflow
```bash
# 1. Start services
docker-compose up -d
# 2. Check connectivity
praisonai persistence doctor
# 3. Run agent
praisonai persistence run --session-id demo-session
# 4. Export for backup
praisonai persistence export --session-id demo-session --output backup.jsonl
# 5. Import to another environment
praisonai persistence import --file backup.jsonl
```
+124
View File
@@ -0,0 +1,124 @@
# PraisonAI Persistence CLI - End-to-End Guide
This guide demonstrates the CLI commands for database persistence.
## Prerequisites
```bash
pip install "praisonai[tools]"
# Start Docker containers
docker run -d --name praison-postgres -p 5432:5432 \
-e POSTGRES_PASSWORD=praison123 \
-e POSTGRES_DB=praisonai \
postgres:16
docker run -d --name praison-qdrant -p 6333:6333 qdrant/qdrant
docker run -d --name praison-redis -p 6379:6379 redis:7
```
## 1. Doctor Command - Validate Connectivity
```bash
# Check all stores
praisonai persistence doctor \
--conversation-url "postgresql://postgres:praison123@localhost:5432/praisonai" \
--knowledge-url "http://localhost:6333" \
--state-url "redis://localhost:6379"
```
**Expected Output:**
```
==================================================
PraisonAI Persistence Doctor
==================================================
[Conversation Store] Testing: postgresql://postgres:****@localhost:5432/praisonai
✅ Connected (postgres)
[Knowledge Store] Testing: http://localhost:6333
✅ Connected (qdrant)
[State Store] Testing: redis://localhost:6379
✅ Connected (redis)
==================================================
Results: 3/3 stores connected successfully
==================================================
```
## 2. Run Command - Execute Agent with Persistence
```bash
# Run agent with persistence (dry-run first)
praisonai persistence run \
--session-id "cli-demo-001" \
--conversation-url "postgresql://postgres:praison123@localhost:5432/praisonai" \
--dry-run \
"Hello, my name is Bob"
# Actually run it
praisonai persistence run \
--session-id "cli-demo-001" \
--conversation-url "postgresql://postgres:praison123@localhost:5432/praisonai" \
"Hello, my name is Bob"
```
**Expected Output:**
```
[Session: cli-demo-001]
Agent: Hello Bob! Nice to meet you. How can I help you today?
```
## 3. Resume Command - Continue a Session
```bash
# Show history
praisonai persistence resume \
--session-id "cli-demo-001" \
--conversation-url "postgresql://postgres:praison123@localhost:5432/praisonai" \
--show-history
# Continue conversation
praisonai persistence resume \
--session-id "cli-demo-001" \
--conversation-url "postgresql://postgres:praison123@localhost:5432/praisonai" \
--continue "What's my name?"
```
**Expected Output:**
```
[Session: cli-demo-001]
Messages in history: 2
--- Conversation History ---
[USER] Hello, my name is Bob
[ASSISTANT] Hello Bob! Nice to meet you...
--- End History ---
Agent: Your name is Bob!
```
## Environment Variables
Instead of passing URLs on command line, use environment variables:
```bash
export PRAISON_CONVERSATION_URL="postgresql://postgres:praison123@localhost:5432/praisonai"
export PRAISON_KNOWLEDGE_URL="http://localhost:6333"
export PRAISON_STATE_URL="redis://localhost:6379"
export OPENAI_API_KEY="your-key"
# Now commands are simpler
praisonai persistence doctor --all
praisonai persistence run --session-id "my-session" "Hello!"
```
## Full CLI Reference
```
praisonai persistence --help
praisonai persistence doctor --help
praisonai persistence run --help
praisonai persistence resume --help
```
@@ -0,0 +1,122 @@
"""
Multi-Agent Persistence Example
Demonstrates how multiple agents can share or isolate conversation
history using the persistence layer.
"""
from praisonai.persistence.factory import create_conversation_store, create_state_store
from praisonai.persistence.conversation.base import ConversationSession, ConversationMessage
import uuid
def main():
# Create stores
conv_store = create_conversation_store("sqlite", path=":memory:")
state_store = create_state_store("memory")
print("=== Multi-Agent Persistence Demo ===\n")
# Scenario 1: Shared session (agents collaborate)
print("--- Scenario 1: Shared Session ---")
shared_session_id = f"shared-{uuid.uuid4().hex[:8]}"
session = ConversationSession(
session_id=shared_session_id,
user_id="user-123",
name="Shared Conversation"
)
conv_store.create_session(session)
# Agent 1 (Researcher) adds context
conv_store.add_message(shared_session_id, ConversationMessage(
role="assistant",
content="[Researcher] I found relevant information about the topic.",
metadata={"agent_id": "researcher", "agent_role": "research"}
))
# Agent 2 (Writer) uses the context
conv_store.add_message(shared_session_id, ConversationMessage(
role="assistant",
content="[Writer] Based on the research, here's a summary...",
metadata={"agent_id": "writer", "agent_role": "writing"}
))
# Agent 3 (Editor) reviews
conv_store.add_message(shared_session_id, ConversationMessage(
role="assistant",
content="[Editor] I've reviewed and polished the content.",
metadata={"agent_id": "editor", "agent_role": "editing"}
))
messages = conv_store.get_messages(shared_session_id)
print(f"Shared session has {len(messages)} messages from different agents")
for msg in messages:
agent = msg.metadata.get("agent_id", "unknown") if msg.metadata else "unknown"
print(f" - {agent}: {msg.content[:50]}...")
# Scenario 2: Isolated sessions (agents work independently)
print("\n--- Scenario 2: Isolated Sessions ---")
agent_sessions = {}
for agent_name in ["planner", "executor", "validator"]:
session_id = f"{agent_name}-{uuid.uuid4().hex[:8]}"
agent_sessions[agent_name] = session_id
conv_store.create_session(ConversationSession(
session_id=session_id,
agent_id=agent_name,
name=f"{agent_name.title()} Session"
))
# Each agent has its own conversation
conv_store.add_message(session_id, ConversationMessage(
role="user",
content=f"Task for {agent_name}"
))
conv_store.add_message(session_id, ConversationMessage(
role="assistant",
content=f"[{agent_name.title()}] Processing task..."
))
print("Created isolated sessions for each agent:")
for agent, sid in agent_sessions.items():
msgs = conv_store.get_messages(sid)
print(f" - {agent}: {len(msgs)} messages")
# Scenario 3: Shared state across agents
print("\n--- Scenario 3: Shared State ---")
# Agents share state for coordination
state_store.set("workflow:status", {"phase": "research", "progress": 0})
state_store.set("workflow:context", {"topic": "AI Agents", "deadline": "2024-12-31"})
# Agent 1 updates progress
state = state_store.get("workflow:status")
state["progress"] = 33
state["phase"] = "writing"
state_store.set("workflow:status", state)
print(f"Agent 1 updated state: {state}")
# Agent 2 reads and updates
state = state_store.get("workflow:status")
state["progress"] = 66
state["phase"] = "editing"
state_store.set("workflow:status", state)
print(f"Agent 2 updated state: {state}")
# Agent 3 completes
state = state_store.get("workflow:status")
state["progress"] = 100
state["phase"] = "complete"
state_store.set("workflow:status", state)
print(f"Agent 3 completed: {state}")
# Cleanup
conv_store.close()
state_store.close()
print("\n✅ Multi-agent persistence demo complete!")
if __name__ == "__main__":
main()
+40
View File
@@ -0,0 +1,40 @@
"""Multi-Agent Shared Session - DB Persistence Test"""
import sys
sys.path.insert(0, 'src/praisonai')
from praisonai.persistence import create_conversation_store
from praisonai.persistence.conversation.base import ConversationSession, ConversationMessage
# Create shared session for multiple agents
store = create_conversation_store("sqlite", path="/tmp/multi_agent.db")
session_id = "shared-session-123"
# Create session
session = ConversationSession(session_id=session_id, agent_id="multi")
try:
store.create_session(session)
except Exception:
pass
# Agent 1 writes
msg1 = ConversationMessage(session_id=session_id, role="user", content="Agent1: Hello")
store.add_message(session_id, msg1)
msg2 = ConversationMessage(session_id=session_id, role="assistant", content="Agent1: Hi there!")
store.add_message(session_id, msg2)
# Agent 2 writes to same session
msg3 = ConversationMessage(session_id=session_id, role="user", content="Agent2: What did Agent1 say?")
store.add_message(session_id, msg3)
msg4 = ConversationMessage(session_id=session_id, role="assistant", content="Agent2: Agent1 said Hello")
store.add_message(session_id, msg4)
# Verify both agents' messages are in session
messages = store.get_messages(session_id)
print(f"Total messages in shared session: {len(messages)}")
assert len(messages) >= 4, "Should have messages from both agents"
# Verify content
contents = [m.content for m in messages]
assert any("Agent1" in c for c in contents), "Should have Agent1 messages"
assert any("Agent2" in c for c in contents), "Should have Agent2 messages"
print("PASSED: Multi-agent shared session persistence")
@@ -0,0 +1,16 @@
"""Multi-Agent Shared Session Demo"""
from praisonaiagents import Agent
from praisonai.persistence import create_conversation_store
store = create_conversation_store("sqlite", path="/tmp/multi_agent.db")
agent1 = Agent(name="Researcher", llm="gpt-4o-mini", db=store, session_id="shared-session")
agent2 = Agent(name="Writer", llm="gpt-4o-mini", db=store, session_id="shared-session")
r1 = agent1.chat("Research topic: AI in healthcare. Key finding: reduces diagnosis time by 50%")
print(f"Researcher: {r1[:50]}...")
r2 = agent2.chat("Based on the research, what was the key finding?")
print(f"Writer: {r2[:50]}...")
assert "50" in r2 or "diagnosis" in r2.lower(), "Writer should see Researcher's context"
print("PASSED: Multi-agent shared session works")
+48
View File
@@ -0,0 +1,48 @@
"""
Basic Observability Usage Example
This example demonstrates how to use the observability module
with PraisonAI agents.
Usage:
# Set your provider's API key first
export LANGFUSE_PUBLIC_KEY=pk-lf-xxx
export LANGFUSE_SECRET_KEY=sk-lf-xxx
python basic_usage.py
"""
from praisonai_tools.observability import obs
from praisonaiagents import Agent
# Initialize observability (auto-detects provider from env vars)
obs.init()
# Or specify a provider explicitly
# obs.init(provider="langfuse", project_name="my-project")
# Create an agent
agent = Agent(
instructions="You are a helpful assistant.",
model="gpt-4o-mini",
)
# Use trace context manager for complete workflow tracing
with obs.trace("chat-session", session_id="user-123"):
# Use span context manager for individual operations
with obs.span("user-query", kind=obs.SpanKind.AGENT):
response = agent.chat("What is the capital of France?")
print(response)
# Log LLM calls manually if needed
obs.log_llm_call(
model="gpt-4o-mini",
input_messages="What is 2+2?",
output="4",
input_tokens=10,
output_tokens=1,
)
# Check diagnostics
print("\nDiagnostics:")
print(obs.doctor())
@@ -0,0 +1,48 @@
"""
Langfuse Integration Example
This example shows how to use Langfuse for LLM observability.
Setup:
1. Sign up at https://langfuse.com/
2. Get your API keys from the project settings
3. Set environment variables:
export LANGFUSE_PUBLIC_KEY=pk-lf-xxx
export LANGFUSE_SECRET_KEY=sk-lf-xxx
4. Install dependencies:
pip install opentelemetry-sdk opentelemetry-exporter-otlp
Usage:
python langfuse_example.py
"""
import os
from praisonai_tools.observability import obs
from praisonaiagents import Agent
# Initialize Langfuse
success = obs.init(
provider="langfuse",
project_name="praisonai-demo",
)
if not success:
print("Failed to initialize Langfuse. Check your API keys.")
print("Required: LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY")
exit(1)
print("Langfuse initialized successfully!")
print(f"View traces at: https://cloud.langfuse.com/")
# Create agent
agent = Agent(
instructions="You are a helpful coding assistant.",
model="gpt-4o-mini",
)
# Run with tracing
with obs.trace("coding-session", user_id="developer-1"):
response = agent.chat("Write a Python function to calculate fibonacci numbers")
print(response)
print("\nCheck Langfuse dashboard for traces!")
@@ -0,0 +1,22 @@
"""MLflow Local Observability Demo (no API key needed)"""
import mlflow
import time
mlflow.set_tracking_uri("file:///tmp/mlflow_demo")
mlflow.set_experiment("praisonai-demo")
with mlflow.start_run(run_name="agent-trace"):
mlflow.log_param("agent_name", "Assistant")
mlflow.log_param("model", "gpt-4o-mini")
# Simulate agent work
start = time.time()
time.sleep(0.1)
latency = time.time() - start
mlflow.log_metric("latency_ms", latency * 1000)
mlflow.log_metric("tokens_used", 150)
print(f"Logged run to MLflow: latency={latency*1000:.1f}ms")
print("PASSED: MLflow local observability works")
@@ -0,0 +1,53 @@
"""
Multi-Agent Tracing Example
This example shows how to trace multi-agent workflows with
parent/child span relationships.
Usage:
# Set your provider's API key
export LANGFUSE_PUBLIC_KEY=pk-lf-xxx
export LANGFUSE_SECRET_KEY=sk-lf-xxx
python multi_agent_tracing.py
"""
from praisonai_tools.observability import obs
from praisonai_tools.observability.base import SpanKind
from praisonaiagents import Agent, PraisonAIAgents
# Initialize observability
obs.init()
# Create agents
researcher = Agent(
name="Researcher",
instructions="You research topics and provide detailed information.",
model="gpt-4o-mini",
)
writer = Agent(
name="Writer",
instructions="You write clear, engaging content based on research.",
model="gpt-4o-mini",
)
# Trace the multi-agent workflow
with obs.trace("content-creation", metadata={"workflow": "research-write"}):
# Research phase
with obs.span("research-phase", kind=SpanKind.AGENT) as research_span:
research_span.attributes["agent"] = "Researcher"
research_result = researcher.chat("Research the benefits of AI in healthcare")
# Writing phase
with obs.span("writing-phase", kind=SpanKind.AGENT) as write_span:
write_span.attributes["agent"] = "Writer"
write_span.attributes["input_length"] = len(str(research_result))
final_content = writer.chat(f"Write a blog post based on: {research_result}")
print("Final Content:")
print(final_content)
# Check trace info
print("\nDiagnostics:", obs.doctor())
+79
View File
@@ -0,0 +1,79 @@
"""
Qdrant Knowledge Store Example for PraisonAI.
Demonstrates vector insert + search through the knowledge store.
Requirements:
pip install "praisonai[tools]"
Docker Setup:
docker run -d --name praison-qdrant -p 6333:6333 qdrant/qdrant
Run:
python knowledge_qdrant.py
Expected Output:
=== Qdrant Knowledge Store Demo ===
Created collection: demo_knowledge
Inserted 3 documents
Search results: 2 documents found
"""
from praisonai.persistence.factory import create_knowledge_store
from praisonai.persistence.knowledge.base import KnowledgeDocument
import random
# Create Qdrant knowledge store
store = create_knowledge_store("qdrant", url="http://localhost:6333")
collection_name = "demo_knowledge"
dimension = 384 # Embedding dimension
# Create collection
print("=== Qdrant Knowledge Store Demo ===")
if store.collection_exists(collection_name):
store.delete_collection(collection_name)
store.create_collection(collection_name, dimension=dimension)
print(f"Created collection: {collection_name}")
# Create sample documents with embeddings
random.seed(42)
documents = [
KnowledgeDocument(
id="doc-1",
content="Python is a versatile programming language used for web development, data science, and AI.",
embedding=[random.random() for _ in range(dimension)],
metadata={"category": "programming", "language": "python"}
),
KnowledgeDocument(
id="doc-2",
content="Machine learning enables computers to learn patterns from data without explicit programming.",
embedding=[random.random() for _ in range(dimension)],
metadata={"category": "ai", "topic": "ml"}
),
KnowledgeDocument(
id="doc-3",
content="PostgreSQL is a powerful open-source relational database system.",
embedding=[random.random() for _ in range(dimension)],
metadata={"category": "database", "type": "sql"}
),
]
# Insert documents
ids = store.insert(collection_name, documents)
print(f"Inserted {len(ids)} documents")
# Search for similar documents
query_embedding = [random.random() for _ in range(dimension)]
results = store.search(collection_name, query_embedding, limit=2)
print(f"Search results: {len(results)} documents found")
for doc in results:
print(f" - {doc.id}: {doc.content[:50]}...")
# Cleanup
store.delete_collection(collection_name)
store.close()
print("\n=== Demo Complete ===")
+43
View File
@@ -0,0 +1,43 @@
"""
Minimal Agent with Database Persistence
This is the simplest way to add persistence to your agent.
Messages, runs, and traces are automatically saved.
Run:
python minimal_agent_db.py
Expected output:
- Agent responds to your question
- Data is persisted to PostgreSQL
- Session can be resumed later with same session_id
"""
from praisonaiagents import Agent, db
# Create database connection (PostgreSQL + Redis)
my_db = db(
database_url="postgresql://postgres:praison123@localhost:5432/praisonai",
state_url="redis://localhost:6379" # Optional: for state/tracing
)
# Create agent with persistence
agent = Agent(
name="Assistant",
instructions="You are a helpful assistant. Be concise.",
db=my_db,
# session_id is optional - defaults to per-hour ID (YYYYMMDDHH-hash)
# Set explicitly for session continuity across runs:
session_id="my-persistent-session"
)
# Chat - messages are automatically persisted
response = agent.chat("What is the capital of France?")
print(f"Response: {response}")
# Verify persistence
data = my_db.export_session("my-persistent-session")
print(f"\nMessages stored: {len(data.get('messages', []))}")
# Clean up
my_db.close()
@@ -0,0 +1,61 @@
"""
PostgreSQL with Runs and Traces
Demonstrates full persistence with run tracking and tracing.
Run:
python postgres_runs_traces.py
Expected output:
- Agent responds with tool usage
- Runs and traces are persisted
- Export shows full session data
"""
from praisonaiagents import Agent, db
def get_weather(city: str) -> str:
"""Get weather for a city."""
return f"Weather in {city}: Sunny, 72°F"
# Create database with PostgreSQL + Redis
my_db = db(
database_url="postgresql://postgres:praison123@localhost:5432/praisonai",
state_url="redis://localhost:6379"
)
# Create agent with tools
agent = Agent(
name="WeatherBot",
instructions="You help with weather. Use the get_weather tool when asked.",
tools=[get_weather],
db=my_db,
session_id="weather-session-001"
)
# First chat - triggers run tracking
print("=== First Question ===")
response1 = agent.chat("What's the weather in Tokyo?")
print(f"Response: {response1}\n")
# Second chat - same session, new run
print("=== Second Question ===")
response2 = agent.chat("How about New York?")
print(f"Response: {response2}\n")
# Export session data
print("=== Session Data ===")
data = my_db.export_session("weather-session-001")
print(f"Session ID: {data.get('session_id')}")
print(f"Messages: {len(data.get('messages', []))}")
print(f"Agent: {data.get('agent_name')}")
# Show messages
print("\n=== Messages ===")
for msg in data.get('messages', []):
role = msg.get('role', 'unknown')
content = msg.get('content', '')[:60]
print(f" [{role}] {content}...")
my_db.close()
print("\n✅ Done")
+83
View File
@@ -0,0 +1,83 @@
"""
Redis State Store
Use Redis for state management, tracing, and caching.
Run:
python redis_state.py
Expected output:
- State operations (set/get/delete)
- Tracing operations
"""
from praisonaiagents import db
# Create Redis-only database (for state/tracing)
redis_db = db.RedisDB(host="localhost", port=6379)
print("=== Redis State Store ===")
# Test state operations
print("\n1. State Operations:")
redis_db._init_stores() # Initialize stores
# Set a value
redis_db._state_store.set("user:123:preferences", {
"theme": "dark",
"language": "en"
})
print(" Set user preferences")
# Get the value
prefs = redis_db._state_store.get("user:123:preferences")
print(f" Got preferences: {prefs}")
# Set with TTL (expires in 60 seconds)
redis_db._state_store.set("temp:token", "abc123", ttl=60)
print(" Set temporary token (60s TTL)")
# Delete
redis_db._state_store.delete("user:123:preferences")
print(" Deleted preferences")
# Test tracing
print("\n2. Tracing Operations:")
# Start a trace
trace_id = "trace-example-001"
redis_db.on_trace_start(
trace_id=trace_id,
session_id="example-session",
agent_name="ExampleAgent",
metadata={"purpose": "demo"}
)
print(f" Started trace: {trace_id}")
# Add spans
span_id = "span-llm-001"
redis_db.on_span_start(
span_id=span_id,
trace_id=trace_id,
name="llm_call",
attributes={"model": "gpt-4", "temperature": 0.7}
)
print(f" Started span: {span_id}")
redis_db.on_span_end(
span_id=span_id,
status="ok",
attributes={"tokens": 150, "latency_ms": 450}
)
print(f" Ended span: {span_id}")
# End trace
redis_db.on_trace_end(
trace_id=trace_id,
status="ok",
metadata={"total_spans": 1}
)
print(f" Ended trace: {trace_id}")
redis_db.close()
print("\n✅ Done")
+66
View File
@@ -0,0 +1,66 @@
"""
Session Resume Example
Demonstrates how to resume a conversation from a previous session.
Run:
python session_resume.py
Expected output:
- First run: Creates new session
- Second run: Resumes with history
"""
from praisonaiagents import Agent, db
# Create database
my_db = db(
database_url="postgresql://postgres:praison123@localhost:5432/praisonai",
state_url="redis://localhost:6379"
)
SESSION_ID = "resume-demo-session"
# Create agent with explicit session_id
agent = Agent(
name="MemoryBot",
instructions="You remember our conversation. Be helpful and concise.",
db=my_db,
session_id=SESSION_ID
)
# Check if this is a new or resumed session
data = my_db.export_session(SESSION_ID)
existing_messages = len(data.get('messages', []))
if existing_messages == 0:
print("=== NEW SESSION ===")
print("This is a new conversation.\n")
response = agent.chat("My name is Alice and I love Python programming.")
print(f"Response: {response}\n")
response = agent.chat("What's my favorite programming language?")
print(f"Response: {response}\n")
else:
print(f"=== RESUMED SESSION ({existing_messages} previous messages) ===")
print("Continuing previous conversation.\n")
# Show previous messages
print("Previous messages:")
for msg in data.get('messages', [])[:4]:
role = msg.get('role', 'unknown')
content = msg.get('content', '')[:60]
print(f" [{role}] {content}...")
print()
# Continue conversation
response = agent.chat("What's my name and what do I like?")
print(f"Response: {response}\n")
# Show final state
final_data = my_db.export_session(SESSION_ID)
print(f"Total messages now: {len(final_data.get('messages', []))}")
my_db.close()
print("✅ Done")
+63
View File
@@ -0,0 +1,63 @@
"""
Simple Database Persistence Example for PraisonAI Agents.
This example shows how to enable automatic conversation persistence
with just 2 lines of code.
Requirements:
pip install "praisonai[tools]"
Docker Setup:
docker run -d --name praison-postgres -p 5432:5432 \
-e POSTGRES_PASSWORD=praison123 \
-e POSTGRES_DB=praisonai \
postgres:16
Environment:
export OPENAI_API_KEY=your-key
Run:
python simple_agent_postgres.py
Expected Output:
=== First Run ===
Agent: Nice to meet you, Alice! Python is a great choice...
Session ID: demo-session-001
Messages persisted: 2
"""
from praisonaiagents import Agent
from praisonai.db import PostgresDB
# Create database adapter
db = PostgresDB(
host="localhost",
port=5432,
database="praisonai",
user="postgres",
password="praison123"
)
# Create agent with persistence enabled
agent = Agent(
name="Assistant",
instructions="You are a helpful assistant. Keep responses brief.",
db=db,
session_id="demo-session-001", # Same session_id = resume conversation
verbose=False
)
# Chat with persistence
print("=== First Run ===")
response = agent.chat("My name is Alice and I love Python programming.")
print(f"Agent: {response}\n")
# Show session info
print(f"Session ID: {agent.session_id}")
print(f"Messages in history: {len(agent.chat_history)}")
# Close DB connection
db.close()
print("\n=== Session persisted! ===")
print("Run again with same session_id to resume.")
+41
View File
@@ -0,0 +1,41 @@
"""
SQLite Local Database
Use SQLite for simple local persistence without external services.
Run:
python sqlite_local.py
Expected output:
- Agent responds
- Data persisted to local SQLite file
"""
from praisonaiagents import Agent, db
import tempfile
import os
# Create SQLite database (local file)
db_path = os.path.join(tempfile.gettempdir(), "my_agent.db")
my_db = db.SQLiteDB(path=db_path)
print(f"Database: {db_path}")
# Create agent
agent = Agent(
name="LocalBot",
instructions="You are a helpful assistant.",
db=my_db,
session_id="local-session"
)
# Chat
response = agent.chat("Hello! What can you help me with?")
print(f"Response: {response}")
# Verify
data = my_db.export_session("local-session")
print(f"\nMessages stored: {len(data.get('messages', []))}")
my_db.close()
print("✅ Done")
+67
View File
@@ -0,0 +1,67 @@
"""
Redis State Store Example for PraisonAI.
Demonstrates set/get/delete + TTL operations.
Requirements:
pip install "praisonai[tools]"
Docker Setup:
docker run -d --name praison-redis -p 6379:6379 redis:7
Run:
python state_redis.py
Expected Output:
=== Redis State Store Demo ===
Set key: session:user123:preferences
Get value: {'theme': 'dark', 'language': 'en'}
TTL remaining: 3600 seconds
Hash operations: {'name': 'Alice', 'role': 'admin'}
"""
from praisonai.persistence.factory import create_state_store
# Create Redis state store
store = create_state_store("redis", url="redis://localhost:6379")
print("=== Redis State Store Demo ===")
# Basic set/get
key = "session:user123:preferences"
value = {"theme": "dark", "language": "en", "notifications": True}
store.set(key, value)
print(f"Set key: {key}")
retrieved = store.get(key)
print(f"Get value: {retrieved}")
# Set with TTL (expires in 1 hour)
ttl_key = "session:user123:token"
store.set(ttl_key, "abc123xyz", ttl=3600)
remaining = store.ttl(ttl_key)
print(f"TTL remaining: {remaining} seconds")
# Hash operations (for structured data)
hash_key = "user:123:profile"
store.hset(hash_key, "name", "Alice")
store.hset(hash_key, "role", "admin")
store.hset(hash_key, "last_login", "2024-12-24")
profile = store.hgetall(hash_key)
print(f"Hash operations: {profile}")
# Check existence
exists = store.exists(key)
print(f"Key exists: {exists}")
# Delete keys
store.delete(key)
store.delete(ttl_key)
store.delete(hash_key)
print("Cleaned up test keys")
store.close()
print("\n=== Demo Complete ===")
+69
View File
@@ -0,0 +1,69 @@
#!/usr/bin/env python3
"""Run all examples and collect PASS/SKIP/FAIL results."""
import subprocess
import sys
import os
from pathlib import Path
EXAMPLES_DIR = Path(__file__).parent
RESULTS = []
def run_example(path: Path) -> tuple:
"""Run an example and return (name, status, reason)."""
name = f"{path.parent.name}/{path.name}"
try:
result = subprocess.run(
[sys.executable, str(path)],
capture_output=True,
text=True,
timeout=60,
env={**os.environ, "PYTHONPATH": str(EXAMPLES_DIR.parent / "src")}
)
output = result.stdout + result.stderr
if "PASSED" in output:
return (name, "PASS", "")
elif "SKIPPED" in output:
return (name, "SKIP", output.split("SKIPPED:")[-1].strip()[:50])
elif result.returncode != 0:
error = output.strip().split("\n")[-1][:80]
return (name, "FAIL", error)
else:
return (name, "PASS", "")
except subprocess.TimeoutExpired:
return (name, "FAIL", "Timeout")
except Exception as e:
return (name, "FAIL", str(e)[:50])
def main():
print("=" * 70)
print("EXAMPLE RUNNER - PASS/SKIP/FAIL SUMMARY")
print("=" * 70)
# Find all example files
examples = []
for subdir in ["db", "vector", "observability", "multi_agent", "workflows"]:
subpath = EXAMPLES_DIR / subdir
if subpath.exists():
examples.extend(sorted(subpath.glob("*_wow.py")))
print(f"\nFound {len(examples)} examples to run\n")
for ex in examples:
name, status, reason = run_example(ex)
RESULTS.append((name, status, reason))
icon = {"PASS": "✅", "SKIP": "⏭️", "FAIL": "❌"}.get(status, "?")
reason_str = f" ({reason})" if reason else ""
print(f"{icon} {status:4} | {name:45} {reason_str}")
# Summary
print("\n" + "=" * 70)
passed = sum(1 for _, s, _ in RESULTS if s == "PASS")
skipped = sum(1 for _, s, _ in RESULTS if s == "SKIP")
failed = sum(1 for _, s, _ in RESULTS if s == "FAIL")
print(f"SUMMARY: {passed} PASS | {skipped} SKIP | {failed} FAIL | {len(RESULTS)} TOTAL")
print("=" * 70)
return 0 if failed == 0 else 1
if __name__ == "__main__":
sys.exit(main())
+44
View File
@@ -0,0 +1,44 @@
"""ArXiv Tool Example.
This example demonstrates how to use the ArXiv tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
Usage:
python tool.py
Note: No API key required!
"""
from praisonai_tools import ArxivTool
def main():
# Initialize ArXiv tool (no API key needed)
arxiv = ArxivTool()
# Example 1: Search ArXiv
print("=" * 60)
print("Example 1: Search ArXiv")
print("=" * 60)
results = arxiv.search("transformer neural networks", max_results=3)
if results and "error" in results[0]:
print(f"Error: {results[0]['error']}")
else:
print(f"Found {len(results)} papers:")
for paper in results:
print(f"\n Title: {paper.get('title', 'N/A')[:60]}...")
authors = paper.get('authors', [])
if authors:
print(f" Authors: {', '.join(authors[:2])}...")
print(f" Published: {paper.get('published', 'N/A')}")
print(f" PDF: {paper.get('pdf_url', 'N/A')}")
print("\n✅ ArXiv tool working correctly!")
if __name__ == "__main__":
main()
+53
View File
@@ -0,0 +1,53 @@
"""Calculator Tool Example.
This example demonstrates how to use the Calculator tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
Usage:
python tool.py
Note: No API key required!
"""
from praisonai_tools import CalculatorTool
def main():
# Initialize Calculator tool
calc = CalculatorTool()
# Example 1: Basic arithmetic
print("=" * 60)
print("Example 1: Basic Arithmetic")
print("=" * 60)
expressions = [
"2 + 2",
"10 * 5",
"100 / 4",
"2 ** 10",
"(10 + 5) * 2 - 3"
]
for expr in expressions:
result = calc.evaluate(expr)
print(f" {expr} = {result.get('result', result)}")
# Example 2: Percentage calculation
print("\n" + "=" * 60)
print("Example 2: Percentage Calculation")
print("=" * 60)
result = calc.evaluate("250 * 0.15")
print(f" 15% of 250 = {result.get('result', result)}")
result = calc.evaluate("1000 * (1 + 0.05) ** 3")
print(f" $1000 at 5% for 3 years = ${result.get('result', 0):.2f}")
print("\n✅ Calculator tool working correctly!")
if __name__ == "__main__":
main()
+75
View File
@@ -0,0 +1,75 @@
"""DuckDuckGo Search Tool Example.
This example demonstrates how to use the DuckDuckGo search tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
Usage:
python tool.py
Note: No API key required!
"""
from praisonai_tools import DuckDuckGoTool
def main():
# Initialize DuckDuckGo tool (no API key needed)
ddg = DuckDuckGoTool()
# Example 1: Web search
print("=" * 60)
print("Example 1: Web Search")
print("=" * 60)
results = ddg.search("Python programming best practices", max_results=3)
if results and "error" in results[0]:
print(f"Error: {results[0]['error']}")
else:
print(f"Found {len(results)} results:")
for r in results:
print(f" - {r['title']}")
print(f" URL: {r['url']}")
print(f" Snippet: {r['snippet'][:100]}...")
print()
# Example 2: News search
print("=" * 60)
print("Example 2: News Search")
print("=" * 60)
news = ddg.news("artificial intelligence", max_results=3)
if news and "error" in news[0]:
print(f"Error: {news[0]['error']}")
else:
print(f"Found {len(news)} news articles:")
for n in news:
print(f" - {n['title']}")
print(f" Source: {n.get('source', 'N/A')}")
print(f" Date: {n.get('date', 'N/A')}")
print()
# Example 3: Image search
print("=" * 60)
print("Example 3: Image Search")
print("=" * 60)
images = ddg.images("sunset landscape", max_results=3)
if images and "error" in images[0]:
print(f"Error: {images[0]['error']}")
else:
print(f"Found {len(images)} images:")
for img in images:
print(f" - {img['title']}")
print(f" Image URL: {img['image_url'][:60]}...")
print()
print("✅ DuckDuckGo tool working correctly!")
if __name__ == "__main__":
main()
+48
View File
@@ -0,0 +1,48 @@
"""Exa Search Tool Example.
This example demonstrates how to use the Exa search tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
export EXA_API_KEY=your_api_key
Usage:
python tool.py
"""
import os
from praisonai_tools import ExaTool
def main():
# Check for API key
if not os.getenv("EXA_API_KEY"):
print("Error: EXA_API_KEY environment variable not set")
print("Get your API key from https://exa.ai/")
return
# Initialize Exa tool
exa = ExaTool()
# Example 1: Neural search
print("=" * 60)
print("Example 1: Neural Search")
print("=" * 60)
results = exa.search("best practices for Python development", num_results=3)
if results and "error" in results[0]:
print(f"Error: {results[0]['error']}")
else:
print(f"Found {len(results)} results:")
for r in results:
print(f" - {r.get('title', 'N/A')[:50]}...")
print(f" URL: {r.get('url', 'N/A')[:60]}...")
print(f" Score: {r.get('score', 'N/A')}")
print()
print("✅ Exa tool working correctly!")
if __name__ == "__main__":
main()
+45
View File
@@ -0,0 +1,45 @@
"""Firecrawl Tool Example.
This example demonstrates how to use the Firecrawl tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
export FIRECRAWL_API_KEY=your_api_key
Usage:
python tool.py
"""
import os
from praisonai_tools import FirecrawlTool
def main():
# Check for API key
if not os.getenv("FIRECRAWL_API_KEY"):
print("Error: FIRECRAWL_API_KEY environment variable not set")
print("Get your API key from https://firecrawl.dev/")
return
# Initialize Firecrawl tool
firecrawl = FirecrawlTool()
# Example 1: Scrape a page
print("=" * 60)
print("Example 1: Scrape a Page")
print("=" * 60)
result = firecrawl.scrape("https://example.com")
if "error" in result:
print(f"Error: {result['error']}")
else:
print(f"URL: {result.get('url', 'N/A')}")
markdown = result.get('markdown', '')
print(f"Content preview: {markdown[:300]}...")
print("\n✅ Firecrawl tool working correctly!")
if __name__ == "__main__":
main()
+42
View File
@@ -0,0 +1,42 @@
"""Hacker News Tool Example.
This example demonstrates how to use the Hacker News tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
Usage:
python tool.py
Note: No API key required!
"""
from praisonai_tools import HackerNewsTool
def main():
# Initialize Hacker News tool
hn = HackerNewsTool()
# Example 1: Get top stories
print("=" * 60)
print("Example 1: Top Stories")
print("=" * 60)
stories = hn.get_top_stories(limit=5)
if stories and "error" in stories[0]:
print(f"Error: {stories[0]['error']}")
else:
print(f"Found {len(stories)} top stories:")
for story in stories:
print(f" - {story.get('title', 'N/A')[:50]}...")
print(f" Score: {story.get('score', 'N/A')} | Comments: {story.get('descendants', 'N/A')}")
print(f" URL: {story.get('url', 'N/A')[:60]}...")
print()
print("✅ Hacker News tool working correctly!")
if __name__ == "__main__":
main()
+40
View File
@@ -0,0 +1,40 @@
"""Jina Reader Tool Example.
This example demonstrates how to use the Jina Reader tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
Usage:
python tool.py
Note: Works without API key (with rate limits)
"""
from praisonai_tools import JinaTool
def main():
# Initialize Jina tool (works without API key)
jina = JinaTool()
# Example 1: Read a URL
print("=" * 60)
print("Example 1: Read URL Content")
print("=" * 60)
result = jina.read("https://example.com")
if "error" in result:
print(f"Error: {result['error']}")
else:
print(f"URL: {result.get('url', 'N/A')}")
print(f"Title: {result.get('title', 'N/A')}")
content = result.get('content', '')
print(f"Content preview: {content[:300]}...")
print("\n✅ Jina tool working correctly!")
if __name__ == "__main__":
main()
+56
View File
@@ -0,0 +1,56 @@
"""PostgreSQL Tool Example.
This example demonstrates how to use the PostgreSQL tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
Docker: docker run -d --name postgres -e POSTGRES_PASSWORD=praison123 -e POSTGRES_DB=praisonai -p 5432:5432 postgres:16
Usage:
python tool.py
"""
from praisonai_tools import PostgresTool
def main():
# Initialize PostgreSQL tool
pg = PostgresTool(
host="localhost",
port=5432,
database="praisonai",
user="postgres",
password="praison123"
)
# Example 1: List tables
print("=" * 60)
print("Example 1: List Tables")
print("=" * 60)
tables = pg.list_tables()
if isinstance(tables, dict) and "error" in tables:
print(f"Error: {tables['error']}")
else:
print(f"Found {len(tables)} tables:")
for t in tables[:10]:
print(f" - {t.get('table_name', t)}")
# Example 2: Query
print("\n" + "=" * 60)
print("Example 2: Execute Query")
print("=" * 60)
result = pg.query("SELECT version()")
if isinstance(result, dict) and "error" in result:
print(f"Error: {result['error']}")
else:
print(f"PostgreSQL version: {result}")
print("\n✅ PostgreSQL tool working correctly!")
if __name__ == "__main__":
main()
+47
View File
@@ -0,0 +1,47 @@
"""Serper Tool Example.
This example demonstrates how to use the Serper (Google Search) tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
export SERPER_API_KEY=your_api_key
Usage:
python tool.py
"""
import os
from praisonai_tools import SerperTool
def main():
# Check for API key
if not os.getenv("SERPER_API_KEY"):
print("Error: SERPER_API_KEY environment variable not set")
print("Get your API key from https://serper.dev/")
return
# Initialize Serper tool
serper = SerperTool()
# Example 1: Web search
print("=" * 60)
print("Example 1: Google Web Search")
print("=" * 60)
results = serper.search("Python programming tutorials", max_results=3)
if results and "error" in results[0]:
print(f"Error: {results[0]['error']}")
else:
print(f"Found {len(results)} results:")
for r in results:
print(f" - {r.get('title', 'N/A')[:50]}...")
print(f" URL: {r.get('link', 'N/A')[:60]}...")
print()
print("✅ Serper tool working correctly!")
if __name__ == "__main__":
main()
+57
View File
@@ -0,0 +1,57 @@
"""Tavily Search Tool Example.
This example demonstrates how to use the Tavily search tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
export TAVILY_API_KEY=your_api_key
Usage:
python tool.py
"""
import os
from praisonai_tools import TavilyTool
def main():
# Check for API key
if not os.getenv("TAVILY_API_KEY"):
print("Error: TAVILY_API_KEY environment variable not set")
print("Get your API key from https://tavily.com/")
return
# Initialize Tavily tool
tavily = TavilyTool()
# Example 1: Basic search
print("=" * 60)
print("Example 1: Basic Search")
print("=" * 60)
results = tavily.search("What is quantum computing?", max_results=3)
if "error" in results:
print(f"Error: {results['error']}")
else:
print(f"Query: {results.get('query')}")
if "answer" in results:
print(f"\nAI Answer: {results['answer'][:200]}...")
print(f"\nResults ({len(results.get('results', []))}):")
for r in results.get("results", []):
print(f" - {r['title']}")
print(f" URL: {r['url']}")
print(f" Score: {r.get('score', 'N/A')}")
# Example 2: Search context for RAG
print("\n" + "=" * 60)
print("Example 2: Search Context (for RAG)")
print("=" * 60)
context = tavily.search_context("Python best practices 2024")
print(f"Context (first 300 chars): {context[:300]}...")
print("\n✅ Tavily tool working correctly!")
if __name__ == "__main__":
main()
+69
View File
@@ -0,0 +1,69 @@
"""Wikipedia Tool Example.
This example demonstrates how to use the Wikipedia tool with PraisonAI agents.
Requirements:
pip install "praisonai[tools]"
Usage:
python tool.py
Note: No API key required!
"""
from praisonai_tools import WikipediaTool
def main():
# Initialize Wikipedia tool (no API key needed)
wiki = WikipediaTool()
# Example 1: Search Wikipedia
print("=" * 60)
print("Example 1: Search Wikipedia")
print("=" * 60)
results = wiki.search("Python programming", max_results=5)
if results and "error" in results[0]:
print(f"Error: {results[0]['error']}")
else:
print(f"Found {len(results)} results:")
for r in results:
print(f" - {r['title']}")
# Example 2: Get page summary
print("\n" + "=" * 60)
print("Example 2: Get Page Summary")
print("=" * 60)
summary = wiki.summary("Artificial intelligence", sentences=3)
if "error" in summary:
print(f"Error: {summary['error']}")
else:
print(f"Title: {summary['title']}")
print(f"Summary: {summary['summary'][:300]}...")
# Example 3: Get full page
print("\n" + "=" * 60)
print("Example 3: Get Full Page")
print("=" * 60)
page = wiki.get_page("Machine learning")
if "error" in page:
print(f"Error: {page['error']}")
if "options" in page:
print(f"Options: {page['options'][:5]}")
else:
print(f"Title: {page['title']}")
print(f"URL: {page['url']}")
print(f"Categories: {page['categories'][:3]}")
print(f"Content preview: {page['content'][:200]}...")
print("\n✅ Wikipedia tool working correctly!")
if __name__ == "__main__":
main()
+21
View File
@@ -0,0 +1,21 @@
"""ChromaDB Vector Store - Basic Test"""
import sys
sys.path.insert(0, 'src/praisonai')
from praisonai.persistence import create_knowledge_store
from praisonai.persistence.knowledge.base import KnowledgeDocument
store = create_knowledge_store("chroma", path="/tmp/chroma_demo")
try:
store.delete_collection("demo")
except Exception:
pass
store.create_collection("demo", dimension=1536)
docs = [
KnowledgeDocument(id="1", content="Python is a programming language", embedding=[0.1]*1536),
KnowledgeDocument(id="2", content="JavaScript runs in browsers", embedding=[0.2]*1536)
]
store.insert("demo", docs)
results = store.search("demo", query_embedding=[0.1]*1536, limit=1)
print(f"Found: {len(results)} results")
assert len(results) >= 1
print("PASSED: ChromaDB vector store")
+31
View File
@@ -0,0 +1,31 @@
"""LanceDB Vector Store - Local Test"""
import sys
import os
import shutil
try:
import lancedb
except ImportError:
print("SKIPPED: LanceDB - lancedb not installed")
sys.exit(0)
# Clean up previous test
db_path = "/tmp/lancedb_test"
if os.path.exists(db_path):
shutil.rmtree(db_path)
db = lancedb.connect(db_path)
# Create table with vectors
data = [
{"id": "1", "text": "Machine learning is AI", "vector": [0.1] * 128},
{"id": "2", "text": "Deep learning uses neural nets", "vector": [0.2] * 128},
]
table = db.create_table("demo", data)
# Search
results = table.search([0.1] * 128).limit(1).to_list()
print(f"Found: {len(results)} results")
print(f"Top result: {results[0]['text'][:30]}...")
assert len(results) >= 1
print("PASSED: LanceDB vector store")
+20
View File
@@ -0,0 +1,20 @@
"""LanceDB Vector Store - Basic Test"""
import sys
sys.path.insert(0, 'src/praisonai')
try:
from praisonai.persistence import create_knowledge_store
from praisonai.persistence.knowledge.base import KnowledgeDocument
store = create_knowledge_store("lancedb", path="/tmp/lancedb_demo")
store.create_collection("demo", dimension=1536)
docs = [
KnowledgeDocument(id="doc1", content="The Eiffel Tower is in Paris", embedding=[0.1]*1536),
KnowledgeDocument(id="doc2", content="The Statue of Liberty is in New York", embedding=[0.2]*1536)
]
store.insert("demo", docs)
results = store.search("demo", query_embedding=[0.1]*1536, limit=1)
print(f"Found: {len(results)} results")
assert len(results) >= 1
print("PASSED: LanceDB vector store")
except ImportError as e:
print(f"SKIPPED: LanceDB - {e}")
+59
View File
@@ -0,0 +1,59 @@
"""PGVector Vector Store - Docker Test"""
import sys
import os
# Docker: docker run -d --name pgvector -e POSTGRES_PASSWORD=postgres -p 5433:5432 pgvector/pgvector:pg16
url = os.getenv("PGVECTOR_URL", "postgresql://postgres:postgres@localhost:5433/postgres")
try:
import psycopg2
except ImportError:
print("SKIPPED: PGVector - psycopg2 not installed")
sys.exit(0)
try:
conn = psycopg2.connect(url)
cur = conn.cursor()
# Ensure extension
cur.execute("CREATE EXTENSION IF NOT EXISTS vector")
conn.commit()
# Create test table
cur.execute("DROP TABLE IF EXISTS test_vectors")
cur.execute("""
CREATE TABLE test_vectors (
id SERIAL PRIMARY KEY,
content TEXT,
embedding vector(128)
)
""")
conn.commit()
# Insert vectors
cur.execute("""
INSERT INTO test_vectors (content, embedding) VALUES
('Machine learning is AI', %s),
('Deep learning uses neural nets', %s)
""", ([0.1] * 128, [0.2] * 128))
conn.commit()
# Search (cosine similarity)
cur.execute("""
SELECT content, embedding <=> %s::vector AS distance
FROM test_vectors
ORDER BY distance
LIMIT 1
""", ([0.1] * 128,))
row = cur.fetchone()
print(f"Top result: {row[0][:30]}... (distance: {row[1]:.4f})")
# Cleanup
cur.execute("DROP TABLE test_vectors")
conn.commit()
conn.close()
print("PASSED: PGVector vector store")
except Exception as e:
print(f"SKIPPED: PGVector - {e}")
+23
View File
@@ -0,0 +1,23 @@
"""PGVector Knowledge Store - Basic Test (requires Docker)"""
import sys
import os
sys.path.insert(0, 'src/praisonai')
from praisonai.persistence import create_knowledge_store
from praisonai.persistence.knowledge.base import KnowledgeDocument
# Docker: docker run -d --name pgvector -e POSTGRES_PASSWORD=postgres -p 5433:5432 pgvector/pgvector:pg16
url = os.getenv("PGVECTOR_URL", "postgresql://postgres:postgres@localhost:5433/postgres")
try:
store = create_knowledge_store("pgvector", url=url)
store.create_collection("demo", dimension=1536)
docs = [
KnowledgeDocument(id="1", content="Kubernetes orchestrates containers", embedding=[0.1]*1536),
KnowledgeDocument(id="2", content="Docker packages applications", embedding=[0.2]*1536)
]
store.insert("demo", docs)
results = store.search("demo", query_embedding=[0.1]*1536, limit=1)
print(f"Found: {len(results)} results")
assert len(results) >= 1
print("PASSED: PGVector vector store")
except Exception as e:
print(f"SKIPPED: PGVector - {e}")
+27
View File
@@ -0,0 +1,27 @@
"""Pinecone Vector Store - Real API Test"""
import os
import sys
sys.path.insert(0, 'src/praisonai')
# Requires: export PINECONE_API_KEY=...
api_key = os.getenv("PINECONE_API_KEY")
if not api_key:
print("SKIPPED: Pinecone - PINECONE_API_KEY not set")
sys.exit(0)
from pinecone import Pinecone
pc = Pinecone(api_key=api_key)
indexes = pc.list_indexes()
print(f"Pinecone connected! Found {len(list(indexes))} indexes")
# Use existing 'test' index or create one
index_name = "test"
if index_name not in [idx.name for idx in pc.list_indexes()]:
print(f"SKIPPED: Pinecone - index '{index_name}' not found")
sys.exit(0)
index = pc.Index(index_name)
stats = index.describe_index_stats()
print(f"Index '{index_name}' has {stats.total_vector_count} vectors")
print("PASSED: Pinecone vector store")
+23
View File
@@ -0,0 +1,23 @@
"""Qdrant Vector Store - Basic Test (requires Docker)"""
import sys, os
sys.path.insert(0, 'src/praisonai')
from praisonai.persistence import create_knowledge_store
from praisonai.persistence.knowledge.base import KnowledgeDocument
# Docker: docker run -d --name qdrant -p 6333:6333 qdrant/qdrant
url = os.getenv("QDRANT_URL", "http://localhost:6333")
store = create_knowledge_store("qdrant", url=url)
try:
store.delete_collection("demo")
except Exception:
pass
store.create_collection("demo", dimension=1536)
docs = [
KnowledgeDocument(id="1", content="Machine learning is AI", embedding=[0.1]*1536),
KnowledgeDocument(id="2", content="Deep learning uses neural nets", embedding=[0.2]*1536)
]
store.insert("demo", docs)
results = store.search("demo", query_embedding=[0.1]*1536, limit=1)
print(f"Found: {len(results)} results")
assert len(results) >= 1
print("PASSED: Qdrant vector store")
+24
View File
@@ -0,0 +1,24 @@
"""Weaviate Cloud Vector Store - Real API Test"""
import os
import sys
# Requires: export WEAVIATE_URL=... and WEAVIATE_API_KEY=...
weaviate_url = os.getenv("WEAVIATE_URL", "https://thhoyczshsawyt16ot3uw.c0.europe-west3.gcp.weaviate.cloud")
weaviate_key = os.getenv("WEAVIATE_API_KEY")
if not weaviate_key:
print("SKIPPED: Weaviate - WEAVIATE_API_KEY not set")
sys.exit(0)
import weaviate
from weaviate.classes.init import Auth
client = weaviate.connect_to_weaviate_cloud(
cluster_url=weaviate_url,
auth_credentials=Auth.api_key(weaviate_key)
)
print(f"Weaviate connected: {client.is_ready()}")
collections = client.collections.list_all()
print(f"Found {len(collections)} collections")
client.close()
print("PASSED: Weaviate cloud vector store")
@@ -54,6 +54,8 @@ except ImportError:
MCP = None
from .session import Session
from .memory.memory import Memory
from .db import db
from .obs import obs
# Workflows - import from dedicated workflows module
from .workflows import (
Workflow, WorkflowStep, WorkflowContext, StepResult,
@@ -257,6 +259,8 @@ __all__ = [
'AutoAgents',
'Session',
'Memory',
'db',
'obs',
'display_interaction',
'display_self_reflection',
'display_instruction',
@@ -252,7 +252,9 @@ class Agent:
history_in_context: Optional[int] = None,
auto_save: Optional[str] = None,
skills: Optional[List[str]] = None,
skills_dirs: Optional[List[str]] = None
skills_dirs: Optional[List[str]] = None,
db: Optional[Any] = None,
session_id: Optional[str] = None
):
"""Initialize an Agent instance.
@@ -615,6 +617,11 @@ Your Goal: {self.goal}
self._skill_manager = None # Lazy loaded
self._skills_initialized = False
# Database persistence (lazy - no imports until used)
self._db = db
self._session_id = session_id
self._db_initialized = False
@property
def console(self):
"""Lazily initialize Rich Console only when needed."""
@@ -1957,7 +1964,108 @@ Your Goal: {self.goal}"""
# expand=False
# )
def _init_db_session(self):
"""Initialize DB session if db adapter is provided (lazy, first chat only)."""
if self._db is None or self._db_initialized:
return
# Generate session_id if not provided: default to per-hour ID (YYYYMMDDHH-agentname)
if self._session_id is None:
import hashlib
from datetime import datetime, timezone
# Per-hour session ID: YYYYMMDDHH (UTC) + agent name hash for uniqueness
hour_str = datetime.now(timezone.utc).strftime("%Y%m%d%H")
agent_hash = hashlib.md5(self.name.encode()).hexdigest()[:6]
self._session_id = f"{hour_str}-{agent_hash}"
# Call db adapter's on_agent_start to get previous messages
try:
history = self._db.on_agent_start(
agent_name=self.name,
session_id=self._session_id,
user_id=self.user_id,
metadata={"role": self.role, "goal": self.goal}
)
# Restore chat history from previous session
if history:
for msg in history:
self.chat_history.append({
"role": msg.role,
"content": msg.content
})
logging.info(f"Resumed session {self._session_id} with {len(history)} messages")
except Exception as e:
logging.warning(f"Failed to initialize DB session: {e}")
self._db_initialized = True
self._current_run_id = None # Track current run
def _start_run(self, input_content: str):
"""Start a new run (turn) for persistence tracking."""
if self._db is None:
return
import uuid
self._current_run_id = f"run-{uuid.uuid4().hex[:12]}"
try:
if hasattr(self._db, 'on_run_start'):
self._db.on_run_start(
session_id=self._session_id,
run_id=self._current_run_id,
input_content=input_content,
metadata={"agent_name": self.name}
)
except Exception as e:
logging.warning(f"Failed to start run: {e}")
def _end_run(self, output_content: str, status: str = "completed", metrics: dict = None):
"""End the current run (turn)."""
if self._db is None or self._current_run_id is None:
return
try:
if hasattr(self._db, 'on_run_end'):
self._db.on_run_end(
session_id=self._session_id,
run_id=self._current_run_id,
output_content=output_content,
status=status,
metrics=metrics or {},
metadata={"agent_name": self.name}
)
except Exception as e:
logging.warning(f"Failed to end run: {e}")
self._current_run_id = None
def _persist_message(self, role: str, content: str):
"""Persist a message to the DB if adapter is provided."""
if self._db is None:
return
try:
if role == "user":
self._db.on_user_message(self._session_id, content)
elif role == "assistant":
self._db.on_agent_message(self._session_id, content)
except Exception as e:
logging.warning(f"Failed to persist message: {e}")
@property
def session_id(self) -> Optional[str]:
"""Get the current session ID."""
return self._session_id
def chat(self, prompt, temperature=1.0, tools=None, output_json=None, output_pydantic=None, reasoning_steps=False, stream=None, task_name=None, task_description=None, task_id=None):
# Initialize DB session on first chat (lazy)
self._init_db_session()
# Start a new run for this chat turn
prompt_str = prompt if isinstance(prompt, str) else str(prompt)
self._start_run(prompt_str)
# Reset the final display flag for each new conversation
self._final_display_shown = False
@@ -2054,6 +2162,8 @@ Your Goal: {self.goal}"""
self.chat_history[-1].get("content") == normalized_content):
# Add user message to chat history BEFORE LLM call so handoffs can access it
self.chat_history.append({"role": "user", "content": normalized_content})
# Persist user message to DB
self._persist_message("user", normalized_content)
try:
# Pass everything to LLM class
@@ -2083,6 +2193,8 @@ Your Goal: {self.goal}"""
)
self.chat_history.append({"role": "assistant", "content": response_text})
# Persist assistant message to DB
self._persist_message("assistant", response_text)
# Log completion time if in debug mode
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
@@ -2127,6 +2239,8 @@ Your Goal: {self.goal}"""
self.chat_history[-1].get("content") == normalized_content):
# Add user message to chat history BEFORE LLM call so handoffs can access it
self.chat_history.append({"role": "user", "content": normalized_content})
# Persist user message to DB (OpenAI path)
self._persist_message("user", normalized_content)
reflection_count = 0
start_time = time.time()
@@ -2166,6 +2280,8 @@ Your Goal: {self.goal}"""
# Add to chat history and return raw response
# User message already added before LLM call via _build_messages
self.chat_history.append({"role": "assistant", "content": response_text})
# Persist assistant message to DB
self._persist_message("assistant", response_text)
# Apply guardrail validation even for JSON output
try:
validated_response = self._apply_guardrail_with_retry(response_text, original_prompt, temperature, tools, task_name, task_description, task_id)
@@ -2181,6 +2297,8 @@ Your Goal: {self.goal}"""
if not self.self_reflect:
# User message already added before LLM call via _build_messages
self.chat_history.append({"role": "assistant", "content": response_text})
# Persist assistant message to DB (non-reflect path)
self._persist_message("assistant", response_text)
if self.verbose:
logging.debug(f"Agent {self.name} final response: {response_text}")
# Return only reasoning content if reasoning_steps is True
@@ -2269,11 +2387,13 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
validated_response = self._apply_guardrail_with_retry(response_text, original_prompt, temperature, tools, task_name, task_description, task_id)
# Execute callback after validation
self._execute_callback_and_display(original_prompt, validated_response, time.time() - start_time, task_name, task_description, task_id)
self._end_run(validated_response, "completed", {"duration_ms": (time.time() - start_time) * 1000})
return validated_response
except Exception as e:
logging.error(f"Agent {self.name}: Guardrail validation failed after reflection: {e}")
# Rollback chat history on guardrail failure
self.chat_history = self.chat_history[:chat_history_length]
self._end_run(None, "error", {"error": str(e)})
return None
# Check if we've hit max reflections
@@ -0,0 +1,240 @@
"""
Observability interface for PraisonAI Agents.
This module provides the protocol and types for observability/tracing.
Implementations are provided by the wrapper layer (praisonai_tools.observability).
Usage (simplest - recommended):
from praisonaiagents import Agent, obs
agent = Agent(
name="Assistant",
observability=obs.auto(), # Auto-detect from env vars
)
agent.chat("Hello!") # Auto-traces to configured provider
Alternative (explicit provider):
observability=obs.langfuse() # Langfuse
observability=obs.langsmith() # LangSmith
observability=obs.agentops() # AgentOps
observability=obs.arize() # Arize Phoenix
observability=obs.datadog() # Datadog
"""
from typing import Any, Dict, List, Optional, Protocol, runtime_checkable
from dataclasses import dataclass, field
import time
@dataclass
class Span:
"""Represents a span in a trace."""
span_id: str
trace_id: str
name: str
parent_span_id: Optional[str] = None
start_time: float = field(default_factory=time.time)
end_time: Optional[float] = None
attributes: Dict[str, Any] = field(default_factory=dict)
status: str = "ok"
@dataclass
class Trace:
"""Represents a complete trace."""
trace_id: str
session_id: Optional[str] = None
agent_name: Optional[str] = None
user_id: Optional[str] = None
start_time: float = field(default_factory=time.time)
end_time: Optional[float] = None
spans: List[Span] = field(default_factory=list)
metadata: Dict[str, Any] = field(default_factory=dict)
status: str = "ok"
@runtime_checkable
class ObservabilityAdapter(Protocol):
"""Protocol for observability adapters."""
def on_trace_start(
self,
trace_id: str,
session_id: Optional[str] = None,
agent_name: Optional[str] = None,
user_id: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
) -> None:
"""Called when a trace starts."""
...
def on_trace_end(
self,
trace_id: str,
status: str = "ok",
metadata: Optional[Dict[str, Any]] = None,
) -> None:
"""Called when a trace ends."""
...
def on_span_start(
self,
span_id: str,
trace_id: str,
name: str,
parent_span_id: Optional[str] = None,
attributes: Optional[Dict[str, Any]] = None,
) -> None:
"""Called when a span starts."""
...
def on_span_end(
self,
span_id: str,
status: str = "ok",
attributes: Optional[Dict[str, Any]] = None,
) -> None:
"""Called when a span ends."""
...
def on_llm_call(
self,
span_id: str,
model: str,
messages: List[Dict[str, Any]],
response: Optional[str] = None,
tokens: Optional[Dict[str, int]] = None,
latency_ms: Optional[float] = None,
) -> None:
"""Called when an LLM call is made."""
...
def on_tool_call(
self,
span_id: str,
tool_name: str,
args: Dict[str, Any],
result: Any,
latency_ms: Optional[float] = None,
) -> None:
"""Called when a tool is invoked."""
...
def flush(self) -> None:
"""Flush any pending data."""
...
def close(self) -> None:
"""Close the adapter and release resources."""
...
class _LazyObsModule:
"""
Lazy proxy for observability backends.
Allows `from praisonaiagents import obs` without importing heavy deps.
Actual provider classes are loaded only when accessed.
"""
_PROVIDERS = {
"langfuse": "praisonai_tools.observability.providers.langfuse_provider",
"langsmith": "praisonai_tools.observability.providers.langsmith_provider",
"agentops": "praisonai_tools.observability.providers.agentops_provider",
"arize": "praisonai_tools.observability.providers.arize_phoenix_provider",
"phoenix": "praisonai_tools.observability.providers.arize_phoenix_provider",
"datadog": "praisonai_tools.observability.providers.datadog_provider",
"mlflow": "praisonai_tools.observability.providers.mlflow_provider",
"openlit": "praisonai_tools.observability.providers.openlit_provider",
"opik": "praisonai_tools.observability.providers.opik_provider",
"portkey": "praisonai_tools.observability.providers.portkey_provider",
"traceloop": "praisonai_tools.observability.providers.traceloop_provider",
"weave": "praisonai_tools.observability.providers.weave_provider",
"braintrust": "praisonai_tools.observability.providers.braintrust_provider",
"langtrace": "praisonai_tools.observability.providers.langtrace_provider",
"langwatch": "praisonai_tools.observability.providers.langwatch_provider",
"maxim": "praisonai_tools.observability.providers.maxim_provider",
}
def __getattr__(self, name: str):
if name in self._PROVIDERS:
return self._create_provider_factory(name)
raise AttributeError(f"module 'obs' has no attribute {name!r}")
def _create_provider_factory(self, provider_name: str):
"""Create a factory function for a provider."""
def factory(**kwargs):
try:
import importlib
module = importlib.import_module(self._PROVIDERS[provider_name])
# Get the provider class (naming convention: XxxProvider)
class_name = f"{provider_name.title().replace('_', '')}Provider"
provider_cls = getattr(module, class_name, None)
if provider_cls:
return provider_cls(**kwargs)
# Fallback: try to find any class ending with Provider
for attr_name in dir(module):
if attr_name.endswith("Provider"):
return getattr(module, attr_name)(**kwargs)
raise AttributeError(f"No provider class found in {module}")
except ImportError as e:
raise ImportError(
f"Observability provider '{provider_name}' requires praisonai-tools. "
f"Install with: pip install praisonai-tools\n"
f"Original error: {e}"
) from e
return factory
def auto(self, **kwargs):
"""
Auto-detect observability provider from environment variables.
Checks for common env vars like LANGFUSE_PUBLIC_KEY, LANGSMITH_API_KEY, etc.
Returns None if no provider is detected.
"""
import os
# Check for common provider env vars
provider_env_map = {
"LANGFUSE_PUBLIC_KEY": "langfuse",
"LANGSMITH_API_KEY": "langsmith",
"AGENTOPS_API_KEY": "agentops",
"ARIZE_API_KEY": "arize",
"DATADOG_API_KEY": "datadog",
"MLFLOW_TRACKING_URI": "mlflow",
"OPENLIT_API_KEY": "openlit",
"OPIK_API_KEY": "opik",
"PORTKEY_API_KEY": "portkey",
"TRACELOOP_API_KEY": "traceloop",
"WANDB_API_KEY": "weave",
"BRAINTRUST_API_KEY": "braintrust",
}
for env_var, provider in provider_env_map.items():
if os.getenv(env_var):
return self._create_provider_factory(provider)(**kwargs)
return None
def __repr__(self):
return "<module 'praisonaiagents.obs' (lazy observability backends)>"
def __call__(self, provider: str = "auto", **kwargs):
"""
Shortcut: obs("langfuse") or obs() for auto-detection.
"""
if provider == "auto":
return self.auto(**kwargs)
return self._create_provider_factory(provider)(**kwargs)
# Singleton lazy module instance
obs = _LazyObsModule()
__all__ = [
"ObservabilityAdapter",
"Span",
"Trace",
"obs",
]
+1 -1
View File
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "praisonaiagents"
version = "0.2.0"
version = "0.2.2"
description = "Praison AI agents for completing complex tasks with Self Reflection Agents"
readme = "README.md"
requires-python = ">=3.10"
+1 -1
View File
@@ -2422,7 +2422,7 @@ wheels = [
[[package]]
name = "praisonaiagents"
version = "0.2.0"
version = "0.2.2"
source = { editable = "." }
dependencies = [
{ name = "aiohttp" },
+287 -1
View File
@@ -261,6 +261,12 @@ npm install praisonai
| ↳ 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
| ↳ Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
| ↳ Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
| ↳ Shadow Git Checkpoints | [Example](#10-shadow-git-checkpoints) | [📖](https://docs.praison.ai/features/checkpoints) |
| ↳ Background Tasks | [Example](examples/background/basic_background.py) | [📖](https://docs.praison.ai/features/background-tasks) |
| ↳ Policy Engine | [Example](examples/policy/basic_policy.py) | [📖](https://docs.praison.ai/features/policy-engine) |
| ↳ Thinking Budgets | [Example](examples/thinking/basic_thinking.py) | [📖](https://docs.praison.ai/features/thinking-budgets) |
| ↳ Output Styles | [Example](examples/output/basic_output.py) | [📖](https://docs.praison.ai/features/output-styles) |
| ↳ Context Compaction | [Example](examples/compaction/basic_compaction.py) | [📖](https://docs.praison.ai/features/context-compaction) |
| **📊 Monitoring & Management** | | |
| ↳ Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
| ↳ Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
@@ -930,8 +936,191 @@ praisonai checkpoint diff # Show changes
praisonai checkpoint restore abc123 # Restore to checkpoint
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/checkpoints)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/checkpoint)
- [💻 Example](examples/checkpoints/basic_checkpoints.py)
### 11. Field Names Reference (A-I-G-S)
---
### 11. Background Tasks
Run agent tasks asynchronously without blocking:
```python
import asyncio
from praisonaiagents.background import BackgroundRunner, BackgroundConfig
async def main():
config = BackgroundConfig(max_concurrent_tasks=3)
runner = BackgroundRunner(config=config)
async def my_task(name: str) -> str:
await asyncio.sleep(2)
return f"Task {name} completed"
task = await runner.submit(my_task, args=("example",), name="my_task")
await task.wait(timeout=10.0)
print(task.result)
asyncio.run(main())
```
**CLI Commands:**
```bash
praisonai background list # List running tasks
praisonai background status <id> # Check task status
praisonai background cancel <id> # Cancel a task
praisonai background clear # Clear completed tasks
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/background-tasks)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/background)
- [💻 Example](examples/background/basic_background.py)
---
### 12. Policy Engine
Control what agents can and cannot do with policy-based execution:
```python
from praisonaiagents.policy import (
PolicyEngine, Policy, PolicyRule, PolicyAction
)
engine = PolicyEngine()
policy = Policy(
name="no_delete",
rules=[
PolicyRule(
action=PolicyAction.DENY,
resource="tool:delete_*",
reason="Delete operations blocked"
)
]
)
engine.add_policy(policy)
result = engine.check("tool:delete_file", {})
print(f"Allowed: {result.allowed}")
```
**CLI Commands:**
```bash
praisonai policy list # List policies
praisonai policy check "tool:name" # Check if allowed
praisonai policy init # Create template
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/policy-engine)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/policy)
- [💻 Example](examples/policy/basic_policy.py)
---
### 13. Thinking Budgets
Configure token budgets for extended thinking:
```python
from praisonaiagents.thinking import ThinkingBudget, ThinkingTracker
# Use predefined levels
budget = ThinkingBudget.high() # 16,000 tokens
# Track usage
tracker = ThinkingTracker()
session = tracker.start_session(budget_tokens=16000)
tracker.end_session(session, tokens_used=12000)
summary = tracker.get_summary()
print(f"Utilization: {summary['average_utilization']:.1%}")
```
**CLI Commands:**
```bash
praisonai thinking status # Show current budget
praisonai thinking set high # Set budget level
praisonai thinking stats # Show usage statistics
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/thinking-budgets)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/thinking)
- [💻 Example](examples/thinking/basic_thinking.py)
---
### 14. Output Styles
Configure how agents format their responses:
```python
from praisonaiagents.output import OutputStyle, OutputFormatter
# Use preset styles
style = OutputStyle.concise()
formatter = OutputFormatter(style)
# Format output
text = "# Hello\n\nThis is **bold** text."
plain = formatter.format(text)
print(plain)
```
**CLI Commands:**
```bash
praisonai output status # Show current style
praisonai output set concise # Set output style
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/output-styles)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/output-style)
- [💻 Example](examples/output/basic_output.py)
---
### 15. Context Compaction
Automatically manage context window size:
```python
from praisonaiagents.compaction import (
ContextCompactor, CompactionStrategy
)
compactor = ContextCompactor(
max_tokens=4000,
strategy=CompactionStrategy.SLIDING,
preserve_recent=3
)
messages = [...] # Your conversation history
compacted, result = compactor.compact(messages)
print(f"Compression: {result.compression_ratio:.1%}")
```
**CLI Commands:**
```bash
praisonai compaction status # Show settings
praisonai compaction set sliding # Set strategy
praisonai compaction stats # Show statistics
```
**Links:**
- [📖 Coding Docs](https://docs.praison.ai/features/context-compaction)
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/compaction)
- [💻 Example](examples/compaction/basic_compaction.py)
---
### 16. Field Names Reference (A-I-G-S)
PraisonAI accepts both old (agents.yaml) and new (workflow.yaml) field names. Use the **canonical names** for new projects:
@@ -2535,6 +2724,103 @@ PraisonAI provides zero-dependency persistent memory for agents. For detailed ex
---
## 💾 Persistence (Databases)
Enable automatic conversation persistence with 2 lines of code:
```python
from praisonaiagents import Agent, db
agent = Agent(
name="Assistant",
db=db(database_url="postgresql://localhost/mydb"), # db(...) shortcut
session_id="my-session" # Optional: defaults to per-hour ID (YYYYMMDDHH)
)
agent.chat("Hello!") # Auto-persists messages, runs, traces
```
### Persistence CLI Commands
| Command | Description |
|---------|-------------|
| `praisonai persistence doctor` | Validate DB connectivity |
| `praisonai persistence run` | Run agent with persistence |
| `praisonai persistence resume` | Resume existing session |
| `praisonai persistence export` | Export session to JSONL |
| `praisonai persistence import` | Import session from JSONL |
| `praisonai persistence migrate` | Apply schema migrations |
| `praisonai persistence status` | Show schema status |
### Databases Table
| Database | Store Type | Install | Example | Docs |
|----------|------------|---------|---------|------|
| PostgreSQL | Conversation | `pip install "praisonai[tools]"` | [simple_db_agent.py](examples/persistence/simple_db_agent.py) | [docs](https://docs.praison.ai/docs/databases/postgres) |
| MySQL | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| SQLite | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| SingleStore | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Supabase | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| SurrealDB | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Qdrant | Knowledge | `pip install "praisonai[tools]"` | [knowledge_qdrant.py](examples/persistence/knowledge_qdrant.py) | [docs](https://docs.praison.ai/docs/databases/qdrant) |
| ChromaDB | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Pinecone | Knowledge | `pip install pinecone` | [pinecone_wow.py](examples/vector/pinecone_wow.py) | [docs](https://docs.praison.ai/docs/databases/pinecone) |
| Weaviate | Knowledge | `pip install weaviate-client` | [weaviate_wow.py](examples/vector/weaviate_wow.py) | [docs](https://docs.praison.ai/docs/databases/weaviate) |
| LanceDB | Knowledge | `pip install lancedb` | [lancedb_real_wow.py](examples/vector/lancedb_real_wow.py) | [docs](https://docs.praison.ai/docs/databases/lancedb) |
| Milvus | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| PGVector | Knowledge | `pip install psycopg2-binary` | [pgvector_real_wow.py](examples/vector/pgvector_real_wow.py) | [docs](https://docs.praison.ai/docs/databases/pgvector) |
| Redis Vector | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Cassandra | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| ClickHouse | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Redis | State | `pip install "praisonai[tools]"` | [state_redis.py](examples/persistence/state_redis.py) | [docs](https://docs.praison.ai/docs/databases/redis) |
| MongoDB | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| DynamoDB | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Firestore | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Upstash | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
| Memory | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
---
## 🔧 Tools Table
Install all tools with: `pip install "praisonai[tools]"`
| Tool | Category | Import | Docs |
|------|----------|--------|------|
| Tavily | Web Search | `from praisonai_tools import TavilyTool` | [docs](https://docs.praison.ai/docs/tools/external/tavily) |
| DuckDuckGo | Web Search | `from praisonai_tools import DuckDuckGoTool` | [docs](https://docs.praison.ai/docs/tools/external/duckduckgo) |
| Exa | Web Search | `from praisonai_tools import ExaTool` | [docs](https://docs.praison.ai/docs/tools/external/exa) |
| Serper | Web Search | `from praisonai_tools import SerperTool` | [docs](https://docs.praison.ai/docs/tools/external/serper) |
| Jina | Web Reader | `from praisonai_tools import JinaTool` | [docs](https://docs.praison.ai/docs/tools/external/jina) |
| Firecrawl | Web Scraping | `from praisonai_tools import FirecrawlTool` | [docs](https://docs.praison.ai/docs/tools/external/firecrawl) |
| Crawl4AI | Web Scraping | `from praisonai_tools import Crawl4AITool` | [docs](https://docs.praison.ai/docs/tools/external/crawl4ai) |
| Wikipedia | Knowledge | `from praisonai_tools import WikipediaTool` | [docs](https://docs.praison.ai/docs/tools/external/wikipedia) |
| ArXiv | Research | `from praisonai_tools import ArxivTool` | [docs](https://docs.praison.ai/docs/tools/external/arxiv) |
| HackerNews | News | `from praisonai_tools import HackerNewsTool` | [docs](https://docs.praison.ai/docs/tools/external/hackernews) |
| YouTube | Media | `from praisonai_tools import YouTubeTool` | [docs](https://docs.praison.ai/docs/tools/external/youtube) |
| Weather | Data | `from praisonai_tools import WeatherTool` | [docs](https://docs.praison.ai/docs/tools/external/weather) |
| PostgreSQL | Database | `from praisonai_tools import PostgresTool` | [docs](https://docs.praison.ai/docs/tools/external/postgres) |
| MySQL | Database | `from praisonai_tools import MySQLTool` | [docs](https://docs.praison.ai/docs/tools/external/mysql) |
| SQLite | Database | `from praisonai_tools import SQLiteTool` | [docs](https://docs.praison.ai/docs/tools/external/sqlite) |
| MongoDB | Database | `from praisonai_tools import MongoDBTool` | [docs](https://docs.praison.ai/docs/tools/external/mongodb) |
| Redis | Database | `from praisonai_tools import RedisTool` | [docs](https://docs.praison.ai/docs/tools/external/redis) |
| Qdrant | Vector DB | `from praisonai_tools import QdrantTool` | [docs](https://docs.praison.ai/docs/tools/external/qdrant) |
| GitHub | DevOps | `from praisonai_tools import GitHubTool` | [docs](https://docs.praison.ai/docs/tools/external/github) |
| Slack | Communication | `from praisonai_tools import SlackTool` | [docs](https://docs.praison.ai/docs/tools/external/slack) |
| Discord | Communication | `from praisonai_tools import DiscordTool` | [docs](https://docs.praison.ai/docs/tools/external/discord) |
| Telegram | Communication | `from praisonai_tools import TelegramTool` | [docs](https://docs.praison.ai/docs/tools/external/telegram) |
| Email | Communication | `from praisonai_tools import EmailTool` | [docs](https://docs.praison.ai/docs/tools/external/email) |
| Notion | Productivity | `from praisonai_tools import NotionTool` | [docs](https://docs.praison.ai/docs/tools/external/notion) |
| File | File System | `from praisonai_tools import FileTool` | [docs](https://docs.praison.ai/docs/tools/external/file) |
| Shell | System | `from praisonai_tools import ShellTool` | [docs](https://docs.praison.ai/docs/tools/external/shell) |
| Python | Code | `from praisonai_tools import PythonTool` | [docs](https://docs.praison.ai/docs/tools/external/python) |
| JSON | Data | `from praisonai_tools import JSONTool` | [docs](https://docs.praison.ai/docs/tools/external/json) |
| CSV | Data | `from praisonai_tools import CSVTool` | [docs](https://docs.praison.ai/docs/tools/external/csv) |
| Calculator | Math | `from praisonai_tools import CalculatorTool` | [docs](https://docs.praison.ai/docs/tools/external/calculator) |
> See [full tools documentation](https://docs.praison.ai/docs/tools/tools) for all 100+ available tools.
---
## 🎓 Video Tutorials
Learn PraisonAI through our comprehensive video series:
+2 -2
View File
@@ -3,8 +3,8 @@ class Praisonai < Formula
desc "AI tools for various AI applications"
homepage "https://github.com/MervinPraison/PraisonAI"
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.4.0.tar.gz"
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.4.0.tar.gz | shasum -a 256`.split.first
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.4.4.tar.gz"
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.4.4.tar.gz | shasum -a 256`.split.first
license "MIT"
depends_on "[email protected]"
@@ -0,0 +1,293 @@
"""
Observability CLI Handler
CLI commands for managing observability providers.
Usage: praisonai obs <command>
"""
import os
from typing import Any, Dict, List, Optional, Tuple
from .base import FlagHandler
class ObservabilityHandler(FlagHandler):
"""
Handler for observability CLI commands.
Commands:
praisonai obs list - List available providers
praisonai obs doctor - Check provider connectivity
praisonai obs init <name> - Initialize a provider
"""
@property
def feature_name(self) -> str:
return "observability"
@property
def flag_name(self) -> str:
return "obs"
@property
def flag_help(self) -> str:
return "Observability commands (list, doctor, init)"
def check_dependencies(self) -> Tuple[bool, str]:
"""Check if observability module is available."""
try:
from praisonai_tools.observability import obs
return True, ""
except ImportError:
return False, "Observability requires praisonai-tools. Install with: pip install praisonai-tools"
def list_providers(self) -> Dict[str, Any]:
"""List all available observability providers."""
result = {
"registered": [],
"available": [],
"configured": [],
}
try:
from praisonai_tools.observability.manager import ObservabilityManager
from praisonai_tools.observability.config import PROVIDER_ENV_KEYS
mgr = ObservabilityManager()
# Get all registered providers
result["registered"] = mgr.list_providers()
# Check which have env vars configured
for provider, keys in PROVIDER_ENV_KEYS.items():
if all(os.getenv(key) for key in keys):
result["configured"].append(provider)
# Check which SDKs are available
for provider in result["registered"]:
try:
mgr._load_provider(provider)
if provider in mgr._providers:
provider_class = mgr._providers[provider]
instance = provider_class()
if instance.is_available():
result["available"].append(provider)
except Exception:
pass
except ImportError:
pass
return result
def doctor(self, provider: Optional[str] = None) -> Dict[str, Any]:
"""Run diagnostics on observability setup."""
try:
from praisonai_tools.observability import obs
if provider:
obs.init(provider=provider)
return obs.doctor()
except ImportError:
return {
"error": "praisonai-tools not installed",
"enabled": False,
}
def init_provider(self, provider: str, **kwargs) -> bool:
"""Initialize a specific provider."""
try:
from praisonai_tools.observability import obs
return obs.init(provider=provider, **kwargs)
except ImportError:
return False
def execute(self, action: str = "list", **kwargs) -> Any:
"""
Execute observability command.
Args:
action: Command to execute (list, doctor, init)
**kwargs: Additional arguments
Returns:
Command result
"""
if action == "list":
return self._execute_list()
elif action == "doctor":
return self._execute_doctor(kwargs.get("provider"))
elif action == "init":
return self._execute_init(kwargs.get("provider"), **kwargs)
else:
self.print_status(f"Unknown action: {action}", "error")
return None
def _execute_list(self) -> Dict[str, Any]:
"""Execute list command."""
result = self.list_providers()
self.print_status("\n📊 Observability Providers", "info")
self.print_status("=" * 40, "info")
# Provider status table
all_providers = [
"agentops", "langfuse", "langsmith", "traceloop",
"arize_phoenix", "openlit", "langtrace", "langwatch",
"datadog", "mlflow", "opik", "portkey",
"braintrust", "maxim", "weave", "neatlogs",
"langdb", "atla", "patronus", "truefoundry",
]
for provider in all_providers:
status_parts = []
if provider in result.get("available", []):
status_parts.append("✅ SDK")
else:
status_parts.append("❌ SDK")
if provider in result.get("configured", []):
status_parts.append("🔑 Keys")
else:
status_parts.append("⚪ Keys")
status = " | ".join(status_parts)
self.print_status(f" {provider:20} {status}", "info")
self.print_status("\n✅ = Available ❌ = Not installed 🔑 = API key configured", "info")
return result
def _execute_doctor(self, provider: Optional[str] = None) -> Dict[str, Any]:
"""Execute doctor command."""
result = self.doctor(provider)
self.print_status("\n🩺 Observability Diagnostics", "info")
self.print_status("=" * 40, "info")
self.print_status(f" Enabled: {result.get('enabled', False)}", "info")
self.print_status(f" Provider: {result.get('provider', 'None')}", "info")
if result.get("connection_status") is not None:
status = "✅" if result["connection_status"] else "❌"
self.print_status(f" Connection: {status} {result.get('connection_message', '')}", "info")
if result.get("available_providers"):
self.print_status(f" Available: {', '.join(result['available_providers'])}", "info")
return result
def _execute_init(self, provider: str, **kwargs) -> bool:
"""Execute init command."""
if not provider:
self.print_status("Provider name required. Usage: praisonai obs init <provider>", "error")
return False
success = self.init_provider(provider, **kwargs)
if success:
self.print_status(f"✅ {provider} initialized successfully", "success")
else:
self.print_status(f"❌ Failed to initialize {provider}", "error")
self.print_status(f" Check that {provider} SDK is installed and API keys are set", "info")
return success
def apply_to_agent_config(self, config: Dict[str, Any], flag_value: Any) -> Dict[str, Any]:
"""Apply observability to agent config."""
if flag_value:
try:
from praisonai_tools.observability import obs
if isinstance(flag_value, str):
obs.init(provider=flag_value)
else:
obs.init()
self.print_status("📊 Observability enabled", "success")
except ImportError:
self.print_status("Observability requires praisonai-tools", "warning")
return config
# Provider environment variable reference
PROVIDER_SETUP_GUIDE = {
"agentops": {
"env_vars": ["AGENTOPS_API_KEY"],
"install": "pip install agentops",
"docs": "https://agentops.ai/",
},
"langfuse": {
"env_vars": ["LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY"],
"install": "pip install opentelemetry-sdk opentelemetry-exporter-otlp",
"docs": "https://langfuse.com/",
},
"langsmith": {
"env_vars": ["LANGSMITH_API_KEY"],
"install": "pip install opentelemetry-sdk opentelemetry-exporter-otlp",
"docs": "https://smith.langchain.com/",
},
"traceloop": {
"env_vars": ["TRACELOOP_API_KEY"],
"install": "pip install traceloop-sdk",
"docs": "https://traceloop.com/",
},
"arize_phoenix": {
"env_vars": ["PHOENIX_API_KEY"],
"install": "pip install arize-phoenix openinference-instrumentation",
"docs": "https://phoenix.arize.com/",
},
"openlit": {
"env_vars": [],
"install": "pip install openlit",
"docs": "https://github.com/openlit/openlit",
},
"langtrace": {
"env_vars": ["LANGTRACE_API_KEY"],
"install": "pip install langtrace-python-sdk",
"docs": "https://langtrace.ai/",
},
"langwatch": {
"env_vars": ["LANGWATCH_API_KEY"],
"install": "pip install langwatch",
"docs": "https://langwatch.ai/",
},
"datadog": {
"env_vars": ["DD_API_KEY"],
"install": "pip install ddtrace",
"docs": "https://www.datadoghq.com/product/llm-observability/",
},
"mlflow": {
"env_vars": ["MLFLOW_TRACKING_URI"],
"install": "pip install mlflow",
"docs": "https://mlflow.org/",
},
"opik": {
"env_vars": ["OPIK_API_KEY"],
"install": "pip install opik",
"docs": "https://www.comet.com/docs/opik/",
},
"portkey": {
"env_vars": ["PORTKEY_API_KEY"],
"install": "pip install portkey-ai",
"docs": "https://portkey.ai/",
},
"braintrust": {
"env_vars": ["BRAINTRUST_API_KEY"],
"install": "pip install braintrust",
"docs": "https://www.braintrust.dev/",
},
"maxim": {
"env_vars": ["MAXIM_API_KEY"],
"install": "pip install maxim-py",
"docs": "https://getmaxim.ai/",
},
"weave": {
"env_vars": ["WANDB_API_KEY"],
"install": "pip install weave",
"docs": "https://weave-docs.wandb.ai/",
},
}
@@ -0,0 +1,637 @@
"""
Persistence CLI feature for PraisonAI.
Provides commands for database persistence configuration and management:
- praisonai persistence doctor - Validate DB connectivity
- praisonai persistence run - Run agent with persistence
- praisonai persistence resume - Resume a session
"""
import os
import sys
import logging
from typing import Optional
logger = logging.getLogger(__name__)
def handle_persistence_command(args: list) -> bool:
"""
Handle persistence subcommands.
Args:
args: Command line arguments after 'persistence'
Returns:
True if command was handled, False otherwise
"""
if not args:
print_persistence_help()
return True
subcommand = args[0].lower()
sub_args = args[1:]
if subcommand == "doctor":
return handle_doctor(sub_args)
elif subcommand == "run":
return handle_run(sub_args)
elif subcommand == "resume":
return handle_resume(sub_args)
elif subcommand == "export":
return handle_export(sub_args)
elif subcommand == "import":
return handle_import(sub_args)
elif subcommand == "migrate":
return handle_migrate(sub_args)
elif subcommand == "status":
return handle_status(sub_args)
elif subcommand in ("--help", "-h", "help"):
print_persistence_help()
return True
else:
print(f"Unknown persistence subcommand: {subcommand}")
print_persistence_help()
return True
def print_persistence_help():
"""Print persistence command help."""
help_text = """
PraisonAI Persistence Commands
==============================
Usage: praisonai persistence <command> [options]
Commands:
doctor Validate database connectivity and configuration
run Run an agent with persistence enabled
resume Resume an existing session
Doctor Command:
praisonai persistence doctor [options]
Options:
--conversation-url URL Conversation store URL (postgres/mysql/sqlite)
--knowledge-url URL Knowledge store URL (qdrant/chroma/pinecone)
--state-url URL State store URL (redis/memory)
--all Test all configured stores
Environment Variables:
PRAISON_CONVERSATION_URL Default conversation store URL
PRAISON_KNOWLEDGE_URL Default knowledge store URL
PRAISON_STATE_URL Default state store URL
Run Command:
praisonai persistence run [options] "prompt"
Options:
--session-id ID Session identifier (auto-generated if not provided)
--user-id ID User identifier (default: "default")
--conversation-url URL Conversation store URL
--knowledge-url URL Knowledge store URL
--state-url URL State store URL
--agent-name NAME Agent name (default: "Assistant")
--agent-instructions TEXT Agent instructions
--dry-run Show configuration without running
Resume Command:
praisonai persistence resume --session-id ID [options]
Options:
--session-id ID Session to resume (required)
--conversation-url URL Conversation store URL
--show-history Display conversation history
--continue "prompt" Continue with a new prompt
Export/Import Commands:
praisonai persistence export --session-id ID [options]
praisonai persistence import --file FILE [options]
Options:
--session-id ID Session to export (required for export)
--conversation-url URL Conversation store URL
--file FILE File path for export/import (JSONL format)
--output FILE Output file path (default: session-{id}.jsonl)
Examples:
# Check PostgreSQL connectivity
praisonai persistence doctor --conversation-url "postgresql://localhost/mydb"
# Run agent with persistence
praisonai persistence run --session-id "my-session" "Hello, remember my name is Alice"
# Resume a session
praisonai persistence resume --session-id "my-session" --show-history
# Continue a session
praisonai persistence resume --session-id "my-session" --continue "What's my name?"
# Export a session
praisonai persistence export --session-id "my-session" --output session.jsonl
# Import a session
praisonai persistence import --file session.jsonl
"""
print(help_text)
def parse_persistence_args(args: list) -> dict:
"""Parse persistence command arguments."""
config = {
"session_id": None,
"user_id": "default",
"conversation_url": os.getenv("PRAISON_CONVERSATION_URL"),
"knowledge_url": os.getenv("PRAISON_KNOWLEDGE_URL"),
"state_url": os.getenv("PRAISON_STATE_URL"),
"agent_name": "Assistant",
"agent_instructions": "You are a helpful assistant.",
"dry_run": False,
"show_history": False,
"all_stores": False,
"prompt": None,
"continue_prompt": None,
"output_file": None,
"input_file": None,
}
i = 0
while i < len(args):
arg = args[i]
if arg == "--session-id" and i + 1 < len(args):
config["session_id"] = args[i + 1]
i += 2
elif arg == "--user-id" and i + 1 < len(args):
config["user_id"] = args[i + 1]
i += 2
elif arg == "--conversation-url" and i + 1 < len(args):
config["conversation_url"] = args[i + 1]
i += 2
elif arg == "--knowledge-url" and i + 1 < len(args):
config["knowledge_url"] = args[i + 1]
i += 2
elif arg == "--state-url" and i + 1 < len(args):
config["state_url"] = args[i + 1]
i += 2
elif arg == "--agent-name" and i + 1 < len(args):
config["agent_name"] = args[i + 1]
i += 2
elif arg == "--agent-instructions" and i + 1 < len(args):
config["agent_instructions"] = args[i + 1]
i += 2
elif arg == "--dry-run":
config["dry_run"] = True
i += 1
elif arg == "--show-history":
config["show_history"] = True
i += 1
elif arg == "--all":
config["all_stores"] = True
i += 1
elif arg == "--continue" and i + 1 < len(args):
config["continue_prompt"] = args[i + 1]
i += 2
elif arg == "--output" and i + 1 < len(args):
config["output_file"] = args[i + 1]
i += 2
elif arg == "--file" and i + 1 < len(args):
config["input_file"] = args[i + 1]
i += 2
elif not arg.startswith("-"):
config["prompt"] = arg
i += 1
else:
i += 1
return config
def handle_doctor(args: list) -> bool:
"""Handle persistence doctor command."""
config = parse_persistence_args(args)
print("=" * 50)
print("PraisonAI Persistence Doctor")
print("=" * 50)
results = []
# Test conversation store
if config["conversation_url"] or config["all_stores"]:
url = config["conversation_url"]
if url:
print(f"\n[Conversation Store] Testing: {_mask_url(url)}")
success, msg = _test_conversation_store(url)
results.append(("Conversation", success, msg))
print(f" {'✅' if success else '❌'} {msg}")
else:
print("\n[Conversation Store] Not configured (set --conversation-url or PRAISON_CONVERSATION_URL)")
# Test knowledge store
if config["knowledge_url"] or config["all_stores"]:
url = config["knowledge_url"]
if url:
print(f"\n[Knowledge Store] Testing: {_mask_url(url)}")
success, msg = _test_knowledge_store(url)
results.append(("Knowledge", success, msg))
print(f" {'✅' if success else '❌'} {msg}")
else:
print("\n[Knowledge Store] Not configured (set --knowledge-url or PRAISON_KNOWLEDGE_URL)")
# Test state store
if config["state_url"] or config["all_stores"]:
url = config["state_url"]
if url:
print(f"\n[State Store] Testing: {_mask_url(url)}")
success, msg = _test_state_store(url)
results.append(("State", success, msg))
print(f" {'✅' if success else '❌'} {msg}")
else:
print("\n[State Store] Not configured (set --state-url or PRAISON_STATE_URL)")
# Summary
print("\n" + "=" * 50)
if results:
passed = sum(1 for _, s, _ in results if s)
total = len(results)
print(f"Results: {passed}/{total} stores connected successfully")
else:
print("No stores configured. Use --conversation-url, --knowledge-url, or --state-url")
print("=" * 50)
return True
def handle_run(args: list) -> bool:
"""Handle persistence run command."""
config = parse_persistence_args(args)
if not config["prompt"]:
print("Error: No prompt provided")
print("Usage: praisonai persistence run [options] \"your prompt\"")
return True
if not config["conversation_url"]:
print("Error: No conversation URL configured")
print("Set --conversation-url or PRAISON_CONVERSATION_URL")
return True
if config["dry_run"]:
print("=" * 50)
print("Dry Run - Configuration")
print("=" * 50)
print(f"Session ID: {config['session_id'] or '(auto-generated)'}")
print(f"User ID: {config['user_id']}")
print(f"Conversation URL: {_mask_url(config['conversation_url'])}")
print(f"Agent Name: {config['agent_name']}")
print(f"Prompt: {config['prompt'][:50]}...")
return True
# Run agent with persistence
try:
from praisonai.db import PraisonDB
from praisonaiagents import Agent
db = PraisonDB(
database_url=config["conversation_url"],
state_url=config["state_url"],
knowledge_url=config["knowledge_url"],
)
agent = Agent(
name=config["agent_name"],
instructions=config["agent_instructions"],
db=db,
session_id=config["session_id"],
verbose=True
)
print(f"\n[Session: {agent.session_id}]")
response = agent.chat(config["prompt"])
print(f"\nAgent: {response}")
db.close()
except Exception as e:
print(f"Error: {e}")
return True
return True
def handle_resume(args: list) -> bool:
"""Handle persistence resume command."""
config = parse_persistence_args(args)
if not config["session_id"]:
print("Error: --session-id is required for resume")
return True
if not config["conversation_url"]:
print("Error: No conversation URL configured")
print("Set --conversation-url or PRAISON_CONVERSATION_URL")
return True
try:
from praisonai.db import PraisonDB
db = PraisonDB(database_url=config["conversation_url"])
# Get session history
history = db.on_agent_start(
agent_name="Resume",
session_id=config["session_id"],
user_id=config["user_id"]
)
print(f"\n[Session: {config['session_id']}]")
print(f"Messages in history: {len(history)}")
if config["show_history"]:
print("\n--- Conversation History ---")
for msg in history:
role = msg.role.upper()
content = msg.content[:100] + "..." if len(msg.content) > 100 else msg.content
print(f"[{role}] {content}")
print("--- End History ---\n")
if config["continue_prompt"]:
from praisonaiagents import Agent
agent = Agent(
name=config["agent_name"],
instructions=config["agent_instructions"],
db=db,
session_id=config["session_id"],
verbose=True
)
response = agent.chat(config["continue_prompt"])
print(f"\nAgent: {response}")
db.close()
except Exception as e:
print(f"Error: {e}")
return True
return True
def handle_export(args: list) -> bool:
"""Handle persistence export command."""
config = parse_persistence_args(args)
if not config["session_id"]:
print("Error: --session-id is required for export")
return True
if not config["conversation_url"]:
print("Error: No conversation URL configured")
return True
try:
from praisonai.db import PraisonDB
import json
db = PraisonDB(database_url=config["conversation_url"])
# Export session
data = db.export_session(config["session_id"])
if not data:
print(f"Error: Session {config['session_id']} not found")
db.close()
return True
# Determine output file
output_file = config.get("output_file") or f"session-{config['session_id']}.jsonl"
with open(output_file, 'w') as f:
f.write(json.dumps(data, default=str) + '\n')
print(f"Exported session to: {output_file}")
print(f"Messages: {len(data.get('messages', []))}")
db.close()
except Exception as e:
print(f"Error: {e}")
return True
return True
def handle_import(args: list) -> bool:
"""Handle persistence import command."""
config = parse_persistence_args(args)
input_file = config.get("input_file")
if not input_file:
print("Error: --file is required for import")
return True
if not config["conversation_url"]:
print("Error: No conversation URL configured")
return True
try:
from praisonai.db import PraisonDB
import json
db = PraisonDB(database_url=config["conversation_url"])
with open(input_file, 'r') as f:
data = json.loads(f.readline())
session_id = db.import_session(data)
print(f"Imported session: {session_id}")
print(f"Messages: {len(data.get('messages', []))}")
db.close()
except Exception as e:
print(f"Error: {e}")
return True
return True
def handle_migrate(args: list) -> bool:
"""Handle persistence migrate command."""
config = parse_persistence_args(args)
if not config["conversation_url"] and not config["state_url"]:
print("Error: No database URL configured")
print("Set --conversation-url or --state-url")
return True
try:
from praisonai.db import PraisonDB
from praisonai.persistence.migrations import MigrationManager
db = PraisonDB(
database_url=config["conversation_url"],
state_url=config["state_url"]
)
manager = MigrationManager(db)
# Check for --up or --down flags
direction = "up"
target = None
for i, arg in enumerate(args):
if arg == "--down":
direction = "down"
elif arg == "--target" and i + 1 < len(args):
target = args[i + 1]
if direction == "up":
print("Applying pending migrations...")
applied = manager.migrate_up(target)
if applied:
print(f"Applied migrations: {', '.join(applied)}")
else:
print("No pending migrations")
else:
if not target:
print("Error: --target required for --down migration")
return True
print(f"Rolling back to version {target}...")
rolled_back = manager.migrate_down(target)
if rolled_back:
print(f"Rolled back: {', '.join(rolled_back)}")
else:
print("No migrations to rollback")
db.close()
except Exception as e:
print(f"Error: {e}")
return True
return True
def handle_status(args: list) -> bool:
"""Handle persistence status command."""
config = parse_persistence_args(args)
if not config["conversation_url"] and not config["state_url"]:
print("Error: No database URL configured")
return True
try:
from praisonai.db import PraisonDB
from praisonai.persistence.migrations import MigrationManager
db = PraisonDB(
database_url=config["conversation_url"],
state_url=config["state_url"]
)
manager = MigrationManager(db)
status = manager.get_migration_status()
print("\n=== Schema Status ===")
print(f"Current version: {status['current_version'] or '(not set)'}")
print(f"Latest version: {status['latest_version']}")
print(f"Pending: {status['pending_count']} migration(s)")
if status['pending_versions']:
print(f"Pending versions: {', '.join(status['pending_versions'])}")
db.close()
except Exception as e:
print(f"Error: {e}")
return True
return True
def _mask_url(url: str) -> str:
"""Mask sensitive parts of URL."""
if not url:
return "(not set)"
# Mask password in URLs
import re
masked = re.sub(r'://([^:]+):([^@]+)@', r'://\1:****@', url)
return masked
def _test_conversation_store(url: str) -> tuple:
"""Test conversation store connectivity."""
try:
from praisonai.persistence.factory import create_conversation_store
# Detect backend
if "postgresql" in url or "postgres" in url:
backend = "postgres"
elif "mysql" in url:
backend = "mysql"
elif url.endswith(".db") or "sqlite" in url:
backend = "sqlite"
else:
backend = "sqlite"
store = create_conversation_store(backend, url=url)
# Try a simple operation
sessions = store.list_sessions(limit=1)
store.close()
return True, f"Connected ({backend})"
except Exception as e:
return False, str(e)
def _test_knowledge_store(url: str) -> tuple:
"""Test knowledge store connectivity."""
try:
from praisonai.persistence.factory import create_knowledge_store
# Detect backend
if "qdrant" in url or ":6333" in url:
backend = "qdrant"
elif "chroma" in url:
backend = "chroma"
else:
backend = "qdrant"
store = create_knowledge_store(backend, url=url)
# Try a simple operation
exists = store.collection_exists("test_doctor")
store.close()
return True, f"Connected ({backend})"
except Exception as e:
return False, str(e)
def _test_state_store(url: str) -> tuple:
"""Test state store connectivity."""
try:
from praisonai.persistence.factory import create_state_store
# Detect backend
if "redis" in url:
backend = "redis"
else:
backend = "memory"
store = create_state_store(backend, url=url)
# Try a simple operation
store.set("_doctor_test", "ok", ttl=5)
value = store.get("_doctor_test")
store.delete("_doctor_test")
store.close()
return True, f"Connected ({backend})"
except Exception as e:
return False, str(e)
+6
View File
@@ -266,6 +266,12 @@ class PraisonAI:
file_input = self.read_file_if_provided(getattr(args, 'file', None))
if args.command:
# Handle persistence command
if args.command == "persistence":
from praisonai.cli.features.persistence import handle_persistence_command
handle_persistence_command(unknown_args)
return None
# Handle schedule command
if args.command == "schedule":
from praisonai.cli.features.agent_scheduler import AgentSchedulerHandler
+1 -1
View File
@@ -57,7 +57,7 @@ class CloudDeployer:
file.write("FROM python:3.11-slim\n")
file.write("WORKDIR /app\n")
file.write("COPY . .\n")
file.write("RUN pip install flask praisonai==2.4.0 gunicorn markdown\n")
file.write("RUN pip install flask praisonai==2.4.4 gunicorn markdown\n")
file.write("EXPOSE 8080\n")
file.write('CMD ["gunicorn", "-b", "0.0.0.0:8080", "api:app"]\n')
@@ -0,0 +1,84 @@
"""
PraisonAI Persistence Layer
Provides database integrations for:
- Conversation history persistence (ConversationStore)
- Knowledge storage and retrieval (KnowledgeStore)
- Session/state management (StateStore)
All integrations use lazy loading to avoid importing unused dependencies.
Supported Backends (22 total):
- ConversationStore (6): postgres, mysql, sqlite, singlestore, supabase, surrealdb
- KnowledgeStore (10): qdrant, pinecone, chroma, weaviate, lancedb, milvus, pgvector, redis, cassandra, clickhouse
- StateStore (6): redis, dynamodb, firestore, mongodb, upstash, memory
"""
from typing import TYPE_CHECKING
__all__ = [
# Base interfaces
"ConversationStore",
"ConversationSession",
"ConversationMessage",
"KnowledgeStore",
"KnowledgeDocument",
"StateStore",
# Orchestrator
"PersistenceOrchestrator",
# Factory functions
"create_conversation_store",
"create_knowledge_store",
"create_state_store",
"create_stores_from_config",
# Config
"PersistenceConfig",
# Agent hooks
"wrap_agent_with_persistence",
"PersistentAgent",
"create_persistent_session",
]
# Lazy imports to avoid loading dependencies until needed
def __getattr__(name: str):
if name in ("ConversationStore", "ConversationSession", "ConversationMessage"):
from .conversation.base import ConversationStore, ConversationSession, ConversationMessage
return locals()[name]
if name in ("KnowledgeStore", "KnowledgeDocument"):
from .knowledge.base import KnowledgeStore, KnowledgeDocument
return locals()[name]
if name == "StateStore":
from .state.base import StateStore
return StateStore
if name == "PersistenceOrchestrator":
from .orchestrator import PersistenceOrchestrator
return PersistenceOrchestrator
if name == "PersistenceConfig":
from .config import PersistenceConfig
return PersistenceConfig
if name == "create_conversation_store":
from .factory import create_conversation_store
return create_conversation_store
if name == "create_knowledge_store":
from .factory import create_knowledge_store
return create_knowledge_store
if name == "create_state_store":
from .factory import create_state_store
return create_state_store
if name == "create_stores_from_config":
from .factory import create_stores_from_config
return create_stores_from_config
if name in ("wrap_agent_with_persistence", "PersistentAgent", "create_persistent_session"):
from .hooks.agent_hooks import wrap_agent_with_persistence, PersistentAgent, create_persistent_session
return locals()[name]
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -0,0 +1,238 @@
"""
Configuration handling for PraisonAI persistence layer.
Supports configuration via:
- Environment variables
- Config file (YAML/JSON)
- CLI arguments
- Direct Python API
"""
import os
import logging
from dataclasses import dataclass, field
from typing import Any, Dict, Optional
logger = logging.getLogger(__name__)
# Environment variable mappings
ENV_VARS = {
# Conversation stores
"conversation_store": "PRAISONAI_CONVERSATION_STORE",
"conversation_url": "PRAISONAI_CONVERSATION_URL",
# Knowledge stores
"knowledge_store": "PRAISONAI_KNOWLEDGE_STORE",
"knowledge_url": "PRAISONAI_KNOWLEDGE_URL",
# State stores
"state_store": "PRAISONAI_STATE_STORE",
"state_url": "PRAISONAI_STATE_URL",
# Common
"session_id": "PRAISONAI_SESSION_ID",
"user_id": "PRAISONAI_USER_ID",
# PostgreSQL
"postgres_url": "POSTGRES_URL",
"postgres_host": "POSTGRES_HOST",
"postgres_port": "POSTGRES_PORT",
"postgres_database": "POSTGRES_DATABASE",
"postgres_user": "POSTGRES_USER",
"postgres_password": "POSTGRES_PASSWORD",
# Qdrant
"qdrant_url": "QDRANT_URL",
"qdrant_host": "QDRANT_HOST",
"qdrant_port": "QDRANT_PORT",
"qdrant_api_key": "QDRANT_API_KEY",
# Redis
"redis_url": "REDIS_URL",
"redis_host": "REDIS_HOST",
"redis_port": "REDIS_PORT",
"redis_password": "REDIS_PASSWORD",
# Pinecone
"pinecone_api_key": "PINECONE_API_KEY",
"pinecone_environment": "PINECONE_ENVIRONMENT",
"pinecone_index": "PINECONE_INDEX",
# ChromaDB
"chroma_path": "CHROMA_DB_PATH",
"chroma_host": "CHROMA_HOST",
"chroma_port": "CHROMA_PORT",
# Weaviate
"weaviate_url": "WEAVIATE_URL",
"weaviate_api_key": "WEAVIATE_API_KEY",
# Milvus
"milvus_host": "MILVUS_HOST",
"milvus_port": "MILVUS_PORT",
"milvus_token": "MILVUS_TOKEN",
# MongoDB
"mongodb_url": "MONGODB_URL",
"mongodb_database": "MONGODB_DATABASE",
# DynamoDB
"aws_region": "AWS_REGION",
"dynamodb_table": "DYNAMODB_TABLE",
# Firestore
"google_credentials": "GOOGLE_APPLICATION_CREDENTIALS",
"firestore_project": "FIRESTORE_PROJECT",
# Upstash
"upstash_redis_url": "UPSTASH_REDIS_URL",
"upstash_redis_token": "UPSTASH_REDIS_TOKEN",
"upstash_vector_url": "UPSTASH_VECTOR_URL",
"upstash_vector_token": "UPSTASH_VECTOR_TOKEN",
# Supabase
"supabase_url": "SUPABASE_URL",
"supabase_key": "SUPABASE_KEY",
# SurrealDB
"surrealdb_url": "SURREALDB_URL",
"surrealdb_namespace": "SURREALDB_NS",
"surrealdb_database": "SURREALDB_DB",
# MySQL
"mysql_url": "MYSQL_URL",
"mysql_host": "MYSQL_HOST",
"mysql_port": "MYSQL_PORT",
"mysql_database": "MYSQL_DATABASE",
"mysql_user": "MYSQL_USER",
"mysql_password": "MYSQL_PASSWORD",
# SingleStore
"singlestore_url": "SINGLESTORE_URL",
# ClickHouse
"clickhouse_host": "CLICKHOUSE_HOST",
"clickhouse_port": "CLICKHOUSE_PORT",
"clickhouse_user": "CLICKHOUSE_USER",
"clickhouse_password": "CLICKHOUSE_PASSWORD",
# Cassandra
"cassandra_hosts": "CASSANDRA_HOSTS",
"cassandra_keyspace": "CASSANDRA_KEYSPACE",
# LanceDB
"lancedb_path": "LANCEDB_PATH",
"lancedb_uri": "LANCEDB_URI",
}
# Supported backends
CONVERSATION_BACKENDS = ["postgres", "mysql", "sqlite", "singlestore", "supabase", "surrealdb"]
KNOWLEDGE_BACKENDS = ["qdrant", "pinecone", "chroma", "weaviate", "lancedb", "milvus", "pgvector", "redis", "cassandra", "clickhouse"]
STATE_BACKENDS = ["redis", "dynamodb", "firestore", "mongodb", "upstash", "memory"]
@dataclass
class PersistenceConfig:
"""
Configuration for PraisonAI persistence layer.
Example:
config = PersistenceConfig(
conversation_store="postgres",
conversation_url="postgresql://localhost:5432/praisonai",
knowledge_store="qdrant",
knowledge_url="http://localhost:6333",
state_store="redis",
state_url="redis://localhost:6379",
)
"""
# Store backends
conversation_store: Optional[str] = None
conversation_url: Optional[str] = None
conversation_options: Dict[str, Any] = field(default_factory=dict)
knowledge_store: Optional[str] = None
knowledge_url: Optional[str] = None
knowledge_options: Dict[str, Any] = field(default_factory=dict)
state_store: Optional[str] = None
state_url: Optional[str] = None
state_options: Dict[str, Any] = field(default_factory=dict)
# Session/user context
session_id: Optional[str] = None
user_id: Optional[str] = None
# General options
auto_create_tables: bool = True
lazy_init: bool = True
@classmethod
def from_env(cls) -> "PersistenceConfig":
"""Create config from environment variables."""
return cls(
conversation_store=os.getenv(ENV_VARS["conversation_store"]),
conversation_url=os.getenv(ENV_VARS["conversation_url"]) or os.getenv(ENV_VARS["postgres_url"]),
knowledge_store=os.getenv(ENV_VARS["knowledge_store"]),
knowledge_url=os.getenv(ENV_VARS["knowledge_url"]) or os.getenv(ENV_VARS["qdrant_url"]),
state_store=os.getenv(ENV_VARS["state_store"]),
state_url=os.getenv(ENV_VARS["state_url"]) or os.getenv(ENV_VARS["redis_url"]),
session_id=os.getenv(ENV_VARS["session_id"]),
user_id=os.getenv(ENV_VARS["user_id"]),
)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "PersistenceConfig":
"""Create config from dictionary."""
return cls(**{k: v for k, v in data.items() if hasattr(cls, k)})
@classmethod
def from_yaml(cls, path: str) -> "PersistenceConfig":
"""Load config from YAML file."""
try:
import yaml
except ImportError:
raise ImportError("PyYAML required for YAML config. Install with: pip install pyyaml")
with open(path) as f:
data = yaml.safe_load(f)
# Handle nested persistence config
if "persistence" in data:
data = data["persistence"]
return cls.from_dict(data)
def to_dict(self) -> Dict[str, Any]:
"""Convert config to dictionary."""
return {
"conversation_store": self.conversation_store,
"conversation_url": self.conversation_url,
"conversation_options": self.conversation_options,
"knowledge_store": self.knowledge_store,
"knowledge_url": self.knowledge_url,
"knowledge_options": self.knowledge_options,
"state_store": self.state_store,
"state_url": self.state_url,
"state_options": self.state_options,
"session_id": self.session_id,
"user_id": self.user_id,
"auto_create_tables": self.auto_create_tables,
"lazy_init": self.lazy_init,
}
def validate(self) -> bool:
"""Validate configuration."""
valid = True
if self.conversation_store and self.conversation_store not in CONVERSATION_BACKENDS:
logger.error(f"Invalid conversation_store: {self.conversation_store}. "
f"Supported: {CONVERSATION_BACKENDS}")
valid = False
if self.knowledge_store and self.knowledge_store not in KNOWLEDGE_BACKENDS:
logger.error(f"Invalid knowledge_store: {self.knowledge_store}. "
f"Supported: {KNOWLEDGE_BACKENDS}")
valid = False
if self.state_store and self.state_store not in STATE_BACKENDS:
logger.error(f"Invalid state_store: {self.state_store}. "
f"Supported: {STATE_BACKENDS}")
valid = False
return valid
def get_env_var(key: str, default: Optional[str] = None) -> Optional[str]:
"""Get environment variable by config key name."""
env_name = ENV_VARS.get(key, key.upper())
return os.getenv(env_name, default)
def list_available_backends() -> Dict[str, list]:
"""List all available backends by store type."""
return {
"conversation": CONVERSATION_BACKENDS,
"knowledge": KNOWLEDGE_BACKENDS,
"state": STATE_BACKENDS,
}
@@ -0,0 +1,25 @@
"""
ConversationStore implementations for session and message persistence.
Supported backends:
- PostgreSQL (postgres)
- MySQL (mysql)
- SQLite (sqlite)
- SingleStore (singlestore)
- Supabase (supabase)
- SurrealDB (surrealdb)
"""
from typing import TYPE_CHECKING
__all__ = [
"ConversationStore",
"ConversationSession",
"ConversationMessage",
]
def __getattr__(name: str):
if name in ("ConversationStore", "ConversationSession", "ConversationMessage"):
from .base import ConversationStore, ConversationSession, ConversationMessage
return locals()[name]
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -0,0 +1,427 @@
"""
Async MySQL implementation of ConversationStore.
Requires: aiomysql
Install: pip install aiomysql
"""
import asyncio
import json
import logging
import time
from typing import List, Optional
from .base import ConversationStore, ConversationSession, ConversationMessage
logger = logging.getLogger(__name__)
class AsyncMySQLConversationStore(ConversationStore):
"""
Async MySQL conversation store using aiomysql.
Provides high-performance async database operations.
Example:
store = AsyncMySQLConversationStore(
host="localhost",
user="root",
password="password",
database="praisonai"
)
await store.init()
"""
def __init__(
self,
url: Optional[str] = None,
host: str = "localhost",
port: int = 3306,
database: str = "praisonai",
user: str = "root",
password: str = "",
table_prefix: str = "praisonai_",
pool_size: int = 10,
):
"""
Initialize async MySQL store.
Args:
url: MySQL connection URL (takes precedence)
host: Database host
port: Database port
database: Database name
user: Database user
password: Database password
table_prefix: Prefix for table names
pool_size: Connection pool size
"""
self.url = url
self.host = host
self.port = port
self.database = database
self.user = user
self.password = password
self.table_prefix = table_prefix
self.pool_size = pool_size
self._pool = None
self._initialized = False
if url:
self._parse_url(url)
def _parse_url(self, url: str):
"""Parse MySQL URL into components."""
from urllib.parse import urlparse
parsed = urlparse(url)
self.host = parsed.hostname or "localhost"
self.port = parsed.port or 3306
self.database = parsed.path.lstrip('/') or "praisonai"
self.user = parsed.username or "root"
self.password = parsed.password or ""
async def init(self):
"""Initialize connection pool and create tables."""
if self._initialized:
return
try:
import aiomysql
except ImportError:
raise ImportError(
"aiomysql is required for async MySQL support. "
"Install with: pip install aiomysql"
)
self._pool = await aiomysql.create_pool(
host=self.host,
port=self.port,
user=self.user,
password=self.password,
db=self.database,
minsize=1,
maxsize=self.pool_size,
autocommit=True
)
await self._create_tables()
self._initialized = True
async def _create_tables(self):
"""Create required tables if they don't exist."""
sessions_table = f"{self.table_prefix}sessions"
messages_table = f"{self.table_prefix}messages"
async with self._pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(f"""
CREATE TABLE IF NOT EXISTS {sessions_table} (
session_id VARCHAR(255) PRIMARY KEY,
user_id VARCHAR(255),
agent_id VARCHAR(255),
name VARCHAR(255),
metadata JSON,
created_at DOUBLE,
updated_at DOUBLE,
INDEX idx_user_id (user_id),
INDEX idx_agent_id (agent_id)
)
""")
await cur.execute(f"""
CREATE TABLE IF NOT EXISTS {messages_table} (
id VARCHAR(255) PRIMARY KEY,
session_id VARCHAR(255),
role VARCHAR(50),
content TEXT,
metadata JSON,
created_at DOUBLE,
INDEX idx_session_id (session_id),
FOREIGN KEY (session_id) REFERENCES {sessions_table}(session_id) ON DELETE CASCADE
)
""")
async def async_create_session(self, session: ConversationSession) -> ConversationSession:
"""Create a new session asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
async with self._pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(f"""
INSERT INTO {table} (session_id, user_id, agent_id, name, metadata, created_at, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, %s)
""", (session.session_id, session.user_id, session.agent_id, session.name,
json.dumps(session.metadata) if session.metadata else None,
session.created_at, session.updated_at))
return session
def create_session(self, session: ConversationSession) -> ConversationSession:
"""Sync wrapper for create_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_create_session(session)
)
async def async_get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Get a session by ID asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
async with self._pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(f"""
SELECT session_id, user_id, agent_id, name, metadata, created_at, updated_at
FROM {table} WHERE session_id = %s
""", (session_id,))
row = await cur.fetchone()
if row:
return ConversationSession(
session_id=row[0],
user_id=row[1],
agent_id=row[2],
name=row[3],
metadata=json.loads(row[4]) if row[4] else None,
created_at=row[5],
updated_at=row[6]
)
return None
def get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Sync wrapper for get_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_get_session(session_id)
)
async def async_update_session(self, session: ConversationSession) -> ConversationSession:
"""Update an existing session asynchronously."""
if not self._initialized:
await self.init()
session.updated_at = time.time()
table = f"{self.table_prefix}sessions"
async with self._pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(f"""
UPDATE {table}
SET name = %s, metadata = %s, updated_at = %s
WHERE session_id = %s
""", (session.name, json.dumps(session.metadata) if session.metadata else None,
session.updated_at, session.session_id))
return session
def update_session(self, session: ConversationSession) -> ConversationSession:
"""Sync wrapper for update_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_update_session(session)
)
async def async_delete_session(self, session_id: str) -> bool:
"""Delete a session asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
async with self._pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(f"""
DELETE FROM {table} WHERE session_id = %s
""", (session_id,))
return cur.rowcount > 0
def delete_session(self, session_id: str) -> bool:
"""Sync wrapper for delete_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_delete_session(session_id)
)
async def async_list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""List sessions asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
conditions = []
params = []
if user_id:
conditions.append("user_id = %s")
params.append(user_id)
if agent_id:
conditions.append("agent_id = %s")
params.append(agent_id)
where_clause = f"WHERE {' AND '.join(conditions)}" if conditions else ""
params.extend([limit, offset])
async with self._pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(f"""
SELECT session_id, user_id, agent_id, name, metadata, created_at, updated_at
FROM {table} {where_clause}
ORDER BY updated_at DESC
LIMIT %s OFFSET %s
""", params)
rows = await cur.fetchall()
return [
ConversationSession(
session_id=row[0],
user_id=row[1],
agent_id=row[2],
name=row[3],
metadata=json.loads(row[4]) if row[4] else None,
created_at=row[5],
updated_at=row[6]
)
for row in rows
]
def list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""Sync wrapper for list_sessions."""
return asyncio.get_event_loop().run_until_complete(
self.async_list_sessions(user_id, agent_id, limit, offset)
)
async def async_add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Add a message asynchronously."""
if not self._initialized:
await self.init()
message.session_id = session_id
table = f"{self.table_prefix}messages"
async with self._pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(f"""
INSERT INTO {table} (id, session_id, role, content, metadata, created_at)
VALUES (%s, %s, %s, %s, %s, %s)
""", (message.id, session_id, message.role, message.content,
json.dumps(message.metadata) if message.metadata else None,
message.created_at))
return message
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Sync wrapper for add_message."""
return asyncio.get_event_loop().run_until_complete(
self.async_add_message(session_id, message)
)
async def async_get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Get messages asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}messages"
conditions = ["session_id = %s"]
params = [session_id]
if before:
conditions.append("created_at < %s")
params.append(before)
if after:
conditions.append("created_at > %s")
params.append(after)
where_clause = f"WHERE {' AND '.join(conditions)}"
limit_clause = "LIMIT %s" if limit else ""
if limit:
params.append(limit)
async with self._pool.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute(f"""
SELECT id, session_id, role, content, metadata, created_at
FROM {table} {where_clause}
ORDER BY created_at ASC {limit_clause}
""", params)
rows = await cur.fetchall()
return [
ConversationMessage(
id=row[0],
session_id=row[1],
role=row[2],
content=row[3],
metadata=json.loads(row[4]) if row[4] else None,
created_at=row[5]
)
for row in rows
]
def get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Sync wrapper for get_messages."""
return asyncio.get_event_loop().run_until_complete(
self.async_get_messages(session_id, limit, before, after)
)
async def async_delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Delete messages asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}messages"
async with self._pool.acquire() as conn:
async with conn.cursor() as cur:
if message_ids is None:
await cur.execute(f"""
DELETE FROM {table} WHERE session_id = %s
""", (session_id,))
else:
placeholders = ','.join(['%s'] * len(message_ids))
await cur.execute(f"""
DELETE FROM {table} WHERE session_id = %s AND id IN ({placeholders})
""", [session_id] + message_ids)
return cur.rowcount
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Sync wrapper for delete_messages."""
return asyncio.get_event_loop().run_until_complete(
self.async_delete_messages(session_id, message_ids)
)
async def async_close(self) -> None:
"""Close the connection pool."""
if self._pool:
self._pool.close()
await self._pool.wait_closed()
self._pool = None
self._initialized = False
def close(self) -> None:
"""Sync wrapper for close."""
if self._pool:
asyncio.get_event_loop().run_until_complete(self.async_close())
@@ -0,0 +1,410 @@
"""
Async PostgreSQL implementation of ConversationStore.
Requires: asyncpg
Install: pip install asyncpg
"""
import asyncio
import json
import logging
import time
from typing import List, Optional
from .base import ConversationStore, ConversationSession, ConversationMessage
logger = logging.getLogger(__name__)
class AsyncPostgresConversationStore(ConversationStore):
"""
Async PostgreSQL conversation store using asyncpg.
Provides high-performance async database operations.
Example:
store = AsyncPostgresConversationStore(
url="postgresql://user:pass@localhost:5432/praisonai"
)
await store.init()
"""
def __init__(
self,
url: Optional[str] = None,
host: str = "localhost",
port: int = 5432,
database: str = "praisonai",
user: str = "postgres",
password: str = "",
table_prefix: str = "praisonai_",
pool_size: int = 10,
):
"""
Initialize async PostgreSQL store.
Args:
url: PostgreSQL connection URL (takes precedence)
host: Database host
port: Database port
database: Database name
user: Database user
password: Database password
table_prefix: Prefix for table names
pool_size: Connection pool size
"""
self.url = url
self.host = host
self.port = port
self.database = database
self.user = user
self.password = password
self.table_prefix = table_prefix
self.pool_size = pool_size
self._pool = None
self._initialized = False
async def init(self):
"""Initialize connection pool and create tables."""
if self._initialized:
return
try:
import asyncpg
except ImportError:
raise ImportError(
"asyncpg is required for async PostgreSQL support. "
"Install with: pip install asyncpg"
)
if self.url:
self._pool = await asyncpg.create_pool(self.url, min_size=1, max_size=self.pool_size)
else:
self._pool = await asyncpg.create_pool(
host=self.host,
port=self.port,
database=self.database,
user=self.user,
password=self.password,
min_size=1,
max_size=self.pool_size
)
await self._create_tables()
self._initialized = True
async def _create_tables(self):
"""Create required tables if they don't exist."""
sessions_table = f"{self.table_prefix}sessions"
messages_table = f"{self.table_prefix}messages"
async with self._pool.acquire() as conn:
await conn.execute(f"""
CREATE TABLE IF NOT EXISTS {sessions_table} (
session_id VARCHAR(255) PRIMARY KEY,
user_id VARCHAR(255),
agent_id VARCHAR(255),
name VARCHAR(255),
metadata JSONB,
created_at DOUBLE PRECISION,
updated_at DOUBLE PRECISION
)
""")
await conn.execute(f"""
CREATE TABLE IF NOT EXISTS {messages_table} (
id VARCHAR(255) PRIMARY KEY,
session_id VARCHAR(255) REFERENCES {sessions_table}(session_id) ON DELETE CASCADE,
role VARCHAR(50),
content TEXT,
metadata JSONB,
created_at DOUBLE PRECISION
)
""")
await conn.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{messages_table}_session
ON {messages_table}(session_id)
""")
async def async_create_session(self, session: ConversationSession) -> ConversationSession:
"""Create a new session asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
async with self._pool.acquire() as conn:
await conn.execute(f"""
INSERT INTO {table} (session_id, user_id, agent_id, name, metadata, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7)
""", session.session_id, session.user_id, session.agent_id, session.name,
json.dumps(session.metadata) if session.metadata else None,
session.created_at, session.updated_at)
return session
def create_session(self, session: ConversationSession) -> ConversationSession:
"""Sync wrapper for create_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_create_session(session)
)
async def async_get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Get a session by ID asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
async with self._pool.acquire() as conn:
row = await conn.fetchrow(f"""
SELECT * FROM {table} WHERE session_id = $1
""", session_id)
if row:
return ConversationSession(
session_id=row['session_id'],
user_id=row['user_id'],
agent_id=row['agent_id'],
name=row['name'],
metadata=json.loads(row['metadata']) if row['metadata'] else None,
created_at=row['created_at'],
updated_at=row['updated_at']
)
return None
def get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Sync wrapper for get_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_get_session(session_id)
)
async def async_update_session(self, session: ConversationSession) -> ConversationSession:
"""Update an existing session asynchronously."""
if not self._initialized:
await self.init()
session.updated_at = time.time()
table = f"{self.table_prefix}sessions"
async with self._pool.acquire() as conn:
await conn.execute(f"""
UPDATE {table}
SET name = $2, metadata = $3, updated_at = $4
WHERE session_id = $1
""", session.session_id, session.name,
json.dumps(session.metadata) if session.metadata else None,
session.updated_at)
return session
def update_session(self, session: ConversationSession) -> ConversationSession:
"""Sync wrapper for update_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_update_session(session)
)
async def async_delete_session(self, session_id: str) -> bool:
"""Delete a session asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
async with self._pool.acquire() as conn:
result = await conn.execute(f"""
DELETE FROM {table} WHERE session_id = $1
""", session_id)
return "DELETE 1" in result
def delete_session(self, session_id: str) -> bool:
"""Sync wrapper for delete_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_delete_session(session_id)
)
async def async_list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""List sessions asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
conditions = []
params = []
param_idx = 1
if user_id:
conditions.append(f"user_id = ${param_idx}")
params.append(user_id)
param_idx += 1
if agent_id:
conditions.append(f"agent_id = ${param_idx}")
params.append(agent_id)
param_idx += 1
where_clause = f"WHERE {' AND '.join(conditions)}" if conditions else ""
params.extend([limit, offset])
async with self._pool.acquire() as conn:
rows = await conn.fetch(f"""
SELECT * FROM {table} {where_clause}
ORDER BY updated_at DESC
LIMIT ${param_idx} OFFSET ${param_idx + 1}
""", *params)
return [
ConversationSession(
session_id=row['session_id'],
user_id=row['user_id'],
agent_id=row['agent_id'],
name=row['name'],
metadata=json.loads(row['metadata']) if row['metadata'] else None,
created_at=row['created_at'],
updated_at=row['updated_at']
)
for row in rows
]
def list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""Sync wrapper for list_sessions."""
return asyncio.get_event_loop().run_until_complete(
self.async_list_sessions(user_id, agent_id, limit, offset)
)
async def async_add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Add a message asynchronously."""
if not self._initialized:
await self.init()
message.session_id = session_id
table = f"{self.table_prefix}messages"
async with self._pool.acquire() as conn:
await conn.execute(f"""
INSERT INTO {table} (id, session_id, role, content, metadata, created_at)
VALUES ($1, $2, $3, $4, $5, $6)
""", message.id, session_id, message.role, message.content,
json.dumps(message.metadata) if message.metadata else None,
message.created_at)
return message
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Sync wrapper for add_message."""
return asyncio.get_event_loop().run_until_complete(
self.async_add_message(session_id, message)
)
async def async_get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Get messages asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}messages"
conditions = ["session_id = $1"]
params = [session_id]
param_idx = 2
if before:
conditions.append(f"created_at < ${param_idx}")
params.append(before)
param_idx += 1
if after:
conditions.append(f"created_at > ${param_idx}")
params.append(after)
param_idx += 1
where_clause = f"WHERE {' AND '.join(conditions)}"
limit_clause = f"LIMIT ${param_idx}" if limit else ""
if limit:
params.append(limit)
async with self._pool.acquire() as conn:
rows = await conn.fetch(f"""
SELECT * FROM {table} {where_clause}
ORDER BY created_at ASC {limit_clause}
""", *params)
return [
ConversationMessage(
id=row['id'],
session_id=row['session_id'],
role=row['role'],
content=row['content'],
metadata=json.loads(row['metadata']) if row['metadata'] else None,
created_at=row['created_at']
)
for row in rows
]
def get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Sync wrapper for get_messages."""
return asyncio.get_event_loop().run_until_complete(
self.async_get_messages(session_id, limit, before, after)
)
async def async_delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Delete messages asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}messages"
async with self._pool.acquire() as conn:
if message_ids is None:
result = await conn.execute(f"""
DELETE FROM {table} WHERE session_id = $1
""", session_id)
else:
result = await conn.execute(f"""
DELETE FROM {table} WHERE session_id = $1 AND id = ANY($2)
""", session_id, message_ids)
# Parse count from result like "DELETE 5"
try:
return int(result.split()[-1])
except (ValueError, IndexError):
return 0
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Sync wrapper for delete_messages."""
return asyncio.get_event_loop().run_until_complete(
self.async_delete_messages(session_id, message_ids)
)
async def async_close(self) -> None:
"""Close the connection pool."""
if self._pool:
await self._pool.close()
self._pool = None
self._initialized = False
def close(self) -> None:
"""Sync wrapper for close."""
if self._pool:
asyncio.get_event_loop().run_until_complete(self.async_close())
@@ -0,0 +1,371 @@
"""
Async SQLite implementation of ConversationStore.
Requires: aiosqlite
Install: pip install aiosqlite
"""
import asyncio
import json
import logging
import time
from typing import List, Optional
from .base import ConversationStore, ConversationSession, ConversationMessage
logger = logging.getLogger(__name__)
class AsyncSQLiteConversationStore(ConversationStore):
"""
Async SQLite conversation store using aiosqlite.
Provides async database operations for SQLite.
Example:
store = AsyncSQLiteConversationStore(path="./conversations.db")
await store.init()
"""
def __init__(
self,
path: str = "praisonai_conversations.db",
table_prefix: str = "praisonai_",
):
"""
Initialize async SQLite store.
Args:
path: Path to SQLite database file
table_prefix: Prefix for table names
"""
self.path = path
self.table_prefix = table_prefix
self._conn = None
self._initialized = False
async def init(self):
"""Initialize connection and create tables."""
if self._initialized:
return
try:
import aiosqlite
except ImportError:
raise ImportError(
"aiosqlite is required for async SQLite support. "
"Install with: pip install aiosqlite"
)
self._aiosqlite = aiosqlite
self._conn = await aiosqlite.connect(self.path)
self._conn.row_factory = aiosqlite.Row
await self._create_tables()
self._initialized = True
async def _create_tables(self):
"""Create required tables if they don't exist."""
sessions_table = f"{self.table_prefix}sessions"
messages_table = f"{self.table_prefix}messages"
await self._conn.execute(f"""
CREATE TABLE IF NOT EXISTS {sessions_table} (
session_id TEXT PRIMARY KEY,
user_id TEXT,
agent_id TEXT,
name TEXT,
metadata TEXT,
created_at REAL,
updated_at REAL
)
""")
await self._conn.execute(f"""
CREATE TABLE IF NOT EXISTS {messages_table} (
id TEXT PRIMARY KEY,
session_id TEXT REFERENCES {sessions_table}(session_id) ON DELETE CASCADE,
role TEXT,
content TEXT,
metadata TEXT,
created_at REAL
)
""")
await self._conn.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{messages_table}_session
ON {messages_table}(session_id)
""")
await self._conn.commit()
async def async_create_session(self, session: ConversationSession) -> ConversationSession:
"""Create a new session asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
await self._conn.execute(f"""
INSERT INTO {table} (session_id, user_id, agent_id, name, metadata, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?)
""", (session.session_id, session.user_id, session.agent_id, session.name,
json.dumps(session.metadata) if session.metadata else None,
session.created_at, session.updated_at))
await self._conn.commit()
return session
def create_session(self, session: ConversationSession) -> ConversationSession:
"""Sync wrapper for create_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_create_session(session)
)
async def async_get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Get a session by ID asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
async with self._conn.execute(f"""
SELECT * FROM {table} WHERE session_id = ?
""", (session_id,)) as cursor:
row = await cursor.fetchone()
if row:
return ConversationSession(
session_id=row['session_id'],
user_id=row['user_id'],
agent_id=row['agent_id'],
name=row['name'],
metadata=json.loads(row['metadata']) if row['metadata'] else None,
created_at=row['created_at'],
updated_at=row['updated_at']
)
return None
def get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Sync wrapper for get_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_get_session(session_id)
)
async def async_update_session(self, session: ConversationSession) -> ConversationSession:
"""Update an existing session asynchronously."""
if not self._initialized:
await self.init()
session.updated_at = time.time()
table = f"{self.table_prefix}sessions"
await self._conn.execute(f"""
UPDATE {table}
SET name = ?, metadata = ?, updated_at = ?
WHERE session_id = ?
""", (session.name, json.dumps(session.metadata) if session.metadata else None,
session.updated_at, session.session_id))
await self._conn.commit()
return session
def update_session(self, session: ConversationSession) -> ConversationSession:
"""Sync wrapper for update_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_update_session(session)
)
async def async_delete_session(self, session_id: str) -> bool:
"""Delete a session asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
cursor = await self._conn.execute(f"""
DELETE FROM {table} WHERE session_id = ?
""", (session_id,))
await self._conn.commit()
return cursor.rowcount > 0
def delete_session(self, session_id: str) -> bool:
"""Sync wrapper for delete_session."""
return asyncio.get_event_loop().run_until_complete(
self.async_delete_session(session_id)
)
async def async_list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""List sessions asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}sessions"
conditions = []
params = []
if user_id:
conditions.append("user_id = ?")
params.append(user_id)
if agent_id:
conditions.append("agent_id = ?")
params.append(agent_id)
where_clause = f"WHERE {' AND '.join(conditions)}" if conditions else ""
params.extend([limit, offset])
async with self._conn.execute(f"""
SELECT * FROM {table} {where_clause}
ORDER BY updated_at DESC
LIMIT ? OFFSET ?
""", params) as cursor:
rows = await cursor.fetchall()
return [
ConversationSession(
session_id=row['session_id'],
user_id=row['user_id'],
agent_id=row['agent_id'],
name=row['name'],
metadata=json.loads(row['metadata']) if row['metadata'] else None,
created_at=row['created_at'],
updated_at=row['updated_at']
)
for row in rows
]
def list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""Sync wrapper for list_sessions."""
return asyncio.get_event_loop().run_until_complete(
self.async_list_sessions(user_id, agent_id, limit, offset)
)
async def async_add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Add a message asynchronously."""
if not self._initialized:
await self.init()
message.session_id = session_id
table = f"{self.table_prefix}messages"
await self._conn.execute(f"""
INSERT INTO {table} (id, session_id, role, content, metadata, created_at)
VALUES (?, ?, ?, ?, ?, ?)
""", (message.id, session_id, message.role, message.content,
json.dumps(message.metadata) if message.metadata else None,
message.created_at))
await self._conn.commit()
return message
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Sync wrapper for add_message."""
return asyncio.get_event_loop().run_until_complete(
self.async_add_message(session_id, message)
)
async def async_get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Get messages asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}messages"
conditions = ["session_id = ?"]
params = [session_id]
if before:
conditions.append("created_at < ?")
params.append(before)
if after:
conditions.append("created_at > ?")
params.append(after)
where_clause = f"WHERE {' AND '.join(conditions)}"
limit_clause = f"LIMIT ?" if limit else ""
if limit:
params.append(limit)
async with self._conn.execute(f"""
SELECT * FROM {table} {where_clause}
ORDER BY created_at ASC {limit_clause}
""", params) as cursor:
rows = await cursor.fetchall()
return [
ConversationMessage(
id=row['id'],
session_id=row['session_id'],
role=row['role'],
content=row['content'],
metadata=json.loads(row['metadata']) if row['metadata'] else None,
created_at=row['created_at']
)
for row in rows
]
def get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Sync wrapper for get_messages."""
return asyncio.get_event_loop().run_until_complete(
self.async_get_messages(session_id, limit, before, after)
)
async def async_delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Delete messages asynchronously."""
if not self._initialized:
await self.init()
table = f"{self.table_prefix}messages"
if message_ids is None:
cursor = await self._conn.execute(f"""
DELETE FROM {table} WHERE session_id = ?
""", (session_id,))
else:
placeholders = ','.join(['?' for _ in message_ids])
cursor = await self._conn.execute(f"""
DELETE FROM {table} WHERE session_id = ? AND id IN ({placeholders})
""", [session_id] + message_ids)
await self._conn.commit()
return cursor.rowcount
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Sync wrapper for delete_messages."""
return asyncio.get_event_loop().run_until_complete(
self.async_delete_messages(session_id, message_ids)
)
async def async_close(self) -> None:
"""Close the connection."""
if self._conn:
await self._conn.close()
self._conn = None
self._initialized = False
def close(self) -> None:
"""Sync wrapper for close."""
if self._conn:
asyncio.get_event_loop().run_until_complete(self.async_close())
@@ -0,0 +1,151 @@
"""
Base interfaces for ConversationStore.
ConversationStore handles session and message persistence for conversation history.
"""
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional
import time
import uuid
@dataclass
class ConversationMessage:
"""A single message in a conversation."""
id: str = field(default_factory=lambda: str(uuid.uuid4()))
session_id: str = ""
role: str = "" # user, assistant, system, tool
content: str = ""
tool_calls: Optional[List[Dict[str, Any]]] = None
tool_call_id: Optional[str] = None
metadata: Optional[Dict[str, Any]] = None
created_at: float = field(default_factory=time.time)
def to_dict(self) -> Dict[str, Any]:
return {
"id": self.id,
"session_id": self.session_id,
"role": self.role,
"content": self.content,
"tool_calls": self.tool_calls,
"tool_call_id": self.tool_call_id,
"metadata": self.metadata,
"created_at": self.created_at,
}
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "ConversationMessage":
return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__})
@dataclass
class ConversationSession:
"""A conversation session containing messages."""
session_id: str = field(default_factory=lambda: str(uuid.uuid4()))
user_id: Optional[str] = None
agent_id: Optional[str] = None
name: Optional[str] = None
state: Optional[Dict[str, Any]] = None
metadata: Optional[Dict[str, Any]] = None
created_at: float = field(default_factory=time.time)
updated_at: float = field(default_factory=time.time)
def to_dict(self) -> Dict[str, Any]:
return {
"session_id": self.session_id,
"user_id": self.user_id,
"agent_id": self.agent_id,
"name": self.name,
"state": self.state,
"metadata": self.metadata,
"created_at": self.created_at,
"updated_at": self.updated_at,
}
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "ConversationSession":
return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__})
class ConversationStore(ABC):
"""
Abstract base class for conversation persistence.
Implementations handle session and message storage for different backends:
- PostgreSQL, MySQL, SQLite (relational)
- SingleStore, Supabase, SurrealDB (hybrid)
"""
@abstractmethod
def create_session(self, session: ConversationSession) -> ConversationSession:
"""Create a new session."""
raise NotImplementedError
@abstractmethod
def get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Get a session by ID."""
raise NotImplementedError
@abstractmethod
def update_session(self, session: ConversationSession) -> ConversationSession:
"""Update an existing session."""
raise NotImplementedError
@abstractmethod
def delete_session(self, session_id: str) -> bool:
"""Delete a session and all its messages."""
raise NotImplementedError
@abstractmethod
def list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""List sessions, optionally filtered by user or agent."""
raise NotImplementedError
@abstractmethod
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Add a message to a session."""
raise NotImplementedError
@abstractmethod
def get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Get messages from a session."""
raise NotImplementedError
@abstractmethod
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Delete messages. If message_ids is None, delete all messages in session."""
raise NotImplementedError
def upsert_session(self, session: ConversationSession) -> ConversationSession:
"""Create or update a session."""
existing = self.get_session(session.session_id)
if existing:
session.updated_at = time.time()
return self.update_session(session)
return self.create_session(session)
@abstractmethod
def close(self) -> None:
"""Close the store and release resources."""
raise NotImplementedError
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
return False
@@ -0,0 +1,250 @@
"""
JSON file-based implementation of ConversationStore.
Simple file-based storage for development and testing.
No external dependencies required.
"""
import json
import logging
import time
from pathlib import Path
from typing import Dict, List, Optional
import threading
from .base import ConversationStore, ConversationSession, ConversationMessage
logger = logging.getLogger(__name__)
class JSONConversationStore(ConversationStore):
"""
JSON file-based conversation store.
Stores sessions and messages in JSON files.
Suitable for development, testing, and small-scale deployments.
Example:
store = JSONConversationStore(path="./data/conversations")
"""
def __init__(
self,
path: str = "./praisonai_conversations",
pretty: bool = True,
):
"""
Initialize JSON conversation store.
Args:
path: Directory path for JSON files
pretty: Use pretty-printed JSON (default: True)
"""
self.path = Path(path)
self.pretty = pretty
self._lock = threading.Lock()
# Create directory if it doesn't exist
self.path.mkdir(parents=True, exist_ok=True)
# Sessions index file
self._index_file = self.path / "_sessions_index.json"
self._load_index()
def _load_index(self):
"""Load or create sessions index."""
if self._index_file.exists():
with open(self._index_file, 'r') as f:
self._index = json.load(f)
else:
self._index = {"sessions": {}}
self._save_index()
def _save_index(self):
"""Save sessions index."""
with open(self._index_file, 'w') as f:
if self.pretty:
json.dump(self._index, f, indent=2, default=str)
else:
json.dump(self._index, f, default=str)
def _session_file(self, session_id: str) -> Path:
"""Get path to session file."""
return self.path / f"{session_id}.json"
def _load_session_data(self, session_id: str) -> Optional[Dict]:
"""Load session data from file."""
file_path = self._session_file(session_id)
if file_path.exists():
with open(file_path, 'r') as f:
return json.load(f)
return None
def _save_session_data(self, session_id: str, data: Dict):
"""Save session data to file."""
file_path = self._session_file(session_id)
with open(file_path, 'w') as f:
if self.pretty:
json.dump(data, f, indent=2, default=str)
else:
json.dump(data, f, default=str)
def create_session(self, session: ConversationSession) -> ConversationSession:
"""Create a new session."""
with self._lock:
data = {
"session": session.to_dict(),
"messages": []
}
self._save_session_data(session.session_id, data)
# Update index
self._index["sessions"][session.session_id] = {
"user_id": session.user_id,
"agent_id": session.agent_id,
"name": session.name,
"created_at": session.created_at,
"updated_at": session.updated_at,
}
self._save_index()
return session
def get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Get a session by ID."""
data = self._load_session_data(session_id)
if data and "session" in data:
return ConversationSession.from_dict(data["session"])
return None
def update_session(self, session: ConversationSession) -> ConversationSession:
"""Update an existing session."""
with self._lock:
data = self._load_session_data(session.session_id)
if not data:
raise ValueError(f"Session {session.session_id} not found")
session.updated_at = time.time()
data["session"] = session.to_dict()
self._save_session_data(session.session_id, data)
# Update index
self._index["sessions"][session.session_id].update({
"name": session.name,
"updated_at": session.updated_at,
})
self._save_index()
return session
def delete_session(self, session_id: str) -> bool:
"""Delete a session and all its messages."""
with self._lock:
file_path = self._session_file(session_id)
if file_path.exists():
file_path.unlink()
# Update index
if session_id in self._index["sessions"]:
del self._index["sessions"][session_id]
self._save_index()
return True
return False
def list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""List sessions, optionally filtered by user or agent."""
sessions = []
for sid, info in self._index["sessions"].items():
if user_id and info.get("user_id") != user_id:
continue
if agent_id and info.get("agent_id") != agent_id:
continue
session = self.get_session(sid)
if session:
sessions.append(session)
# Sort by updated_at descending
sessions.sort(key=lambda s: s.updated_at, reverse=True)
return sessions[offset:offset + limit]
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Add a message to a session."""
with self._lock:
data = self._load_session_data(session_id)
if not data:
raise ValueError(f"Session {session_id} not found")
message.session_id = session_id
data["messages"].append(message.to_dict())
# Update session timestamp
data["session"]["updated_at"] = time.time()
self._save_session_data(session_id, data)
# Update index
if session_id in self._index["sessions"]:
self._index["sessions"][session_id]["updated_at"] = time.time()
self._save_index()
return message
def get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Get messages from a session."""
data = self._load_session_data(session_id)
if not data:
return []
messages = [ConversationMessage.from_dict(m) for m in data.get("messages", [])]
# Apply filters
if before:
messages = [m for m in messages if m.created_at < before]
if after:
messages = [m for m in messages if m.created_at > after]
# Sort by created_at
messages.sort(key=lambda m: m.created_at)
if limit:
messages = messages[-limit:]
return messages
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Delete messages. If message_ids is None, delete all messages in session."""
with self._lock:
data = self._load_session_data(session_id)
if not data:
return 0
if message_ids is None:
count = len(data["messages"])
data["messages"] = []
else:
original_count = len(data["messages"])
data["messages"] = [m for m in data["messages"] if m.get("id") not in message_ids]
count = original_count - len(data["messages"])
self._save_session_data(session_id, data)
return count
def close(self) -> None:
"""Close the store."""
pass # No resources to release
@@ -0,0 +1,387 @@
"""
MySQL implementation of ConversationStore.
Requires: mysql-connector-python or pymysql
Install: pip install mysql-connector-python
"""
import json
import logging
from typing import Any, List, Optional
from .base import ConversationStore, ConversationSession, ConversationMessage
logger = logging.getLogger(__name__)
class MySQLConversationStore(ConversationStore):
"""
MySQL-based conversation store.
Example:
store = MySQLConversationStore(
url="mysql://user:password@localhost:3306/praisonai"
)
"""
SCHEMA_VERSION = "1.0.0"
def __init__(
self,
url: Optional[str] = None,
host: str = "localhost",
port: int = 3306,
database: str = "praisonai",
user: str = "root",
password: str = "",
table_prefix: str = "praison_",
auto_create_tables: bool = True,
pool_size: int = 5,
):
"""
Initialize MySQL conversation store.
Args:
url: Full connection URL (overrides individual params)
host: Database host
port: Database port
database: Database name
user: Database user
password: Database password
table_prefix: Prefix for table names
auto_create_tables: Create tables if they don't exist
pool_size: Connection pool size
"""
try:
import mysql.connector
from mysql.connector import pooling
except ImportError:
raise ImportError(
"mysql-connector-python is required for MySQL support. "
"Install with: pip install mysql-connector-python"
)
self._mysql = mysql.connector
self.table_prefix = table_prefix
self.sessions_table = f"{table_prefix}sessions"
self.messages_table = f"{table_prefix}messages"
# Parse URL if provided
if url:
# Parse mysql://user:pass@host:port/database
from urllib.parse import urlparse
parsed = urlparse(url)
host = parsed.hostname or host
port = parsed.port or port
database = parsed.path.lstrip("/") or database
user = parsed.username or user
password = parsed.password or password
# Create connection pool
self._pool = pooling.MySQLConnectionPool(
pool_name="praison_mysql_pool",
pool_size=pool_size,
host=host,
port=port,
database=database,
user=user,
password=password,
)
if auto_create_tables:
self._create_tables()
def _get_conn(self):
"""Get a connection from the pool."""
return self._pool.get_connection()
def _create_tables(self) -> None:
"""Create tables if they don't exist."""
conn = self._get_conn()
try:
cur = conn.cursor()
# Sessions table
cur.execute(f"""
CREATE TABLE IF NOT EXISTS {self.sessions_table} (
session_id VARCHAR(255) PRIMARY KEY,
user_id VARCHAR(255),
agent_id VARCHAR(255),
name VARCHAR(255),
state JSON,
metadata JSON,
created_at DOUBLE,
updated_at DOUBLE,
INDEX idx_user (user_id),
INDEX idx_agent (agent_id),
INDEX idx_updated (updated_at DESC)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
""")
# Messages table
cur.execute(f"""
CREATE TABLE IF NOT EXISTS {self.messages_table} (
id VARCHAR(255) PRIMARY KEY,
session_id VARCHAR(255) NOT NULL,
role VARCHAR(50) NOT NULL,
content TEXT,
tool_calls JSON,
tool_call_id VARCHAR(255),
metadata JSON,
created_at DOUBLE,
INDEX idx_session (session_id),
INDEX idx_created (session_id, created_at),
FOREIGN KEY (session_id) REFERENCES {self.sessions_table}(session_id) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
""")
conn.commit()
logger.info(f"MySQL tables created: {self.sessions_table}, {self.messages_table}")
finally:
cur.close()
conn.close()
def create_session(self, session: ConversationSession) -> ConversationSession:
"""Create a new session."""
conn = self._get_conn()
try:
cur = conn.cursor()
cur.execute(f"""
INSERT INTO {self.sessions_table}
(session_id, user_id, agent_id, name, state, metadata, created_at, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
""", (
session.session_id,
session.user_id,
session.agent_id,
session.name,
json.dumps(session.state) if session.state else None,
json.dumps(session.metadata) if session.metadata else None,
session.created_at,
session.updated_at,
))
conn.commit()
return session
finally:
cur.close()
conn.close()
def get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Get a session by ID."""
conn = self._get_conn()
try:
cur = conn.cursor(dictionary=True)
cur.execute(f"""
SELECT * FROM {self.sessions_table} WHERE session_id = %s
""", (session_id,))
row = cur.fetchone()
if not row:
return None
return ConversationSession(
session_id=row["session_id"],
user_id=row["user_id"],
agent_id=row["agent_id"],
name=row["name"],
state=row["state"] if isinstance(row["state"], dict) else (json.loads(row["state"]) if row["state"] else None),
metadata=row["metadata"] if isinstance(row["metadata"], dict) else (json.loads(row["metadata"]) if row["metadata"] else None),
created_at=row["created_at"],
updated_at=row["updated_at"],
)
finally:
cur.close()
conn.close()
def update_session(self, session: ConversationSession) -> ConversationSession:
"""Update an existing session."""
conn = self._get_conn()
try:
cur = conn.cursor()
cur.execute(f"""
UPDATE {self.sessions_table}
SET user_id = %s, agent_id = %s, name = %s,
state = %s, metadata = %s, updated_at = %s
WHERE session_id = %s
""", (
session.user_id,
session.agent_id,
session.name,
json.dumps(session.state) if session.state else None,
json.dumps(session.metadata) if session.metadata else None,
session.updated_at,
session.session_id,
))
conn.commit()
return session
finally:
cur.close()
conn.close()
def delete_session(self, session_id: str) -> bool:
"""Delete a session and all its messages."""
conn = self._get_conn()
try:
cur = conn.cursor()
cur.execute(f"""
DELETE FROM {self.sessions_table} WHERE session_id = %s
""", (session_id,))
deleted = cur.rowcount > 0
conn.commit()
return deleted
finally:
cur.close()
conn.close()
def list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""List sessions."""
conn = self._get_conn()
try:
cur = conn.cursor(dictionary=True)
conditions = []
params: List[Any] = []
if user_id:
conditions.append("user_id = %s")
params.append(user_id)
if agent_id:
conditions.append("agent_id = %s")
params.append(agent_id)
where_clause = ""
if conditions:
where_clause = "WHERE " + " AND ".join(conditions)
params.extend([limit, offset])
cur.execute(f"""
SELECT * FROM {self.sessions_table}
{where_clause}
ORDER BY updated_at DESC
LIMIT %s OFFSET %s
""", params)
return [
ConversationSession(
session_id=row["session_id"],
user_id=row["user_id"],
agent_id=row["agent_id"],
name=row["name"],
state=row["state"] if isinstance(row["state"], dict) else (json.loads(row["state"]) if row["state"] else None),
metadata=row["metadata"] if isinstance(row["metadata"], dict) else (json.loads(row["metadata"]) if row["metadata"] else None),
created_at=row["created_at"],
updated_at=row["updated_at"],
)
for row in cur.fetchall()
]
finally:
cur.close()
conn.close()
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Add a message to a session."""
message.session_id = session_id
conn = self._get_conn()
try:
cur = conn.cursor()
cur.execute(f"""
INSERT INTO {self.messages_table}
(id, session_id, role, content, tool_calls, tool_call_id, metadata, created_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
""", (
message.id,
session_id,
message.role,
message.content,
json.dumps(message.tool_calls) if message.tool_calls else None,
message.tool_call_id,
json.dumps(message.metadata) if message.metadata else None,
message.created_at,
))
conn.commit()
return message
finally:
cur.close()
conn.close()
def get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Get messages from a session."""
conn = self._get_conn()
try:
cur = conn.cursor(dictionary=True)
conditions = ["session_id = %s"]
params: List[Any] = [session_id]
if before:
conditions.append("created_at < %s")
params.append(before)
if after:
conditions.append("created_at > %s")
params.append(after)
where_clause = "WHERE " + " AND ".join(conditions)
limit_clause = f"LIMIT {limit}" if limit else ""
cur.execute(f"""
SELECT * FROM {self.messages_table}
{where_clause}
ORDER BY created_at ASC
{limit_clause}
""", params)
return [
ConversationMessage(
id=row["id"],
session_id=row["session_id"],
role=row["role"],
content=row["content"],
tool_calls=row["tool_calls"] if isinstance(row["tool_calls"], list) else (json.loads(row["tool_calls"]) if row["tool_calls"] else None),
tool_call_id=row["tool_call_id"],
metadata=row["metadata"] if isinstance(row["metadata"], dict) else (json.loads(row["metadata"]) if row["metadata"] else None),
created_at=row["created_at"],
)
for row in cur.fetchall()
]
finally:
cur.close()
conn.close()
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Delete messages."""
conn = self._get_conn()
try:
cur = conn.cursor()
if message_ids:
placeholders = ",".join(["%s"] * len(message_ids))
cur.execute(f"""
DELETE FROM {self.messages_table}
WHERE session_id = %s AND id IN ({placeholders})
""", [session_id] + message_ids)
else:
cur.execute(f"""
DELETE FROM {self.messages_table} WHERE session_id = %s
""", (session_id,))
deleted = cur.rowcount
conn.commit()
return deleted
finally:
cur.close()
conn.close()
def close(self) -> None:
"""Close the store."""
pass # Pool handles cleanup
@@ -0,0 +1,401 @@
"""
PostgreSQL implementation of ConversationStore.
Requires: psycopg2-binary or psycopg2
Install: pip install psycopg2-binary
"""
import json
import logging
from typing import Any, Dict, List, Optional
from .base import ConversationStore, ConversationSession, ConversationMessage
logger = logging.getLogger(__name__)
class PostgresConversationStore(ConversationStore):
"""
PostgreSQL-based conversation store.
Connection URL format: postgresql://user:password@host:port/database
Example:
store = PostgresConversationStore(
url="postgresql://postgres:password@localhost:5432/praisonai"
)
"""
SCHEMA_VERSION = "1.0.0"
def __init__(
self,
url: Optional[str] = None,
host: str = "localhost",
port: int = 5432,
database: str = "praisonai",
user: str = "postgres",
password: str = "",
schema: str = "public",
table_prefix: str = "praison_",
auto_create_tables: bool = True,
pool_size: int = 5,
):
"""
Initialize PostgreSQL conversation store.
Args:
url: Full connection URL (overrides individual params)
host: Database host
port: Database port
database: Database name
user: Database user
password: Database password
schema: PostgreSQL schema
table_prefix: Prefix for table names
auto_create_tables: Create tables if they don't exist
pool_size: Connection pool size
"""
try:
import psycopg2
from psycopg2 import pool as pg_pool
from psycopg2.extras import RealDictCursor
except ImportError:
raise ImportError(
"psycopg2 is required for PostgreSQL support. "
"Install with: pip install psycopg2-binary"
)
self._psycopg2 = psycopg2
self._RealDictCursor = RealDictCursor
self.schema = schema
self.table_prefix = table_prefix
self.sessions_table = f"{schema}.{table_prefix}sessions"
self.messages_table = f"{schema}.{table_prefix}messages"
# Build connection params
if url:
self._pool = pg_pool.ThreadedConnectionPool(1, pool_size, url)
else:
self._pool = pg_pool.ThreadedConnectionPool(
1, pool_size,
host=host,
port=port,
database=database,
user=user,
password=password,
)
if auto_create_tables:
self._create_tables()
def _get_conn(self):
"""Get a connection from the pool."""
return self._pool.getconn()
def _put_conn(self, conn):
"""Return a connection to the pool."""
self._pool.putconn(conn)
def _create_tables(self) -> None:
"""Create tables if they don't exist."""
conn = self._get_conn()
try:
with conn.cursor() as cur:
# Create schema if not exists
cur.execute(f"CREATE SCHEMA IF NOT EXISTS {self.schema}")
# Sessions table
cur.execute(f"""
CREATE TABLE IF NOT EXISTS {self.sessions_table} (
session_id VARCHAR(255) PRIMARY KEY,
user_id VARCHAR(255),
agent_id VARCHAR(255),
name VARCHAR(255),
state JSONB,
metadata JSONB,
created_at DOUBLE PRECISION,
updated_at DOUBLE PRECISION
)
""")
# Sessions indexes
cur.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}sessions_user
ON {self.sessions_table}(user_id)
""")
cur.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}sessions_agent
ON {self.sessions_table}(agent_id)
""")
cur.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}sessions_updated
ON {self.sessions_table}(updated_at DESC)
""")
# Messages table
cur.execute(f"""
CREATE TABLE IF NOT EXISTS {self.messages_table} (
id VARCHAR(255) PRIMARY KEY,
session_id VARCHAR(255) NOT NULL
REFERENCES {self.sessions_table}(session_id) ON DELETE CASCADE,
role VARCHAR(50) NOT NULL,
content TEXT,
tool_calls JSONB,
tool_call_id VARCHAR(255),
metadata JSONB,
created_at DOUBLE PRECISION DEFAULT EXTRACT(EPOCH FROM NOW())
)
""")
# Messages indexes
cur.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}messages_session
ON {self.messages_table}(session_id)
""")
cur.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}messages_created
ON {self.messages_table}(session_id, created_at)
""")
conn.commit()
logger.info(f"PostgreSQL tables created: {self.sessions_table}, {self.messages_table}")
finally:
self._put_conn(conn)
def create_session(self, session: ConversationSession) -> ConversationSession:
"""Create a new session."""
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute(f"""
INSERT INTO {self.sessions_table}
(session_id, user_id, agent_id, name, state, metadata, created_at, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
RETURNING session_id
""", (
session.session_id,
session.user_id,
session.agent_id,
session.name,
json.dumps(session.state) if session.state else None,
json.dumps(session.metadata) if session.metadata else None,
session.created_at,
session.updated_at,
))
conn.commit()
return session
finally:
self._put_conn(conn)
def get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Get a session by ID."""
conn = self._get_conn()
try:
with conn.cursor(cursor_factory=self._RealDictCursor) as cur:
cur.execute(f"""
SELECT * FROM {self.sessions_table} WHERE session_id = %s
""", (session_id,))
row = cur.fetchone()
if not row:
return None
return ConversationSession(
session_id=row["session_id"],
user_id=row["user_id"],
agent_id=row["agent_id"],
name=row["name"],
state=row["state"],
metadata=row["metadata"],
created_at=row["created_at"],
updated_at=row["updated_at"],
)
finally:
self._put_conn(conn)
def update_session(self, session: ConversationSession) -> ConversationSession:
"""Update an existing session."""
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute(f"""
UPDATE {self.sessions_table}
SET user_id = %s, agent_id = %s, name = %s,
state = %s, metadata = %s, updated_at = %s
WHERE session_id = %s
""", (
session.user_id,
session.agent_id,
session.name,
json.dumps(session.state) if session.state else None,
json.dumps(session.metadata) if session.metadata else None,
session.updated_at,
session.session_id,
))
conn.commit()
return session
finally:
self._put_conn(conn)
def delete_session(self, session_id: str) -> bool:
"""Delete a session and all its messages."""
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute(f"""
DELETE FROM {self.sessions_table} WHERE session_id = %s
""", (session_id,))
deleted = cur.rowcount > 0
conn.commit()
return deleted
finally:
self._put_conn(conn)
def list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""List sessions, optionally filtered by user or agent."""
conn = self._get_conn()
try:
with conn.cursor(cursor_factory=self._RealDictCursor) as cur:
conditions = []
params = []
if user_id:
conditions.append("user_id = %s")
params.append(user_id)
if agent_id:
conditions.append("agent_id = %s")
params.append(agent_id)
where_clause = ""
if conditions:
where_clause = "WHERE " + " AND ".join(conditions)
params.extend([limit, offset])
cur.execute(f"""
SELECT * FROM {self.sessions_table}
{where_clause}
ORDER BY updated_at DESC
LIMIT %s OFFSET %s
""", params)
return [
ConversationSession(
session_id=row["session_id"],
user_id=row["user_id"],
agent_id=row["agent_id"],
name=row["name"],
state=row["state"],
metadata=row["metadata"],
created_at=row["created_at"],
updated_at=row["updated_at"],
)
for row in cur.fetchall()
]
finally:
self._put_conn(conn)
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Add a message to a session."""
message.session_id = session_id
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute(f"""
INSERT INTO {self.messages_table}
(id, session_id, role, content, tool_calls, tool_call_id, metadata, created_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
""", (
message.id,
session_id,
message.role,
message.content,
json.dumps(message.tool_calls) if message.tool_calls else None,
message.tool_call_id,
json.dumps(message.metadata) if message.metadata else None,
message.created_at,
))
conn.commit()
return message
finally:
self._put_conn(conn)
def get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Get messages from a session."""
conn = self._get_conn()
try:
with conn.cursor(cursor_factory=self._RealDictCursor) as cur:
conditions = ["session_id = %s"]
params: List[Any] = [session_id]
if before:
conditions.append("created_at < %s")
params.append(before)
if after:
conditions.append("created_at > %s")
params.append(after)
where_clause = "WHERE " + " AND ".join(conditions)
limit_clause = f"LIMIT {limit}" if limit else ""
cur.execute(f"""
SELECT * FROM {self.messages_table}
{where_clause}
ORDER BY created_at ASC
{limit_clause}
""", params)
return [
ConversationMessage(
id=row["id"],
session_id=row["session_id"],
role=row["role"],
content=row["content"],
tool_calls=row["tool_calls"],
tool_call_id=row["tool_call_id"],
metadata=row["metadata"],
created_at=row["created_at"],
)
for row in cur.fetchall()
]
finally:
self._put_conn(conn)
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Delete messages. If message_ids is None, delete all messages in session."""
conn = self._get_conn()
try:
with conn.cursor() as cur:
if message_ids:
placeholders = ",".join(["%s"] * len(message_ids))
cur.execute(f"""
DELETE FROM {self.messages_table}
WHERE session_id = %s AND id IN ({placeholders})
""", [session_id] + message_ids)
else:
cur.execute(f"""
DELETE FROM {self.messages_table} WHERE session_id = %s
""", (session_id,))
deleted = cur.rowcount
conn.commit()
return deleted
finally:
self._put_conn(conn)
def close(self) -> None:
"""Close the store and release resources."""
if self._pool:
self._pool.closeall()
self._pool = None
@@ -0,0 +1,240 @@
"""
SingleStore implementation of ConversationStore.
Requires: singlestoredb
Install: pip install singlestoredb
"""
import json
import logging
from typing import Any, List, Optional
from .base import ConversationStore, ConversationSession, ConversationMessage
logger = logging.getLogger(__name__)
class SingleStoreConversationStore(ConversationStore):
"""
SingleStore-based conversation store.
SingleStore is MySQL-compatible with additional vector capabilities.
Example:
store = SingleStoreConversationStore(
url="singlestoredb://user:password@host:3306/database"
)
"""
SCHEMA_VERSION = "1.0.0"
def __init__(
self,
url: Optional[str] = None,
host: str = "localhost",
port: int = 3306,
database: str = "praisonai",
user: str = "root",
password: str = "",
table_prefix: str = "praison_",
auto_create_tables: bool = True,
):
try:
import singlestoredb as s2
except ImportError:
raise ImportError(
"singlestoredb is required for SingleStore support. "
"Install with: pip install singlestoredb"
)
self._s2 = s2
self.table_prefix = table_prefix
self.sessions_table = f"{table_prefix}sessions"
self.messages_table = f"{table_prefix}messages"
if url:
self._conn = s2.connect(url)
else:
self._conn = s2.connect(
host=host,
port=port,
database=database,
user=user,
password=password,
)
if auto_create_tables:
self._create_tables()
def _create_tables(self) -> None:
"""Create tables if they don't exist."""
cur = self._conn.cursor()
cur.execute(f"""
CREATE TABLE IF NOT EXISTS {self.sessions_table} (
session_id VARCHAR(255) PRIMARY KEY,
user_id VARCHAR(255),
agent_id VARCHAR(255),
name VARCHAR(255),
state JSON,
metadata JSON,
created_at DOUBLE,
updated_at DOUBLE,
KEY idx_user (user_id),
KEY idx_agent (agent_id)
)
""")
cur.execute(f"""
CREATE TABLE IF NOT EXISTS {self.messages_table} (
id VARCHAR(255) PRIMARY KEY,
session_id VARCHAR(255) NOT NULL,
role VARCHAR(50) NOT NULL,
content TEXT,
tool_calls JSON,
tool_call_id VARCHAR(255),
metadata JSON,
created_at DOUBLE,
KEY idx_session (session_id),
KEY idx_created (session_id, created_at)
)
""")
self._conn.commit()
logger.info(f"SingleStore tables created")
def create_session(self, session: ConversationSession) -> ConversationSession:
cur = self._conn.cursor()
cur.execute(f"""
INSERT INTO {self.sessions_table}
(session_id, user_id, agent_id, name, state, metadata, created_at, updated_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
""", (
session.session_id, session.user_id, session.agent_id, session.name,
json.dumps(session.state) if session.state else None,
json.dumps(session.metadata) if session.metadata else None,
session.created_at, session.updated_at,
))
self._conn.commit()
return session
def get_session(self, session_id: str) -> Optional[ConversationSession]:
cur = self._conn.cursor()
cur.execute(f"SELECT * FROM {self.sessions_table} WHERE session_id = %s", (session_id,))
row = cur.fetchone()
if not row:
return None
cols = [d[0] for d in cur.description]
data = dict(zip(cols, row))
return ConversationSession(
session_id=data["session_id"], user_id=data["user_id"],
agent_id=data["agent_id"], name=data["name"],
state=json.loads(data["state"]) if data["state"] else None,
metadata=json.loads(data["metadata"]) if data["metadata"] else None,
created_at=data["created_at"], updated_at=data["updated_at"],
)
def update_session(self, session: ConversationSession) -> ConversationSession:
cur = self._conn.cursor()
cur.execute(f"""
UPDATE {self.sessions_table}
SET user_id=%s, agent_id=%s, name=%s, state=%s, metadata=%s, updated_at=%s
WHERE session_id=%s
""", (
session.user_id, session.agent_id, session.name,
json.dumps(session.state) if session.state else None,
json.dumps(session.metadata) if session.metadata else None,
session.updated_at, session.session_id,
))
self._conn.commit()
return session
def delete_session(self, session_id: str) -> bool:
cur = self._conn.cursor()
cur.execute(f"DELETE FROM {self.messages_table} WHERE session_id = %s", (session_id,))
cur.execute(f"DELETE FROM {self.sessions_table} WHERE session_id = %s", (session_id,))
self._conn.commit()
return cur.rowcount > 0
def list_sessions(self, user_id: Optional[str] = None, agent_id: Optional[str] = None,
limit: int = 100, offset: int = 0) -> List[ConversationSession]:
cur = self._conn.cursor()
conditions, params = [], []
if user_id:
conditions.append("user_id = %s")
params.append(user_id)
if agent_id:
conditions.append("agent_id = %s")
params.append(agent_id)
where = "WHERE " + " AND ".join(conditions) if conditions else ""
params.extend([limit, offset])
cur.execute(f"SELECT * FROM {self.sessions_table} {where} ORDER BY updated_at DESC LIMIT %s OFFSET %s", params)
cols = [d[0] for d in cur.description]
return [ConversationSession(
session_id=dict(zip(cols, r))["session_id"],
user_id=dict(zip(cols, r))["user_id"],
agent_id=dict(zip(cols, r))["agent_id"],
name=dict(zip(cols, r))["name"],
state=json.loads(dict(zip(cols, r))["state"]) if dict(zip(cols, r))["state"] else None,
metadata=json.loads(dict(zip(cols, r))["metadata"]) if dict(zip(cols, r))["metadata"] else None,
created_at=dict(zip(cols, r))["created_at"],
updated_at=dict(zip(cols, r))["updated_at"],
) for r in cur.fetchall()]
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
message.session_id = session_id
cur = self._conn.cursor()
cur.execute(f"""
INSERT INTO {self.messages_table}
(id, session_id, role, content, tool_calls, tool_call_id, metadata, created_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
""", (
message.id, session_id, message.role, message.content,
json.dumps(message.tool_calls) if message.tool_calls else None,
message.tool_call_id,
json.dumps(message.metadata) if message.metadata else None,
message.created_at,
))
self._conn.commit()
return message
def get_messages(self, session_id: str, limit: Optional[int] = None,
before: Optional[float] = None, after: Optional[float] = None) -> List[ConversationMessage]:
cur = self._conn.cursor()
conditions, params = ["session_id = %s"], [session_id]
if before:
conditions.append("created_at < %s")
params.append(before)
if after:
conditions.append("created_at > %s")
params.append(after)
where = "WHERE " + " AND ".join(conditions)
limit_clause = f"LIMIT {limit}" if limit else ""
cur.execute(f"SELECT * FROM {self.messages_table} {where} ORDER BY created_at ASC {limit_clause}", params)
cols = [d[0] for d in cur.description]
return [ConversationMessage(
id=dict(zip(cols, r))["id"],
session_id=dict(zip(cols, r))["session_id"],
role=dict(zip(cols, r))["role"],
content=dict(zip(cols, r))["content"],
tool_calls=json.loads(dict(zip(cols, r))["tool_calls"]) if dict(zip(cols, r))["tool_calls"] else None,
tool_call_id=dict(zip(cols, r))["tool_call_id"],
metadata=json.loads(dict(zip(cols, r))["metadata"]) if dict(zip(cols, r))["metadata"] else None,
created_at=dict(zip(cols, r))["created_at"],
) for r in cur.fetchall()]
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
cur = self._conn.cursor()
if message_ids:
placeholders = ",".join(["%s"] * len(message_ids))
cur.execute(f"DELETE FROM {self.messages_table} WHERE session_id = %s AND id IN ({placeholders})",
[session_id] + message_ids)
else:
cur.execute(f"DELETE FROM {self.messages_table} WHERE session_id = %s", (session_id,))
self._conn.commit()
return cur.rowcount
def close(self) -> None:
if self._conn:
self._conn.close()
self._conn = None
@@ -0,0 +1,341 @@
"""
SQLite implementation of ConversationStore.
Zero external dependencies - uses built-in sqlite3 module.
"""
import json
import logging
import sqlite3
import threading
from typing import Any, List, Optional
from .base import ConversationStore, ConversationSession, ConversationMessage
logger = logging.getLogger(__name__)
class SQLiteConversationStore(ConversationStore):
"""
SQLite-based conversation store.
Zero external dependencies - uses Python's built-in sqlite3.
Example:
store = SQLiteConversationStore(
path="./praisonai.db"
)
"""
SCHEMA_VERSION = "1.0.0"
def __init__(
self,
path: str = "praisonai_conversations.db",
table_prefix: str = "praison_",
auto_create_tables: bool = True,
check_same_thread: bool = False,
):
"""
Initialize SQLite conversation store.
Args:
path: Path to SQLite database file
table_prefix: Prefix for table names
auto_create_tables: Create tables if they don't exist
check_same_thread: SQLite check_same_thread parameter
"""
self.path = path
self.table_prefix = table_prefix
self.sessions_table = f"{table_prefix}sessions"
self.messages_table = f"{table_prefix}messages"
self._check_same_thread = check_same_thread
self._local = threading.local()
if auto_create_tables:
self._create_tables()
def _get_conn(self) -> sqlite3.Connection:
"""Get thread-local connection."""
if not hasattr(self._local, "conn") or self._local.conn is None:
self._local.conn = sqlite3.connect(
self.path,
check_same_thread=self._check_same_thread
)
self._local.conn.row_factory = sqlite3.Row
return self._local.conn
def _create_tables(self) -> None:
"""Create tables if they don't exist."""
conn = self._get_conn()
cur = conn.cursor()
# Sessions table
cur.execute(f"""
CREATE TABLE IF NOT EXISTS {self.sessions_table} (
session_id TEXT PRIMARY KEY,
user_id TEXT,
agent_id TEXT,
name TEXT,
state TEXT,
metadata TEXT,
created_at REAL,
updated_at REAL
)
""")
# Sessions indexes
cur.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}sessions_user
ON {self.sessions_table}(user_id)
""")
cur.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}sessions_agent
ON {self.sessions_table}(agent_id)
""")
# Messages table
cur.execute(f"""
CREATE TABLE IF NOT EXISTS {self.messages_table} (
id TEXT PRIMARY KEY,
session_id TEXT NOT NULL,
role TEXT NOT NULL,
content TEXT,
tool_calls TEXT,
tool_call_id TEXT,
metadata TEXT,
created_at REAL,
FOREIGN KEY (session_id) REFERENCES {self.sessions_table}(session_id) ON DELETE CASCADE
)
""")
# Messages indexes
cur.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}messages_session
ON {self.messages_table}(session_id)
""")
cur.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}messages_created
ON {self.messages_table}(session_id, created_at)
""")
# Enable foreign keys
cur.execute("PRAGMA foreign_keys = ON")
conn.commit()
logger.info(f"SQLite tables created: {self.sessions_table}, {self.messages_table}")
def create_session(self, session: ConversationSession) -> ConversationSession:
"""Create a new session."""
conn = self._get_conn()
cur = conn.cursor()
cur.execute(f"""
INSERT INTO {self.sessions_table}
(session_id, user_id, agent_id, name, state, metadata, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""", (
session.session_id,
session.user_id,
session.agent_id,
session.name,
json.dumps(session.state) if session.state else None,
json.dumps(session.metadata) if session.metadata else None,
session.created_at,
session.updated_at,
))
conn.commit()
return session
def get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Get a session by ID."""
conn = self._get_conn()
cur = conn.cursor()
cur.execute(f"""
SELECT * FROM {self.sessions_table} WHERE session_id = ?
""", (session_id,))
row = cur.fetchone()
if not row:
return None
return ConversationSession(
session_id=row["session_id"],
user_id=row["user_id"],
agent_id=row["agent_id"],
name=row["name"],
state=json.loads(row["state"]) if row["state"] else None,
metadata=json.loads(row["metadata"]) if row["metadata"] else None,
created_at=row["created_at"],
updated_at=row["updated_at"],
)
def update_session(self, session: ConversationSession) -> ConversationSession:
"""Update an existing session."""
conn = self._get_conn()
cur = conn.cursor()
cur.execute(f"""
UPDATE {self.sessions_table}
SET user_id = ?, agent_id = ?, name = ?,
state = ?, metadata = ?, updated_at = ?
WHERE session_id = ?
""", (
session.user_id,
session.agent_id,
session.name,
json.dumps(session.state) if session.state else None,
json.dumps(session.metadata) if session.metadata else None,
session.updated_at,
session.session_id,
))
conn.commit()
return session
def delete_session(self, session_id: str) -> bool:
"""Delete a session and all its messages."""
conn = self._get_conn()
cur = conn.cursor()
# Delete messages first (foreign key)
cur.execute(f"DELETE FROM {self.messages_table} WHERE session_id = ?", (session_id,))
cur.execute(f"DELETE FROM {self.sessions_table} WHERE session_id = ?", (session_id,))
deleted = cur.rowcount > 0
conn.commit()
return deleted
def list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
"""List sessions, optionally filtered by user or agent."""
conn = self._get_conn()
cur = conn.cursor()
conditions = []
params: List[Any] = []
if user_id:
conditions.append("user_id = ?")
params.append(user_id)
if agent_id:
conditions.append("agent_id = ?")
params.append(agent_id)
where_clause = ""
if conditions:
where_clause = "WHERE " + " AND ".join(conditions)
params.extend([limit, offset])
cur.execute(f"""
SELECT * FROM {self.sessions_table}
{where_clause}
ORDER BY updated_at DESC
LIMIT ? OFFSET ?
""", params)
return [
ConversationSession(
session_id=row["session_id"],
user_id=row["user_id"],
agent_id=row["agent_id"],
name=row["name"],
state=json.loads(row["state"]) if row["state"] else None,
metadata=json.loads(row["metadata"]) if row["metadata"] else None,
created_at=row["created_at"],
updated_at=row["updated_at"],
)
for row in cur.fetchall()
]
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
"""Add a message to a session."""
message.session_id = session_id
conn = self._get_conn()
cur = conn.cursor()
cur.execute(f"""
INSERT INTO {self.messages_table}
(id, session_id, role, content, tool_calls, tool_call_id, metadata, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""", (
message.id,
session_id,
message.role,
message.content,
json.dumps(message.tool_calls) if message.tool_calls else None,
message.tool_call_id,
json.dumps(message.metadata) if message.metadata else None,
message.created_at,
))
conn.commit()
return message
def get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
"""Get messages from a session."""
conn = self._get_conn()
cur = conn.cursor()
conditions = ["session_id = ?"]
params: List[Any] = [session_id]
if before:
conditions.append("created_at < ?")
params.append(before)
if after:
conditions.append("created_at > ?")
params.append(after)
where_clause = "WHERE " + " AND ".join(conditions)
limit_clause = f"LIMIT {limit}" if limit else ""
cur.execute(f"""
SELECT * FROM {self.messages_table}
{where_clause}
ORDER BY created_at ASC
{limit_clause}
""", params)
return [
ConversationMessage(
id=row["id"],
session_id=row["session_id"],
role=row["role"],
content=row["content"],
tool_calls=json.loads(row["tool_calls"]) if row["tool_calls"] else None,
tool_call_id=row["tool_call_id"],
metadata=json.loads(row["metadata"]) if row["metadata"] else None,
created_at=row["created_at"],
)
for row in cur.fetchall()
]
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
"""Delete messages."""
conn = self._get_conn()
cur = conn.cursor()
if message_ids:
placeholders = ",".join(["?"] * len(message_ids))
cur.execute(f"""
DELETE FROM {self.messages_table}
WHERE session_id = ? AND id IN ({placeholders})
""", [session_id] + message_ids)
else:
cur.execute(f"""
DELETE FROM {self.messages_table} WHERE session_id = ?
""", (session_id,))
deleted = cur.rowcount
conn.commit()
return deleted
def close(self) -> None:
"""Close the store."""
if hasattr(self._local, "conn") and self._local.conn:
self._local.conn.close()
self._local.conn = None
@@ -0,0 +1,203 @@
"""
Supabase implementation of ConversationStore.
Requires: supabase
Install: pip install supabase
"""
import json
import logging
import time
from typing import Any, Dict, List, Optional
from .base import ConversationStore, ConversationSession, ConversationMessage
logger = logging.getLogger(__name__)
class SupabaseConversationStore(ConversationStore):
"""
Supabase-based conversation store.
Uses Supabase's PostgreSQL backend via REST API.
Example:
store = SupabaseConversationStore(
url="https://xxx.supabase.co",
key="your-anon-key"
)
"""
SCHEMA_VERSION = "1.0.0"
def __init__(
self,
url: Optional[str] = None,
key: Optional[str] = None,
table_prefix: str = "praison_",
auto_create_tables: bool = False, # Tables should be created via Supabase dashboard
):
"""
Initialize Supabase conversation store.
Args:
url: Supabase project URL
key: Supabase anon/service key
table_prefix: Prefix for table names
auto_create_tables: Not supported - use Supabase dashboard
"""
try:
from supabase import create_client, Client
except ImportError:
raise ImportError(
"supabase is required for Supabase support. "
"Install with: pip install supabase"
)
import os
url = url or os.getenv("SUPABASE_URL")
key = key or os.getenv("SUPABASE_KEY")
if not url or not key:
raise ValueError("Supabase URL and key are required")
self._client: Client = create_client(url, key)
self.table_prefix = table_prefix
self.sessions_table = f"{table_prefix}sessions"
self.messages_table = f"{table_prefix}messages"
if auto_create_tables:
logger.warning("auto_create_tables not supported for Supabase. Create tables via dashboard.")
def create_session(self, session: ConversationSession) -> ConversationSession:
data = {
"session_id": session.session_id,
"user_id": session.user_id,
"agent_id": session.agent_id,
"name": session.name,
"state": session.state,
"metadata": session.metadata,
"created_at": session.created_at,
"updated_at": session.updated_at,
}
self._client.table(self.sessions_table).insert(data).execute()
return session
def get_session(self, session_id: str) -> Optional[ConversationSession]:
result = self._client.table(self.sessions_table).select("*").eq("session_id", session_id).execute()
if not result.data:
return None
row = result.data[0]
return ConversationSession(
session_id=row["session_id"],
user_id=row.get("user_id"),
agent_id=row.get("agent_id"),
name=row.get("name"),
state=row.get("state"),
metadata=row.get("metadata"),
created_at=row.get("created_at"),
updated_at=row.get("updated_at"),
)
def update_session(self, session: ConversationSession) -> ConversationSession:
data = {
"user_id": session.user_id,
"agent_id": session.agent_id,
"name": session.name,
"state": session.state,
"metadata": session.metadata,
"updated_at": session.updated_at,
}
self._client.table(self.sessions_table).update(data).eq("session_id", session.session_id).execute()
return session
def delete_session(self, session_id: str) -> bool:
# Delete messages first
self._client.table(self.messages_table).delete().eq("session_id", session_id).execute()
result = self._client.table(self.sessions_table).delete().eq("session_id", session_id).execute()
return len(result.data) > 0 if result.data else False
def list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
query = self._client.table(self.sessions_table).select("*")
if user_id:
query = query.eq("user_id", user_id)
if agent_id:
query = query.eq("agent_id", agent_id)
query = query.order("updated_at", desc=True).range(offset, offset + limit - 1)
result = query.execute()
return [
ConversationSession(
session_id=row["session_id"],
user_id=row.get("user_id"),
agent_id=row.get("agent_id"),
name=row.get("name"),
state=row.get("state"),
metadata=row.get("metadata"),
created_at=row.get("created_at"),
updated_at=row.get("updated_at"),
)
for row in (result.data or [])
]
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
message.session_id = session_id
data = {
"id": message.id,
"session_id": session_id,
"role": message.role,
"content": message.content,
"tool_calls": message.tool_calls,
"tool_call_id": message.tool_call_id,
"metadata": message.metadata,
"created_at": message.created_at,
}
self._client.table(self.messages_table).insert(data).execute()
return message
def get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
query = self._client.table(self.messages_table).select("*").eq("session_id", session_id)
if before:
query = query.lt("created_at", before)
if after:
query = query.gt("created_at", after)
query = query.order("created_at", desc=False)
if limit:
query = query.limit(limit)
result = query.execute()
return [
ConversationMessage(
id=row["id"],
session_id=row["session_id"],
role=row["role"],
content=row.get("content", ""),
tool_calls=row.get("tool_calls"),
tool_call_id=row.get("tool_call_id"),
metadata=row.get("metadata"),
created_at=row.get("created_at"),
)
for row in (result.data or [])
]
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
if message_ids:
result = self._client.table(self.messages_table).delete().eq("session_id", session_id).in_("id", message_ids).execute()
else:
result = self._client.table(self.messages_table).delete().eq("session_id", session_id).execute()
return len(result.data) if result.data else 0
def close(self) -> None:
pass # Supabase client doesn't need explicit close
@@ -0,0 +1,287 @@
"""
SurrealDB implementation of ConversationStore.
Requires: surrealdb
Install: pip install surrealdb
"""
import json
import logging
from typing import List, Optional
from .base import ConversationStore, ConversationSession, ConversationMessage
logger = logging.getLogger(__name__)
class SurrealDBConversationStore(ConversationStore):
"""
SurrealDB-based conversation store.
SurrealDB is a multi-model database supporting SQL-like queries.
Example:
store = SurrealDBConversationStore(
url="ws://localhost:8000/rpc",
namespace="praisonai",
database="conversations"
)
"""
SCHEMA_VERSION = "1.0.0"
def __init__(
self,
url: str = "ws://localhost:8000/rpc",
namespace: str = "praisonai",
database: str = "conversations",
username: Optional[str] = None,
password: Optional[str] = None,
table_prefix: str = "praison_",
auto_create_tables: bool = True,
):
try:
from surrealdb import Surreal
except ImportError:
raise ImportError(
"surrealdb is required for SurrealDB support. "
"Install with: pip install surrealdb"
)
self._Surreal = Surreal
self.url = url
self.namespace = namespace
self.database = database
self.username = username
self.password = password
self.table_prefix = table_prefix
self.sessions_table = f"{table_prefix}sessions"
self.messages_table = f"{table_prefix}messages"
self._client = None
# Note: SurrealDB Python client is async, we'll use sync wrapper
self._init_sync()
def _init_sync(self):
"""Initialize synchronously using asyncio."""
import asyncio
async def _init():
self._client = self._Surreal(self.url)
await self._client.connect()
if self.username and self.password:
await self._client.signin({"user": self.username, "pass": self.password})
await self._client.use(self.namespace, self.database)
try:
loop = asyncio.get_event_loop()
except RuntimeError:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(_init())
def _run_sync(self, coro):
"""Run async coroutine synchronously."""
import asyncio
try:
loop = asyncio.get_event_loop()
except RuntimeError:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
return loop.run_until_complete(coro)
def create_session(self, session: ConversationSession) -> ConversationSession:
data = {
"session_id": session.session_id,
"user_id": session.user_id,
"agent_id": session.agent_id,
"name": session.name,
"state": session.state,
"metadata": session.metadata,
"created_at": session.created_at,
"updated_at": session.updated_at,
}
self._run_sync(self._client.create(self.sessions_table, data))
return session
def get_session(self, session_id: str) -> Optional[ConversationSession]:
result = self._run_sync(
self._client.query(
f"SELECT * FROM {self.sessions_table} WHERE session_id = $sid",
{"sid": session_id}
)
)
if not result or not result[0].get("result"):
return None
row = result[0]["result"][0]
return ConversationSession(
session_id=row["session_id"],
user_id=row.get("user_id"),
agent_id=row.get("agent_id"),
name=row.get("name"),
state=row.get("state"),
metadata=row.get("metadata"),
created_at=row.get("created_at"),
updated_at=row.get("updated_at"),
)
def update_session(self, session: ConversationSession) -> ConversationSession:
self._run_sync(
self._client.query(
f"""UPDATE {self.sessions_table} SET
user_id = $user_id, agent_id = $agent_id, name = $name,
state = $state, metadata = $metadata, updated_at = $updated_at
WHERE session_id = $session_id""",
{
"session_id": session.session_id,
"user_id": session.user_id,
"agent_id": session.agent_id,
"name": session.name,
"state": session.state,
"metadata": session.metadata,
"updated_at": session.updated_at,
}
)
)
return session
def delete_session(self, session_id: str) -> bool:
self._run_sync(
self._client.query(
f"DELETE FROM {self.messages_table} WHERE session_id = $sid",
{"sid": session_id}
)
)
result = self._run_sync(
self._client.query(
f"DELETE FROM {self.sessions_table} WHERE session_id = $sid",
{"sid": session_id}
)
)
return True
def list_sessions(
self,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
limit: int = 100,
offset: int = 0
) -> List[ConversationSession]:
conditions = []
params = {"limit": limit, "offset": offset}
if user_id:
conditions.append("user_id = $user_id")
params["user_id"] = user_id
if agent_id:
conditions.append("agent_id = $agent_id")
params["agent_id"] = agent_id
where = "WHERE " + " AND ".join(conditions) if conditions else ""
result = self._run_sync(
self._client.query(
f"SELECT * FROM {self.sessions_table} {where} ORDER BY updated_at DESC LIMIT $limit START $offset",
params
)
)
if not result or not result[0].get("result"):
return []
return [
ConversationSession(
session_id=row["session_id"],
user_id=row.get("user_id"),
agent_id=row.get("agent_id"),
name=row.get("name"),
state=row.get("state"),
metadata=row.get("metadata"),
created_at=row.get("created_at"),
updated_at=row.get("updated_at"),
)
for row in result[0]["result"]
]
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
message.session_id = session_id
data = {
"id": message.id,
"session_id": session_id,
"role": message.role,
"content": message.content,
"tool_calls": message.tool_calls,
"tool_call_id": message.tool_call_id,
"metadata": message.metadata,
"created_at": message.created_at,
}
self._run_sync(self._client.create(self.messages_table, data))
return message
def get_messages(
self,
session_id: str,
limit: Optional[int] = None,
before: Optional[float] = None,
after: Optional[float] = None
) -> List[ConversationMessage]:
conditions = ["session_id = $session_id"]
params = {"session_id": session_id}
if before:
conditions.append("created_at < $before")
params["before"] = before
if after:
conditions.append("created_at > $after")
params["after"] = after
where = "WHERE " + " AND ".join(conditions)
limit_clause = f"LIMIT {limit}" if limit else ""
result = self._run_sync(
self._client.query(
f"SELECT * FROM {self.messages_table} {where} ORDER BY created_at ASC {limit_clause}",
params
)
)
if not result or not result[0].get("result"):
return []
return [
ConversationMessage(
id=row["id"],
session_id=row["session_id"],
role=row["role"],
content=row.get("content", ""),
tool_calls=row.get("tool_calls"),
tool_call_id=row.get("tool_call_id"),
metadata=row.get("metadata"),
created_at=row.get("created_at"),
)
for row in result[0]["result"]
]
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
if message_ids:
self._run_sync(
self._client.query(
f"DELETE FROM {self.messages_table} WHERE session_id = $sid AND id IN $ids",
{"sid": session_id, "ids": message_ids}
)
)
return len(message_ids)
else:
self._run_sync(
self._client.query(
f"DELETE FROM {self.messages_table} WHERE session_id = $sid",
{"sid": session_id}
)
)
return -1
def close(self) -> None:
if self._client:
self._run_sync(self._client.close())
self._client = None
@@ -0,0 +1,301 @@
"""
Factory functions for creating store instances.
Provides lazy loading of backend implementations to avoid importing
unused dependencies.
"""
import logging
from typing import Any, Dict, Optional
from .config import PersistenceConfig
logger = logging.getLogger(__name__)
def create_conversation_store(
backend: str,
url: Optional[str] = None,
**options: Any
):
"""
Create a ConversationStore instance.
Args:
backend: Backend type (postgres, mysql, sqlite, singlestore, supabase, surrealdb)
url: Connection URL
**options: Backend-specific options
Returns:
ConversationStore instance
Example:
store = create_conversation_store(
"postgres",
url="postgresql://localhost:5432/praisonai"
)
"""
backend = backend.lower()
if backend == "postgres":
from .conversation.postgres import PostgresConversationStore
return PostgresConversationStore(url=url, **options)
elif backend == "mysql":
from .conversation.mysql import MySQLConversationStore
return MySQLConversationStore(url=url, **options)
elif backend == "sqlite":
from .conversation.sqlite import SQLiteConversationStore
path = url or options.pop("path", None)
return SQLiteConversationStore(path=path, **options)
elif backend == "singlestore":
from .conversation.singlestore import SingleStoreConversationStore
return SingleStoreConversationStore(url=url, **options)
elif backend == "supabase":
from .conversation.supabase import SupabaseConversationStore
return SupabaseConversationStore(url=url, **options)
elif backend == "surrealdb":
from .conversation.surrealdb import SurrealDBConversationStore
return SurrealDBConversationStore(url=url, **options)
elif backend == "json":
from .conversation.json_store import JSONConversationStore
path = url or options.pop("path", None) or "./praisonai_conversations"
return JSONConversationStore(path=path, **options)
elif backend in ("async_postgres", "asyncpg", "postgres_async"):
from .conversation.async_postgres import AsyncPostgresConversationStore
return AsyncPostgresConversationStore(url=url, **options)
elif backend in ("async_sqlite", "aiosqlite", "sqlite_async"):
from .conversation.async_sqlite import AsyncSQLiteConversationStore
path = url or options.pop("path", None)
return AsyncSQLiteConversationStore(path=path, **options)
elif backend in ("async_mysql", "aiomysql", "mysql_async"):
from .conversation.async_mysql import AsyncMySQLConversationStore
return AsyncMySQLConversationStore(url=url, **options)
else:
raise ValueError(
f"Unknown conversation store backend: {backend}. "
f"Supported: postgres, mysql, sqlite, json, singlestore, supabase, surrealdb, "
f"async_postgres, async_sqlite, async_mysql"
)
def create_knowledge_store(
backend: str,
url: Optional[str] = None,
**options: Any
):
"""
Create a KnowledgeStore instance.
Args:
backend: Backend type (qdrant, pinecone, chroma, weaviate, lancedb, milvus, pgvector, redis, cassandra, clickhouse)
url: Connection URL
**options: Backend-specific options
Returns:
KnowledgeStore instance
Example:
store = create_knowledge_store(
"qdrant",
url="http://localhost:6333"
)
"""
backend = backend.lower()
if backend == "qdrant":
from .knowledge.qdrant import QdrantKnowledgeStore
return QdrantKnowledgeStore(url=url, **options)
elif backend == "pinecone":
from .knowledge.pinecone import PineconeKnowledgeStore
return PineconeKnowledgeStore(**options)
elif backend == "chroma":
from .knowledge.chroma import ChromaKnowledgeStore
path = url or options.pop("path", None)
return ChromaKnowledgeStore(path=path, **options)
elif backend == "weaviate":
from .knowledge.weaviate import WeaviateKnowledgeStore
return WeaviateKnowledgeStore(url=url, **options)
elif backend == "lancedb":
from .knowledge.lancedb import LanceDBKnowledgeStore
path = url or options.pop("path", None)
return LanceDBKnowledgeStore(path=path, **options)
elif backend == "milvus":
from .knowledge.milvus import MilvusKnowledgeStore
return MilvusKnowledgeStore(url=url, **options)
elif backend == "pgvector":
from .knowledge.pgvector import PGVectorKnowledgeStore
return PGVectorKnowledgeStore(url=url, **options)
elif backend == "redis":
from .knowledge.redis_vector import RedisVectorKnowledgeStore
return RedisVectorKnowledgeStore(url=url, **options)
elif backend == "cassandra":
from .knowledge.cassandra import CassandraKnowledgeStore
return CassandraKnowledgeStore(**options)
elif backend == "clickhouse":
from .knowledge.clickhouse import ClickHouseKnowledgeStore
return ClickHouseKnowledgeStore(**options)
elif backend == "couchbase":
from .knowledge.couchbase import CouchbaseKnowledgeStore
return CouchbaseKnowledgeStore(**options)
elif backend in ("mongodb_vector", "mongodb_atlas", "mongo_vector"):
from .knowledge.mongodb_vector import MongoDBVectorKnowledgeStore
return MongoDBVectorKnowledgeStore(url=url, **options)
elif backend in ("singlestore_vector", "singlestore_v"):
from .knowledge.singlestore_vector import SingleStoreVectorKnowledgeStore
return SingleStoreVectorKnowledgeStore(url=url, **options)
elif backend in ("surrealdb_vector", "surrealdb_v"):
from .knowledge.surrealdb_vector import SurrealDBVectorKnowledgeStore
return SurrealDBVectorKnowledgeStore(url=url, **options)
elif backend in ("upstash_vector", "upstash_v"):
from .knowledge.upstash_vector import UpstashVectorKnowledgeStore
return UpstashVectorKnowledgeStore(url=url, **options)
elif backend == "lightrag":
from .knowledge.lightrag_adapter import LightRAGKnowledgeStore
return LightRAGKnowledgeStore(**options)
elif backend in ("langchain", "langchain_adapter"):
from .knowledge.langchain_adapter import LangChainKnowledgeStore
return LangChainKnowledgeStore(**options)
elif backend in ("llamaindex", "llama_index", "llamaindex_adapter"):
from .knowledge.llamaindex_adapter import LlamaIndexKnowledgeStore
return LlamaIndexKnowledgeStore(**options)
elif backend in ("cosmosdb", "cosmos", "azure_cosmos", "cosmosdb_vector"):
from .knowledge.cosmosdb_vector import CosmosDBVectorKnowledgeStore
return CosmosDBVectorKnowledgeStore(**options)
else:
raise ValueError(
f"Unknown knowledge store backend: {backend}. "
f"Supported: qdrant, pinecone, chroma, weaviate, lancedb, milvus, pgvector, redis, cassandra, clickhouse, "
f"couchbase, mongodb_vector, singlestore_vector, surrealdb_vector, upstash_vector, lightrag, langchain, llamaindex, cosmosdb"
)
def create_state_store(
backend: str,
url: Optional[str] = None,
**options: Any
):
"""
Create a StateStore instance.
Args:
backend: Backend type (redis, dynamodb, firestore, mongodb, upstash, memory)
url: Connection URL
**options: Backend-specific options
Returns:
StateStore instance
Example:
store = create_state_store(
"redis",
url="redis://localhost:6379"
)
"""
backend = backend.lower()
if backend == "redis":
from .state.redis import RedisStateStore
return RedisStateStore(url=url, **options)
elif backend == "dynamodb":
from .state.dynamodb import DynamoDBStateStore
return DynamoDBStateStore(**options)
elif backend == "firestore":
from .state.firestore import FirestoreStateStore
return FirestoreStateStore(**options)
elif backend == "mongodb":
from .state.mongodb import MongoDBStateStore
return MongoDBStateStore(url=url, **options)
elif backend == "upstash":
from .state.upstash import UpstashStateStore
return UpstashStateStore(url=url, **options)
elif backend == "memory":
from .state.memory import MemoryStateStore
return MemoryStateStore(**options)
elif backend == "gcs":
from .state.gcs import GCSStateStore
bucket = options.pop("bucket_name", None) or options.pop("bucket", None)
if not bucket:
raise ValueError("GCS state store requires 'bucket_name' option")
return GCSStateStore(bucket_name=bucket, **options)
elif backend in ("async_mongodb", "motor", "mongodb_async"):
from .state.async_mongodb import AsyncMongoDBStateStore
return AsyncMongoDBStateStore(url=url, **options)
else:
raise ValueError(
f"Unknown state store backend: {backend}. "
f"Supported: redis, dynamodb, firestore, mongodb, upstash, memory, gcs, async_mongodb"
)
def create_stores_from_config(config: PersistenceConfig) -> Dict[str, Any]:
"""
Create all configured stores from a PersistenceConfig.
Returns:
Dict with keys: conversation, knowledge, state (values may be None)
"""
stores = {
"conversation": None,
"knowledge": None,
"state": None,
}
if config.conversation_store:
stores["conversation"] = create_conversation_store(
config.conversation_store,
url=config.conversation_url,
**config.conversation_options
)
if config.knowledge_store:
stores["knowledge"] = create_knowledge_store(
config.knowledge_store,
url=config.knowledge_url,
**config.knowledge_options
)
if config.state_store:
stores["state"] = create_state_store(
config.state_store,
url=config.state_url,
**config.state_options
)
return stores
@@ -0,0 +1,18 @@
"""
Agent hooks for automatic persistence integration.
Provides wrapper functions to add persistence capabilities to PraisonAI agents
without modifying the core SDK.
"""
from .agent_hooks import (
wrap_agent_with_persistence,
PersistentAgent,
create_persistent_session,
)
__all__ = [
"wrap_agent_with_persistence",
"PersistentAgent",
"create_persistent_session",
]
@@ -0,0 +1,297 @@
"""
Agent hooks for automatic persistence integration.
Provides wrapper functions to add persistence capabilities to PraisonAI agents
without modifying the core SDK.
"""
import time
import uuid
import logging
from typing import Any, Dict, List, Optional, Callable
from functools import wraps
from ..orchestrator import PersistenceOrchestrator
from ..conversation.base import ConversationMessage, ConversationSession
logger = logging.getLogger(__name__)
def wrap_agent_with_persistence(
agent,
orchestrator: PersistenceOrchestrator,
session_id: Optional[str] = None,
user_id: Optional[str] = None,
auto_resume: bool = True,
):
"""
Wrap an agent to add automatic persistence hooks.
This function wraps the agent's chat method to automatically:
1. Load/create session on first message
2. Persist each user message
3. Persist each assistant response
4. Update session metadata after each interaction
Args:
agent: PraisonAI Agent instance
orchestrator: PersistenceOrchestrator instance
session_id: Session ID (auto-generated if None)
user_id: User ID for the session
auto_resume: Whether to auto-resume existing session
Returns:
The wrapped agent with persistence capabilities
Example:
from praisonai.persistence import PersistenceOrchestrator
from praisonai.persistence.hooks import wrap_agent_with_persistence
from praisonaiagents import Agent
orchestrator = PersistenceOrchestrator(...)
agent = Agent(name="Assistant", role="Helper")
# Wrap agent with persistence
agent = wrap_agent_with_persistence(agent, orchestrator, session_id="my-session")
# Now all chats are automatically persisted
response = agent.chat("Hello!")
"""
session_id = session_id or f"session-{uuid.uuid4().hex[:8]}"
user_id = user_id or getattr(agent, 'user_id', 'default')
# Store original chat method
original_chat = agent.chat
_session_initialized = [False] # Use list to allow mutation in closure
@wraps(original_chat)
def chat_with_persistence(prompt, *args, **kwargs):
nonlocal _session_initialized
# Initialize session on first call
if not _session_initialized[0]:
history = orchestrator.on_agent_start(
agent,
session_id=session_id,
user_id=user_id,
resume=auto_resume
)
# Inject history into agent's chat_history if resuming
if auto_resume and history:
agent.chat_history = [
{"role": msg.role, "content": msg.content}
for msg in history
]
logger.info(f"Resumed session {session_id} with {len(history)} messages")
_session_initialized[0] = True
# Persist user message
orchestrator.on_message(session_id, "user", prompt if isinstance(prompt, str) else str(prompt))
# Call original chat
response = original_chat(prompt, *args, **kwargs)
# Persist assistant response
if response:
orchestrator.on_message(session_id, "assistant", response)
return response
# Replace chat method
agent.chat = chat_with_persistence
# Add persistence-related methods to agent
agent._persistence_orchestrator = orchestrator
agent._persistence_session_id = session_id
def get_session():
"""Get the current session."""
return orchestrator.get_session(session_id)
def get_messages(limit: Optional[int] = None):
"""Get messages from the current session."""
return orchestrator.get_messages(session_id, limit=limit)
def end_session():
"""End the current session."""
orchestrator.on_agent_end(agent, session_id)
agent.get_session = get_session
agent.get_messages = get_messages
agent.end_session = end_session
return agent
class PersistentAgent:
"""
A wrapper class that provides persistence capabilities for any agent.
This is an alternative to wrap_agent_with_persistence that uses composition
instead of monkey-patching.
Example:
from praisonai.persistence import PersistenceOrchestrator
from praisonai.persistence.hooks import PersistentAgent
from praisonaiagents import Agent
orchestrator = PersistenceOrchestrator(...)
base_agent = Agent(name="Assistant", role="Helper")
agent = PersistentAgent(base_agent, orchestrator, session_id="my-session")
response = agent.chat("Hello!")
"""
def __init__(
self,
agent,
orchestrator: PersistenceOrchestrator,
session_id: Optional[str] = None,
user_id: Optional[str] = None,
auto_resume: bool = True,
):
self._agent = agent
self._orchestrator = orchestrator
self._session_id = session_id or f"session-{uuid.uuid4().hex[:8]}"
self._user_id = user_id or getattr(agent, 'user_id', 'default')
self._auto_resume = auto_resume
self._session_initialized = False
def __getattr__(self, name):
"""Delegate attribute access to wrapped agent."""
return getattr(self._agent, name)
def _ensure_session(self):
"""Ensure session is initialized."""
if not self._session_initialized:
history = self._orchestrator.on_agent_start(
self._agent,
session_id=self._session_id,
user_id=self._user_id,
resume=self._auto_resume
)
if self._auto_resume and history:
self._agent.chat_history = [
{"role": msg.role, "content": msg.content}
for msg in history
]
logger.info(f"Resumed session {self._session_id} with {len(history)} messages")
self._session_initialized = True
def chat(self, prompt, *args, **kwargs):
"""Chat with automatic persistence."""
self._ensure_session()
# Persist user message
self._orchestrator.on_message(
self._session_id, "user",
prompt if isinstance(prompt, str) else str(prompt)
)
# Call agent chat
response = self._agent.chat(prompt, *args, **kwargs)
# Persist assistant response
if response:
self._orchestrator.on_message(self._session_id, "assistant", response)
return response
def get_session(self) -> Optional[ConversationSession]:
"""Get the current session."""
return self._orchestrator.get_session(self._session_id)
def get_messages(self, limit: Optional[int] = None) -> List[ConversationMessage]:
"""Get messages from the current session."""
return self._orchestrator.get_messages(self._session_id, limit=limit)
def end_session(self):
"""End the current session."""
self._orchestrator.on_agent_end(self._agent, self._session_id)
@property
def session_id(self) -> str:
"""Get the session ID."""
return self._session_id
def create_persistent_session(
orchestrator: PersistenceOrchestrator,
session_id: Optional[str] = None,
user_id: Optional[str] = None,
):
"""
Create a context manager for persistent sessions.
Example:
from praisonai.persistence import PersistenceOrchestrator
from praisonai.persistence.hooks import create_persistent_session
from praisonaiagents import Agent
orchestrator = PersistenceOrchestrator(...)
with create_persistent_session(orchestrator, session_id="my-session") as session:
agent = Agent(name="Assistant", role="Helper")
# Messages are persisted within the session context
session.persist_message("user", "Hello!")
response = agent.chat("Hello!")
session.persist_message("assistant", response)
"""
class PersistentSession:
def __init__(self, orchestrator, session_id, user_id):
self.orchestrator = orchestrator
self.session_id = session_id or f"session-{uuid.uuid4().hex[:8]}"
self.user_id = user_id or "default"
self._started = False
def __enter__(self):
self.start()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.end()
return False
def start(self, agent=None):
"""Start the session."""
if not self._started:
class MockAgent:
name = "session-agent"
self.orchestrator.on_agent_start(
agent or MockAgent(),
session_id=self.session_id,
user_id=self.user_id
)
self._started = True
def end(self, agent=None):
"""End the session."""
if self._started:
class MockAgent:
name = "session-agent"
self.orchestrator.on_agent_end(
agent or MockAgent(),
self.session_id
)
def persist_message(self, role: str, content: str):
"""Persist a message to the session."""
self.orchestrator.on_message(self.session_id, role, content)
def get_messages(self, limit: Optional[int] = None):
"""Get messages from the session."""
return self.orchestrator.get_messages(self.session_id, limit=limit)
def get_session(self):
"""Get the session object."""
return self.orchestrator.get_session(self.session_id)
return PersistentSession(orchestrator, session_id, user_id)
@@ -0,0 +1,26 @@
"""
KnowledgeStore implementations for vector embeddings and semantic search.
Supported backends:
- Qdrant (qdrant)
- Pinecone (pinecone)
- ChromaDB (chroma)
- Weaviate (weaviate)
- LanceDB (lancedb)
- Milvus (milvus)
- PGVector (pgvector)
- Redis Vector (redis)
- Cassandra Vector (cassandra)
- ClickHouse Vector (clickhouse)
"""
__all__ = [
"KnowledgeStore",
"KnowledgeDocument",
]
def __getattr__(name: str):
if name in ("KnowledgeStore", "KnowledgeDocument"):
from .base import KnowledgeStore, KnowledgeDocument
return locals()[name]
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -0,0 +1,144 @@
"""
Base interfaces for KnowledgeStore.
KnowledgeStore handles vector embeddings and semantic search for RAG.
"""
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional
import time
import uuid
import hashlib
@dataclass
class KnowledgeDocument:
"""A document with vector embedding for semantic search."""
id: str = field(default_factory=lambda: str(uuid.uuid4()))
content: str = ""
embedding: Optional[List[float]] = None
metadata: Optional[Dict[str, Any]] = None
content_hash: Optional[str] = None
created_at: float = field(default_factory=time.time)
def __post_init__(self):
if self.content and not self.content_hash:
self.content_hash = hashlib.sha256(self.content.encode()).hexdigest()[:16]
def to_dict(self) -> Dict[str, Any]:
return {
"id": self.id,
"content": self.content,
"embedding": self.embedding,
"metadata": self.metadata,
"content_hash": self.content_hash,
"created_at": self.created_at,
}
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "KnowledgeDocument":
return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__})
class KnowledgeStore(ABC):
"""
Abstract base class for vector/knowledge persistence.
Implementations handle document storage and semantic search for different backends:
- Qdrant, Pinecone, ChromaDB, Weaviate (dedicated vector DBs)
- LanceDB, Milvus (embedded/distributed vector DBs)
- PGVector, Redis, Cassandra, ClickHouse (vector extensions)
"""
@abstractmethod
def create_collection(
self,
name: str,
dimension: int,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create a new collection/index for vectors."""
raise NotImplementedError
@abstractmethod
def delete_collection(self, name: str) -> bool:
"""Delete a collection and all its documents."""
raise NotImplementedError
@abstractmethod
def collection_exists(self, name: str) -> bool:
"""Check if a collection exists."""
raise NotImplementedError
@abstractmethod
def list_collections(self) -> List[str]:
"""List all collections."""
raise NotImplementedError
@abstractmethod
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents into a collection. Returns list of IDs."""
raise NotImplementedError
@abstractmethod
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert or update documents. Returns list of IDs."""
raise NotImplementedError
@abstractmethod
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents by embedding."""
raise NotImplementedError
@abstractmethod
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
raise NotImplementedError
@abstractmethod
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents by IDs or filters. Returns count deleted."""
raise NotImplementedError
@abstractmethod
def count(self, collection: str) -> int:
"""Count documents in a collection."""
raise NotImplementedError
@abstractmethod
def close(self) -> None:
"""Close the store and release resources."""
raise NotImplementedError
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
return False
@@ -0,0 +1,232 @@
"""
Cassandra Vector implementation of KnowledgeStore.
Requires: cassandra-driver
Install: pip install cassandra-driver
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class CassandraKnowledgeStore(KnowledgeStore):
"""
Cassandra-based knowledge store using vector search (SAI).
Requires Cassandra 5.0+ or DataStax Astra with vector search.
Example:
store = CassandraKnowledgeStore(
hosts=["localhost"],
keyspace="praisonai"
)
"""
def __init__(
self,
hosts: Optional[List[str]] = None,
keyspace: str = "praisonai",
username: Optional[str] = None,
password: Optional[str] = None,
port: int = 9042,
secure_connect_bundle: Optional[str] = None,
):
try:
from cassandra.cluster import Cluster
from cassandra.auth import PlainTextAuthProvider
except ImportError:
raise ImportError(
"cassandra-driver is required for Cassandra support. "
"Install with: pip install cassandra-driver"
)
self._Cluster = Cluster
self.keyspace = keyspace
auth = None
if username and password:
auth = PlainTextAuthProvider(username=username, password=password)
if secure_connect_bundle:
# Astra DB connection
from cassandra.cluster import Cluster
cloud_config = {"secure_connect_bundle": secure_connect_bundle}
self._cluster = Cluster(cloud=cloud_config, auth_provider=auth)
else:
hosts = hosts or ["localhost"]
self._cluster = Cluster(hosts, port=port, auth_provider=auth)
self._session = self._cluster.connect()
# Create keyspace if not exists
self._session.execute(f"""
CREATE KEYSPACE IF NOT EXISTS {keyspace}
WITH replication = {{'class': 'SimpleStrategy', 'replication_factor': 1}}
""")
self._session.set_keyspace(keyspace)
logger.info(f"Connected to Cassandra keyspace: {keyspace}")
def create_collection(
self,
name: str,
dimension: int,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create a new table with vector column."""
similarity_map = {"cosine": "COSINE", "euclidean": "EUCLIDEAN", "dot": "DOT_PRODUCT"}
self._session.execute(f"""
CREATE TABLE IF NOT EXISTS {name} (
id text PRIMARY KEY,
content text,
content_hash text,
created_at double,
embedding vector<float, {dimension}>
)
""")
# Create SAI index for vector search
self._session.execute(f"""
CREATE CUSTOM INDEX IF NOT EXISTS {name}_embedding_idx ON {name} (embedding)
USING 'StorageAttachedIndex'
WITH OPTIONS = {{'similarity_function': '{similarity_map.get(distance, "COSINE")}'}}
""")
logger.info(f"Created Cassandra table: {name}")
def delete_collection(self, name: str) -> bool:
"""Delete a table."""
try:
self._session.execute(f"DROP TABLE IF EXISTS {name}")
return True
except Exception as e:
logger.warning(f"Failed to delete table {name}: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if a table exists."""
result = self._session.execute("""
SELECT table_name FROM system_schema.tables
WHERE keyspace_name = %s AND table_name = %s
""", (self.keyspace, name))
return len(list(result)) > 0
def list_collections(self) -> List[str]:
"""List all tables."""
result = self._session.execute("""
SELECT table_name FROM system_schema.tables WHERE keyspace_name = %s
""", (self.keyspace,))
return [row.table_name for row in result]
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
ids = []
for doc in documents:
if doc.embedding is None:
raise ValueError(f"Document {doc.id} has no embedding")
self._session.execute(f"""
INSERT INTO {collection} (id, content, content_hash, created_at, embedding)
VALUES (%s, %s, %s, %s, %s)
""", (doc.id, doc.content, doc.content_hash or "", doc.created_at, doc.embedding))
ids.append(doc.id)
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert or update documents."""
return self.insert(collection, documents) # Cassandra INSERT is upsert
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents using ANN."""
result = self._session.execute(f"""
SELECT id, content, content_hash, created_at, similarity_cosine(embedding, %s) as score
FROM {collection}
ORDER BY embedding ANN OF %s
LIMIT %s
""", (query_embedding, query_embedding, limit))
documents = []
for row in result:
if score_threshold and row.score < score_threshold:
continue
doc = KnowledgeDocument(
id=row.id,
content=row.content,
embedding=None,
metadata={},
content_hash=row.content_hash,
created_at=row.created_at,
)
documents.append(doc)
return documents
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
documents = []
for doc_id in ids:
result = self._session.execute(f"""
SELECT id, content, content_hash, created_at, embedding
FROM {collection} WHERE id = %s
""", (doc_id,))
for row in result:
doc = KnowledgeDocument(
id=row.id,
content=row.content,
embedding=list(row.embedding) if row.embedding else None,
metadata={},
content_hash=row.content_hash,
created_at=row.created_at,
)
documents.append(doc)
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
if ids:
for doc_id in ids:
self._session.execute(f"DELETE FROM {collection} WHERE id = %s", (doc_id,))
return len(ids)
return 0
def count(self, collection: str) -> int:
"""Count documents."""
result = self._session.execute(f"SELECT COUNT(*) FROM {collection}")
return result.one()[0]
def close(self) -> None:
"""Close the store."""
if self._cluster:
self._cluster.shutdown()
@@ -0,0 +1,295 @@
"""
ChromaDB implementation of KnowledgeStore.
Requires: chromadb
Install: pip install chromadb
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class ChromaKnowledgeStore(KnowledgeStore):
"""
ChromaDB-based knowledge store for vector search.
Zero-config embedded vector database.
Example:
store = ChromaKnowledgeStore(
path="./chroma_db"
)
"""
def __init__(
self,
path: Optional[str] = None,
host: Optional[str] = None,
port: int = 8000,
persistent: bool = True,
):
"""
Initialize ChromaDB knowledge store.
Args:
path: Path for persistent storage (embedded mode)
host: Chroma server host (client mode)
port: Chroma server port
persistent: Use persistent storage
"""
try:
import chromadb
from chromadb.config import Settings
except ImportError:
raise ImportError(
"chromadb is required for ChromaDB support. "
"Install with: pip install chromadb"
)
self._chromadb = chromadb
if host:
# Client mode
self._client = chromadb.HttpClient(host=host, port=port)
elif path and persistent:
# Persistent embedded mode
self._client = chromadb.PersistentClient(
path=path,
settings=Settings(anonymized_telemetry=False)
)
else:
# In-memory mode
self._client = chromadb.Client(
settings=Settings(anonymized_telemetry=False)
)
self._collections: Dict[str, Any] = {}
logger.info(f"ChromaDB initialized (path={path}, host={host})")
def _get_collection(self, name: str):
"""Get or cache a collection."""
if name not in self._collections:
self._collections[name] = self._client.get_collection(name)
return self._collections[name]
def create_collection(
self,
name: str,
dimension: int,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create a new collection."""
distance_map = {
"cosine": "cosine",
"euclidean": "l2",
"dot": "ip",
}
self._client.create_collection(
name=name,
metadata={"hnsw:space": distance_map.get(distance, "cosine"), **(metadata or {})}
)
logger.info(f"Created ChromaDB collection: {name}")
def delete_collection(self, name: str) -> bool:
"""Delete a collection."""
try:
self._client.delete_collection(name)
if name in self._collections:
del self._collections[name]
return True
except Exception as e:
logger.warning(f"Failed to delete collection {name}: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if a collection exists."""
try:
self._client.get_collection(name)
return True
except Exception:
return False
def list_collections(self) -> List[str]:
"""List all collections."""
return [c.name for c in self._client.list_collections()]
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents into a collection."""
col = self._get_collection(collection)
ids = []
embeddings = []
contents = []
metadatas = []
for doc in documents:
if doc.embedding is None:
raise ValueError(f"Document {doc.id} has no embedding")
ids.append(doc.id)
embeddings.append(doc.embedding)
contents.append(doc.content)
metadatas.append({
"content_hash": doc.content_hash or "",
"created_at": doc.created_at,
**(doc.metadata or {})
})
col.add(
ids=ids,
embeddings=embeddings,
documents=contents,
metadatas=metadatas,
)
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert or update documents."""
col = self._get_collection(collection)
ids = []
embeddings = []
contents = []
metadatas = []
for doc in documents:
if doc.embedding is None:
raise ValueError(f"Document {doc.id} has no embedding")
ids.append(doc.id)
embeddings.append(doc.embedding)
contents.append(doc.content)
metadatas.append({
"content_hash": doc.content_hash or "",
"created_at": doc.created_at,
**(doc.metadata or {})
})
col.upsert(
ids=ids,
embeddings=embeddings,
documents=contents,
metadatas=metadatas,
)
return ids
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents."""
col = self._get_collection(collection)
where = None
if filters:
where = {k: v for k, v in filters.items()}
results = col.query(
query_embeddings=[query_embedding],
n_results=limit,
where=where,
include=["documents", "metadatas", "distances"],
)
documents = []
if results["ids"] and results["ids"][0]:
for i, doc_id in enumerate(results["ids"][0]):
distance = results["distances"][0][i] if results.get("distances") else None
# Convert distance to similarity score (for cosine: 1 - distance)
if score_threshold is not None and distance is not None:
score = 1 - distance
if score < score_threshold:
continue
metadata = results["metadatas"][0][i] if results.get("metadatas") else {}
content_hash = metadata.pop("content_hash", None)
created_at = metadata.pop("created_at", 0)
doc = KnowledgeDocument(
id=doc_id,
content=results["documents"][0][i] if results.get("documents") else "",
embedding=None,
metadata=metadata,
content_hash=content_hash,
created_at=created_at,
)
documents.append(doc)
return documents
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
col = self._get_collection(collection)
results = col.get(
ids=ids,
include=["documents", "metadatas", "embeddings"],
)
documents = []
if results["ids"]:
for i, doc_id in enumerate(results["ids"]):
metadata = results["metadatas"][i] if results.get("metadatas") else {}
content_hash = metadata.pop("content_hash", None)
created_at = metadata.pop("created_at", 0)
doc = KnowledgeDocument(
id=doc_id,
content=results["documents"][i] if results.get("documents") else "",
embedding=results["embeddings"][i] if results.get("embeddings") else None,
metadata=metadata,
content_hash=content_hash,
created_at=created_at,
)
documents.append(doc)
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
col = self._get_collection(collection)
if ids:
col.delete(ids=ids)
return len(ids)
elif filters:
col.delete(where=filters)
return -1 # Unknown count
return 0
def count(self, collection: str) -> int:
"""Count documents in a collection."""
col = self._get_collection(collection)
return col.count()
def close(self) -> None:
"""Close the store."""
self._collections.clear()
@@ -0,0 +1,242 @@
"""
ClickHouse Vector implementation of KnowledgeStore.
Requires: clickhouse-connect
Install: pip install clickhouse-connect
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class ClickHouseKnowledgeStore(KnowledgeStore):
"""
ClickHouse-based knowledge store using vector search.
Example:
store = ClickHouseKnowledgeStore(
host="localhost",
port=8123
)
"""
def __init__(
self,
host: str = "localhost",
port: int = 8123,
username: str = "default",
password: str = "",
database: str = "praisonai",
secure: bool = False,
):
try:
import clickhouse_connect
except ImportError:
raise ImportError(
"clickhouse-connect is required for ClickHouse support. "
"Install with: pip install clickhouse-connect"
)
self._client = clickhouse_connect.get_client(
host=host,
port=port,
username=username,
password=password,
database=database,
secure=secure,
)
self.database = database
# Create database if not exists
self._client.command(f"CREATE DATABASE IF NOT EXISTS {database}")
logger.info(f"Connected to ClickHouse database: {database}")
def _table_name(self, collection: str) -> str:
return f"{self.database}.praison_vec_{collection}"
def create_collection(
self,
name: str,
dimension: int,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create a new table with vector column."""
table = self._table_name(name)
self._client.command(f"""
CREATE TABLE IF NOT EXISTS {table} (
id String,
content String,
content_hash String,
created_at Float64,
embedding Array(Float32)
) ENGINE = MergeTree()
ORDER BY id
""")
logger.info(f"Created ClickHouse table: {table}")
def delete_collection(self, name: str) -> bool:
"""Delete a table."""
table = self._table_name(name)
try:
self._client.command(f"DROP TABLE IF EXISTS {table}")
return True
except Exception as e:
logger.warning(f"Failed to delete table {table}: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if a table exists."""
table = self._table_name(name)
result = self._client.query(f"EXISTS TABLE {table}")
return result.result_rows[0][0] == 1
def list_collections(self) -> List[str]:
"""List all collections."""
result = self._client.query(f"""
SELECT name FROM system.tables
WHERE database = '{self.database}' AND name LIKE 'praison_vec_%'
""")
return [row[0].replace("praison_vec_", "") for row in result.result_rows]
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
table = self._table_name(collection)
data = []
ids = []
for doc in documents:
if doc.embedding is None:
raise ValueError(f"Document {doc.id} has no embedding")
data.append([
doc.id,
doc.content,
doc.content_hash or "",
doc.created_at,
doc.embedding,
])
ids.append(doc.id)
self._client.insert(
table,
data,
column_names=["id", "content", "content_hash", "created_at", "embedding"]
)
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert or update documents."""
# ClickHouse doesn't have native upsert for MergeTree
# Delete then insert
ids = [doc.id for doc in documents]
self.delete(collection, ids)
return self.insert(collection, documents)
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents using cosine distance."""
table = self._table_name(collection)
# Convert embedding to string for query
embedding_str = "[" + ",".join(map(str, query_embedding)) + "]"
query = f"""
SELECT
id, content, content_hash, created_at,
1 - cosineDistance(embedding, {embedding_str}) as score
FROM {table}
"""
if score_threshold:
query += f" WHERE 1 - cosineDistance(embedding, {embedding_str}) >= {score_threshold}"
query += f" ORDER BY score DESC LIMIT {limit}"
result = self._client.query(query)
documents = []
for row in result.result_rows:
doc = KnowledgeDocument(
id=row[0],
content=row[1],
embedding=None,
metadata={},
content_hash=row[2],
created_at=row[3],
)
documents.append(doc)
return documents
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
table = self._table_name(collection)
id_list = ", ".join([f"'{i}'" for i in ids])
result = self._client.query(f"""
SELECT id, content, content_hash, created_at, embedding
FROM {table} WHERE id IN ({id_list})
""")
documents = []
for row in result.result_rows:
doc = KnowledgeDocument(
id=row[0],
content=row[1],
embedding=list(row[4]) if row[4] else None,
metadata={},
content_hash=row[2],
created_at=row[3],
)
documents.append(doc)
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
table = self._table_name(collection)
if ids:
id_list = ", ".join([f"'{i}'" for i in ids])
self._client.command(f"ALTER TABLE {table} DELETE WHERE id IN ({id_list})")
return len(ids)
return 0
def count(self, collection: str) -> int:
"""Count documents."""
table = self._table_name(collection)
result = self._client.query(f"SELECT COUNT(*) FROM {table}")
return result.result_rows[0][0]
def close(self) -> None:
"""Close the store."""
if self._client:
self._client.close()
@@ -0,0 +1,438 @@
"""
Azure Cosmos DB Vector implementation of KnowledgeStore.
Supports both MongoDB API and vCore modes.
Requires: pymongo (for MongoDB API) or azure-cosmos (for SQL API)
Install: pip install pymongo # or pip install azure-cosmos
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class CosmosDBVectorKnowledgeStore(KnowledgeStore):
"""
Azure Cosmos DB vector store for knowledge/RAG.
Supports MongoDB API with vector search capabilities.
Example:
store = CosmosDBVectorKnowledgeStore(
connection_string="mongodb+srv://...",
database="praisonai",
collection="vectors"
)
"""
def __init__(
self,
connection_string: Optional[str] = None,
database: str = "praisonai",
collection: str = "vectors",
index_name: str = "vector_index",
embedding_field: str = "embedding",
text_field: str = "content",
embedding_dim: int = 1536,
api_mode: str = "mongodb", # "mongodb" or "sql"
):
"""
Initialize Cosmos DB Vector store.
Args:
connection_string: Azure Cosmos DB connection string
database: Database name
collection: Collection name
index_name: Vector search index name
embedding_field: Field name for embeddings
text_field: Field name for text content
embedding_dim: Embedding dimension
api_mode: API mode ("mongodb" or "sql")
"""
self.connection_string = connection_string
self.database_name = database
self.collection_name = collection
self.index_name = index_name
self.embedding_field = embedding_field
self.text_field = text_field
self.embedding_dim = embedding_dim
self.api_mode = api_mode
self._client = None
self._db = None
self._collection = None
self._initialized = False
def _init_client(self):
"""Initialize Cosmos DB client lazily."""
if self._initialized:
return
if not self.connection_string:
import os
self.connection_string = os.getenv("COSMOS_CONNECTION_STRING")
if not self.connection_string:
raise ValueError(
"connection_string required. Set COSMOS_CONNECTION_STRING env var."
)
if self.api_mode == "mongodb":
self._init_mongodb_client()
else:
self._init_sql_client()
self._initialized = True
def _init_mongodb_client(self):
"""Initialize MongoDB API client."""
try:
from pymongo import MongoClient
except ImportError:
raise ImportError(
"pymongo is required for Cosmos DB MongoDB API. "
"Install with: pip install pymongo"
)
self._client = MongoClient(self.connection_string)
self._db = self._client[self.database_name]
self._collection = self._db[self.collection_name]
def _init_sql_client(self):
"""Initialize SQL API client."""
try:
from azure.cosmos import CosmosClient
except ImportError:
raise ImportError(
"azure-cosmos is required for Cosmos DB SQL API. "
"Install with: pip install azure-cosmos"
)
self._client = CosmosClient.from_connection_string(self.connection_string)
self._db = self._client.get_database_client(self.database_name)
self._collection = self._db.get_container_client(self.collection_name)
def create_collection(
self,
name: str,
dimension: int = 1536,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create collection with vector index."""
self._init_client()
if self.api_mode == "mongodb":
# Create vector search index via MongoDB command
try:
self._db.command({
"createIndexes": name,
"indexes": [{
"name": self.index_name,
"key": {self.embedding_field: "cosmosSearch"},
"cosmosSearchOptions": {
"kind": "vector-ivf",
"numLists": 100,
"similarity": distance,
"dimensions": dimension
}
}]
})
except Exception as e:
logger.warning(f"Index creation may require manual setup: {e}")
def delete_collection(self, name: str) -> bool:
"""Delete collection."""
self._init_client()
try:
if self.api_mode == "mongodb":
self._db.drop_collection(name)
else:
self._db.delete_container(name)
return True
except Exception as e:
logger.error(f"Failed to delete collection: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if collection exists."""
self._init_client()
if self.api_mode == "mongodb":
return name in self._db.list_collection_names()
return True # SQL API containers are pre-created
def list_collections(self) -> List[str]:
"""List collections."""
self._init_client()
if self.api_mode == "mongodb":
return self._db.list_collection_names()
return [c["id"] for c in self._db.list_containers()]
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
self._init_client()
coll = self._db[collection] if self.api_mode == "mongodb" else self._db.get_container_client(collection)
ids = []
for doc in documents:
item = {
"_id" if self.api_mode == "mongodb" else "id": doc.id,
self.text_field: doc.content,
self.embedding_field: doc.embedding,
"metadata": doc.metadata or {},
"content_hash": doc.content_hash,
"created_at": doc.created_at,
}
try:
if self.api_mode == "mongodb":
coll.insert_one(item)
else:
coll.create_item(item)
ids.append(doc.id)
except Exception as e:
logger.warning(f"Failed to insert {doc.id}: {e}")
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Upsert documents."""
self._init_client()
coll = self._db[collection] if self.api_mode == "mongodb" else self._db.get_container_client(collection)
ids = []
for doc in documents:
item = {
"_id" if self.api_mode == "mongodb" else "id": doc.id,
self.text_field: doc.content,
self.embedding_field: doc.embedding,
"metadata": doc.metadata or {},
"content_hash": doc.content_hash,
"created_at": doc.created_at,
}
try:
if self.api_mode == "mongodb":
coll.replace_one({"_id": doc.id}, item, upsert=True)
else:
coll.upsert_item(item)
ids.append(doc.id)
except Exception as e:
logger.warning(f"Failed to upsert {doc.id}: {e}")
return ids
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents using vector search."""
self._init_client()
if self.api_mode == "mongodb":
return self._search_mongodb(collection, query_embedding, limit, filters, score_threshold)
else:
return self._search_sql(collection, query_embedding, limit, filters, score_threshold)
def _search_mongodb(
self,
collection: str,
query_embedding: List[float],
limit: int,
filters: Optional[Dict[str, Any]],
score_threshold: Optional[float]
) -> List[KnowledgeDocument]:
"""MongoDB API vector search."""
coll = self._db[collection]
pipeline = [
{
"$search": {
"cosmosSearch": {
"vector": query_embedding,
"path": self.embedding_field,
"k": limit
},
"returnStoredSource": True
}
},
{
"$project": {
"similarityScore": {"$meta": "searchScore"},
self.text_field: 1,
"metadata": 1,
"content_hash": 1,
"created_at": 1
}
}
]
if filters:
pipeline.insert(1, {"$match": filters})
try:
results = list(coll.aggregate(pipeline))
documents = []
for doc in results:
score = doc.get("similarityScore", 0)
if score_threshold and score < score_threshold:
continue
documents.append(KnowledgeDocument(
id=str(doc.get("_id", "")),
content=doc.get(self.text_field, ""),
metadata={**(doc.get("metadata") or {}), "score": score},
content_hash=doc.get("content_hash"),
created_at=doc.get("created_at", 0)
))
return documents
except Exception as e:
logger.error(f"Vector search failed: {e}")
return []
def _search_sql(
self,
collection: str,
query_embedding: List[float],
limit: int,
filters: Optional[Dict[str, Any]],
score_threshold: Optional[float]
) -> List[KnowledgeDocument]:
"""SQL API vector search (requires DiskANN index)."""
coll = self._db.get_container_client(collection)
# SQL API uses VectorDistance function
query = f"""
SELECT TOP {limit} c.id, c.{self.text_field}, c.metadata, c.content_hash, c.created_at,
VectorDistance(c.{self.embedding_field}, @embedding) AS score
FROM c
ORDER BY VectorDistance(c.{self.embedding_field}, @embedding)
"""
try:
results = list(coll.query_items(
query=query,
parameters=[{"name": "@embedding", "value": query_embedding}],
enable_cross_partition_query=True
))
documents = []
for doc in results:
score = doc.get("score", 0)
if score_threshold and score < score_threshold:
continue
documents.append(KnowledgeDocument(
id=doc.get("id", ""),
content=doc.get(self.text_field, ""),
metadata={**(doc.get("metadata") or {}), "score": score},
content_hash=doc.get("content_hash"),
created_at=doc.get("created_at", 0)
))
return documents
except Exception as e:
logger.error(f"Vector search failed: {e}")
return []
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
self._init_client()
coll = self._db[collection] if self.api_mode == "mongodb" else self._db.get_container_client(collection)
documents = []
for doc_id in ids:
try:
if self.api_mode == "mongodb":
doc = coll.find_one({"_id": doc_id})
else:
doc = coll.read_item(item=doc_id, partition_key=doc_id)
if doc:
documents.append(KnowledgeDocument(
id=str(doc.get("_id" if self.api_mode == "mongodb" else "id", "")),
content=doc.get(self.text_field, ""),
embedding=doc.get(self.embedding_field),
metadata=doc.get("metadata"),
content_hash=doc.get("content_hash"),
created_at=doc.get("created_at", 0)
))
except Exception as e:
logger.warning(f"Failed to get {doc_id}: {e}")
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
self._init_client()
coll = self._db[collection] if self.api_mode == "mongodb" else self._db.get_container_client(collection)
if ids:
count = 0
for doc_id in ids:
try:
if self.api_mode == "mongodb":
coll.delete_one({"_id": doc_id})
else:
coll.delete_item(item=doc_id, partition_key=doc_id)
count += 1
except Exception as e:
logger.warning(f"Failed to delete {doc_id}: {e}")
return count
return 0
def count(self, collection: str) -> int:
"""Count documents."""
self._init_client()
if self.api_mode == "mongodb":
return self._db[collection].count_documents({})
else:
# SQL API count
coll = self._db.get_container_client(collection)
result = list(coll.query_items(
query="SELECT VALUE COUNT(1) FROM c",
enable_cross_partition_query=True
))
return result[0] if result else 0
def close(self) -> None:
"""Close the connection."""
if self._client:
if self.api_mode == "mongodb":
self._client.close()
self._client = None
self._db = None
self._collection = None
self._initialized = False
@@ -0,0 +1,286 @@
"""
Couchbase implementation of KnowledgeStore.
Requires: couchbase
Install: pip install couchbase
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class CouchbaseKnowledgeStore(KnowledgeStore):
"""
Couchbase vector store for knowledge/RAG.
Uses Couchbase's vector search capabilities.
Example:
store = CouchbaseKnowledgeStore(
connection_string="couchbase://localhost",
username="admin",
password="password",
bucket_name="praisonai"
)
"""
def __init__(
self,
connection_string: str = "couchbase://localhost",
username: str = "Administrator",
password: str = "",
bucket_name: str = "praisonai",
scope_name: str = "_default",
collection_name: str = "vectors",
index_name: str = "vector_index",
vector_field: str = "embedding",
text_field: str = "content",
embedding_dim: int = 1536,
):
"""
Initialize Couchbase knowledge store.
Args:
connection_string: Couchbase connection string
username: Couchbase username
password: Couchbase password
bucket_name: Bucket name
scope_name: Scope name
collection_name: Collection name
index_name: Vector search index name
vector_field: Field name for embeddings
text_field: Field name for text content
embedding_dim: Embedding dimension
"""
self.connection_string = connection_string
self.username = username
self.password = password
self.bucket_name = bucket_name
self.scope_name = scope_name
self.collection_name = collection_name
self.index_name = index_name
self.vector_field = vector_field
self.text_field = text_field
self.embedding_dim = embedding_dim
self._cluster = None
self._bucket = None
self._collection = None
self._initialized = False
def _init_client(self):
"""Initialize Couchbase client lazily."""
if self._initialized:
return
try:
from couchbase.cluster import Cluster
from couchbase.options import ClusterOptions
from couchbase.auth import PasswordAuthenticator
from datetime import timedelta
except ImportError:
raise ImportError(
"couchbase is required for Couchbase support. "
"Install with: pip install couchbase"
)
auth = PasswordAuthenticator(self.username, self.password)
options = ClusterOptions(auth)
self._cluster = Cluster(self.connection_string, options)
self._cluster.wait_until_ready(timedelta(seconds=10))
self._bucket = self._cluster.bucket(self.bucket_name)
scope = self._bucket.scope(self.scope_name)
self._collection = scope.collection(self.collection_name)
self._initialized = True
def insert(
self,
collection_name: str,
documents: List[KnowledgeDocument],
embeddings: Optional[List[List[float]]] = None,
) -> List[str]:
"""Insert documents into the store."""
self._init_client()
ids = []
for i, doc in enumerate(documents):
doc_id = doc.id or f"doc_{i}_{hash(doc.content)}"
data = {
self.text_field: doc.content,
"metadata": doc.metadata or {},
}
if embeddings and i < len(embeddings):
data[self.vector_field] = embeddings[i]
self._collection.upsert(doc_id, data)
ids.append(doc_id)
return ids
def upsert(
self,
collection_name: str,
documents: List[KnowledgeDocument],
embeddings: Optional[List[List[float]]] = None,
) -> List[str]:
"""Upsert documents (same as insert for Couchbase)."""
return self.insert(collection_name, documents, embeddings)
def search(
self,
collection_name: str,
query_embedding: List[float],
limit: int = 10,
filters: Optional[Dict[str, Any]] = None,
) -> List[KnowledgeDocument]:
"""Search for similar documents using vector search."""
self._init_client()
try:
from couchbase.vector_search import VectorQuery, VectorSearch
except ImportError:
logger.warning("Vector search not available in this Couchbase version")
return []
vector_query = VectorQuery(
self.vector_field,
query_embedding,
num_candidates=limit * 2
)
search_req = VectorSearch.from_vector_query(vector_query)
try:
scope = self._bucket.scope(self.scope_name)
result = scope.search(
self.index_name,
search_req,
limit=limit
)
documents = []
for row in result.rows():
doc_id = row.id
try:
doc_data = self._collection.get(doc_id).content_as[dict]
documents.append(KnowledgeDocument(
id=doc_id,
content=doc_data.get(self.text_field, ""),
metadata=doc_data.get("metadata", {}),
score=row.score if hasattr(row, 'score') else None
))
except Exception as e:
logger.warning(f"Failed to fetch document {doc_id}: {e}")
return documents
except Exception as e:
logger.error(f"Vector search failed: {e}")
return []
def delete(
self,
collection_name: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None,
) -> int:
"""Delete documents by ID."""
self._init_client()
if not ids:
return 0
count = 0
for doc_id in ids:
try:
self._collection.remove(doc_id)
count += 1
except Exception as e:
logger.warning(f"Failed to delete {doc_id}: {e}")
return count
def get(
self,
collection_name: str,
ids: List[str],
) -> List[KnowledgeDocument]:
"""Get documents by ID."""
self._init_client()
documents = []
for doc_id in ids:
try:
doc_data = self._collection.get(doc_id).content_as[dict]
documents.append(KnowledgeDocument(
id=doc_id,
content=doc_data.get(self.text_field, ""),
metadata=doc_data.get("metadata", {})
))
except Exception as e:
logger.warning(f"Failed to get {doc_id}: {e}")
return documents
def count(self, collection_name: str) -> int:
"""Count documents in collection."""
self._init_client()
try:
query = f"SELECT COUNT(*) as count FROM `{self.bucket_name}`.`{self.scope_name}`.`{self.collection_name}`"
result = self._cluster.query(query)
for row in result:
return row.get("count", 0)
except Exception as e:
logger.error(f"Count query failed: {e}")
return 0
def create_collection(self, collection_name: str, **kwargs) -> bool:
"""Create collection (Couchbase collections are pre-created)."""
self._init_client()
return True
def delete_collection(self, collection_name: str) -> bool:
"""Delete all documents in collection."""
self._init_client()
try:
query = f"DELETE FROM `{self.bucket_name}`.`{self.scope_name}`.`{self.collection_name}`"
self._cluster.query(query)
return True
except Exception as e:
logger.error(f"Delete collection failed: {e}")
return False
def list_collections(self) -> List[str]:
"""List available collections."""
self._init_client()
try:
collections = self._bucket.collections().get_all_scopes()
result = []
for scope in collections:
for coll in scope.collections:
result.append(f"{scope.name}.{coll.name}")
return result
except Exception as e:
logger.error(f"List collections failed: {e}")
return []
def close(self) -> None:
"""Close the connection."""
if self._cluster:
self._cluster.close()
self._cluster = None
self._bucket = None
self._collection = None
self._initialized = False
@@ -0,0 +1,216 @@
"""
LanceDB implementation of KnowledgeStore.
Requires: lancedb
Install: pip install lancedb
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class LanceDBKnowledgeStore(KnowledgeStore):
"""
LanceDB-based knowledge store for vector search.
Embedded serverless vector database.
Example:
store = LanceDBKnowledgeStore(
path="./lancedb"
)
"""
def __init__(
self,
path: str = "./lancedb",
uri: Optional[str] = None,
):
try:
import lancedb
except ImportError:
raise ImportError(
"lancedb is required for LanceDB support. "
"Install with: pip install lancedb"
)
self._lancedb = lancedb
self._db = lancedb.connect(uri or path)
self._tables: Dict[str, Any] = {}
logger.info(f"Connected to LanceDB at {uri or path}")
def create_collection(
self,
name: str,
dimension: int,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create a new table."""
import pyarrow as pa
schema = pa.schema([
pa.field("id", pa.string()),
pa.field("content", pa.string()),
pa.field("content_hash", pa.string()),
pa.field("created_at", pa.float64()),
pa.field("vector", pa.list_(pa.float32(), dimension)),
])
self._db.create_table(name, schema=schema)
logger.info(f"Created LanceDB table: {name}")
def delete_collection(self, name: str) -> bool:
"""Delete a table."""
try:
self._db.drop_table(name)
if name in self._tables:
del self._tables[name]
return True
except Exception as e:
logger.warning(f"Failed to delete table {name}: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if a table exists."""
return name in self._db.table_names()
def list_collections(self) -> List[str]:
"""List all tables."""
return self._db.table_names()
def _get_table(self, name: str):
"""Get or cache a table."""
if name not in self._tables:
self._tables[name] = self._db.open_table(name)
return self._tables[name]
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
table = self._get_table(collection)
data = []
ids = []
for doc in documents:
if doc.embedding is None:
raise ValueError(f"Document {doc.id} has no embedding")
data.append({
"id": doc.id,
"content": doc.content,
"content_hash": doc.content_hash or "",
"created_at": doc.created_at,
"vector": doc.embedding,
})
ids.append(doc.id)
table.add(data)
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert or update documents."""
# LanceDB doesn't have native upsert, delete then insert
ids = [doc.id for doc in documents]
self.delete(collection, ids)
return self.insert(collection, documents)
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents."""
table = self._get_table(collection)
query = table.search(query_embedding).limit(limit)
if filters:
where_clauses = [f"{k} = '{v}'" for k, v in filters.items()]
query = query.where(" AND ".join(where_clauses))
results = query.to_pandas()
documents = []
for _, row in results.iterrows():
if score_threshold and row.get("_distance", 1) > (1 - score_threshold):
continue
doc = KnowledgeDocument(
id=row["id"],
content=row["content"],
embedding=None,
metadata={},
content_hash=row.get("content_hash"),
created_at=row.get("created_at", 0),
)
documents.append(doc)
return documents
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
table = self._get_table(collection)
id_list = ", ".join([f"'{i}'" for i in ids])
results = table.search().where(f"id IN ({id_list})").to_pandas()
documents = []
for _, row in results.iterrows():
doc = KnowledgeDocument(
id=row["id"],
content=row["content"],
embedding=list(row["vector"]) if "vector" in row else None,
metadata={},
content_hash=row.get("content_hash"),
created_at=row.get("created_at", 0),
)
documents.append(doc)
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
table = self._get_table(collection)
if ids:
id_list = ", ".join([f"'{i}'" for i in ids])
table.delete(f"id IN ({id_list})")
return len(ids)
elif filters:
where_clauses = [f"{k} = '{v}'" for k, v in filters.items()]
table.delete(" AND ".join(where_clauses))
return -1
return 0
def count(self, collection: str) -> int:
"""Count documents."""
table = self._get_table(collection)
return table.count_rows()
def close(self) -> None:
"""Close the store."""
self._tables.clear()
@@ -0,0 +1,291 @@
"""
LangChain VectorStore adapter implementation of KnowledgeStore.
Wraps any LangChain VectorStore for use with PraisonAI.
Requires: langchain-core
Install: pip install langchain-core
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class LangChainKnowledgeStore(KnowledgeStore):
"""
LangChain VectorStore adapter for knowledge/RAG.
Wraps any LangChain VectorStore for use with PraisonAI's interface.
Example:
from langchain_chroma import Chroma
lc_store = Chroma(persist_directory="./chroma_db")
store = LangChainKnowledgeStore(vectorstore=lc_store)
"""
def __init__(
self,
vectorstore: Any = None,
vectorstore_cls: Optional[str] = None,
vectorstore_kwargs: Optional[Dict[str, Any]] = None,
):
"""
Initialize LangChain adapter.
Args:
vectorstore: Pre-initialized LangChain VectorStore instance
vectorstore_cls: Class name to import (e.g., "langchain_chroma.Chroma")
vectorstore_kwargs: Kwargs for vectorstore initialization
"""
self._vectorstore = vectorstore
self._vectorstore_cls = vectorstore_cls
self._vectorstore_kwargs = vectorstore_kwargs or {}
self._initialized = vectorstore is not None
def _init_client(self):
"""Initialize vectorstore lazily."""
if self._initialized:
return
if not self._vectorstore_cls:
raise ValueError(
"Either vectorstore or vectorstore_cls must be provided"
)
# Dynamic import of vectorstore class
parts = self._vectorstore_cls.rsplit(".", 1)
if len(parts) == 2:
module_name, class_name = parts
import importlib
module = importlib.import_module(module_name)
cls = getattr(module, class_name)
self._vectorstore = cls(**self._vectorstore_kwargs)
else:
raise ValueError(f"Invalid vectorstore_cls: {self._vectorstore_cls}")
self._initialized = True
def create_collection(
self,
name: str,
dimension: int = 1536,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create collection (handled by underlying vectorstore)."""
self._init_client()
logger.info(f"Collection creation handled by LangChain vectorstore")
def delete_collection(self, name: str) -> bool:
"""Delete collection."""
self._init_client()
try:
if hasattr(self._vectorstore, 'delete_collection'):
self._vectorstore.delete_collection()
return True
logger.warning("Underlying vectorstore doesn't support delete_collection")
return False
except Exception as e:
logger.error(f"Failed to delete collection: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if collection exists."""
self._init_client()
return self._vectorstore is not None
def list_collections(self) -> List[str]:
"""List collections."""
self._init_client()
if hasattr(self._vectorstore, 'list_collections'):
return self._vectorstore.list_collections()
return ["default"]
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
self._init_client()
try:
from langchain_core.documents import Document as LCDocument
except ImportError:
raise ImportError(
"langchain-core is required. Install with: pip install langchain-core"
)
lc_docs = []
ids = []
for doc in documents:
lc_doc = LCDocument(
page_content=doc.content,
metadata={
**(doc.metadata or {}),
"id": doc.id,
"content_hash": doc.content_hash,
"created_at": doc.created_at,
}
)
lc_docs.append(lc_doc)
ids.append(doc.id)
if hasattr(self._vectorstore, 'add_documents'):
self._vectorstore.add_documents(lc_docs, ids=ids)
else:
logger.warning("Vectorstore doesn't support add_documents")
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Upsert documents."""
# Most LangChain vectorstores handle upsert via add_documents
return self.insert(collection, documents)
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents."""
self._init_client()
try:
if hasattr(self._vectorstore, 'similarity_search_by_vector'):
results = self._vectorstore.similarity_search_by_vector(
query_embedding,
k=limit,
filter=filters
)
else:
logger.warning("Vectorstore doesn't support vector search")
return []
documents = []
for lc_doc in results:
metadata = lc_doc.metadata or {}
documents.append(KnowledgeDocument(
id=metadata.get("id", ""),
content=lc_doc.page_content,
metadata=metadata,
content_hash=metadata.get("content_hash"),
created_at=metadata.get("created_at", 0)
))
return documents
except Exception as e:
logger.error(f"Search failed: {e}")
return []
def similarity_search(
self,
query_text: str,
limit: int = 5,
filters: Optional[Dict[str, Any]] = None
) -> List[KnowledgeDocument]:
"""
Text-based similarity search (native LangChain method).
Args:
query_text: Query string
limit: Maximum results
filters: Metadata filters
Returns:
List of matching documents
"""
self._init_client()
try:
results = self._vectorstore.similarity_search(
query_text,
k=limit,
filter=filters
)
documents = []
for lc_doc in results:
metadata = lc_doc.metadata or {}
documents.append(KnowledgeDocument(
id=metadata.get("id", ""),
content=lc_doc.page_content,
metadata=metadata,
content_hash=metadata.get("content_hash"),
created_at=metadata.get("created_at", 0)
))
return documents
except Exception as e:
logger.error(f"Similarity search failed: {e}")
return []
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
self._init_client()
if hasattr(self._vectorstore, 'get'):
try:
results = self._vectorstore.get(ids)
documents = []
for lc_doc in results:
metadata = lc_doc.metadata or {}
documents.append(KnowledgeDocument(
id=metadata.get("id", ""),
content=lc_doc.page_content,
metadata=metadata
))
return documents
except Exception as e:
logger.warning(f"Get by ID failed: {e}")
return []
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
self._init_client()
if ids and hasattr(self._vectorstore, 'delete'):
try:
self._vectorstore.delete(ids)
return len(ids)
except Exception as e:
logger.error(f"Delete failed: {e}")
return 0
def count(self, collection: str) -> int:
"""Count documents."""
self._init_client()
if hasattr(self._vectorstore, '__len__'):
return len(self._vectorstore)
logger.warning("Vectorstore doesn't support counting")
return -1
def close(self) -> None:
"""Close the client."""
self._vectorstore = None
self._initialized = False
@@ -0,0 +1,212 @@
"""
LightRAG adapter implementation of KnowledgeStore.
Requires: lightrag
Install: pip install lightrag
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class LightRAGKnowledgeStore(KnowledgeStore):
"""
LightRAG adapter for knowledge/RAG.
Wraps LightRAG for use with PraisonAI's KnowledgeStore interface.
Example:
store = LightRAGKnowledgeStore(
working_dir="./lightrag_data"
)
"""
def __init__(
self,
working_dir: str = "./lightrag_data",
llm_model: str = "gpt-4o-mini",
embedding_model: str = "text-embedding-3-small",
**kwargs
):
"""
Initialize LightRAG adapter.
Args:
working_dir: Directory for LightRAG data
llm_model: LLM model for graph extraction
embedding_model: Embedding model name
**kwargs: Additional LightRAG options
"""
self.working_dir = working_dir
self.llm_model = llm_model
self.embedding_model = embedding_model
self.kwargs = kwargs
self._rag = None
self._initialized = False
def _init_client(self):
"""Initialize LightRAG client lazily."""
if self._initialized:
return
try:
from lightrag import LightRAG, QueryParam
from lightrag.llm import gpt_4o_mini_complete, openai_embedding
except ImportError:
raise ImportError(
"lightrag is required for LightRAG support. "
"Install with: pip install lightrag"
)
self._QueryParam = QueryParam
self._rag = LightRAG(
working_dir=self.working_dir,
llm_model_func=gpt_4o_mini_complete,
embedding_func=openai_embedding,
**self.kwargs
)
self._initialized = True
def create_collection(
self,
name: str,
dimension: int = 1536,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create collection (LightRAG uses working directory)."""
self._init_client()
logger.info(f"LightRAG uses working_dir: {self.working_dir}")
def delete_collection(self, name: str) -> bool:
"""Delete collection data."""
import shutil
import os
try:
if os.path.exists(self.working_dir):
shutil.rmtree(self.working_dir)
return True
except Exception as e:
logger.error(f"Failed to delete collection: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if working directory exists."""
import os
return os.path.exists(self.working_dir)
def list_collections(self) -> List[str]:
"""List collections (single collection per working_dir)."""
import os
if os.path.exists(self.working_dir):
return ["default"]
return []
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents into LightRAG."""
self._init_client()
ids = []
for doc in documents:
try:
self._rag.insert(doc.content)
ids.append(doc.id)
except Exception as e:
logger.warning(f"Failed to insert document {doc.id}: {e}")
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Upsert documents (same as insert for LightRAG)."""
return self.insert(collection, documents)
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search using LightRAG's query method."""
self._init_client()
# LightRAG uses text query, not embeddings directly
# This is a limitation - we need the original query text
logger.warning(
"LightRAG search requires text query, not embeddings. "
"Use query() method directly for best results."
)
return []
def query(
self,
query_text: str,
mode: str = "hybrid",
limit: int = 5
) -> str:
"""
Query LightRAG with text (native method).
Args:
query_text: The query string
mode: Query mode (naive, local, global, hybrid)
limit: Maximum results
Returns:
Query response as string
"""
self._init_client()
try:
result = self._rag.query(
query_text,
param=self._QueryParam(mode=mode, top_k=limit)
)
return result
except Exception as e:
logger.error(f"Query failed: {e}")
return ""
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs (not supported by LightRAG)."""
logger.warning("LightRAG does not support get by ID")
return []
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents (not directly supported)."""
logger.warning("LightRAG does not support individual document deletion")
return 0
def count(self, collection: str) -> int:
"""Count documents (not directly supported)."""
logger.warning("LightRAG does not support document counting")
return -1
def close(self) -> None:
"""Close the client."""
self._rag = None
self._initialized = False
@@ -0,0 +1,256 @@
"""
LlamaIndex VectorStore adapter implementation of KnowledgeStore.
Wraps any LlamaIndex VectorStore for use with PraisonAI.
Requires: llama-index-core
Install: pip install llama-index-core
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class LlamaIndexKnowledgeStore(KnowledgeStore):
"""
LlamaIndex VectorStore adapter for knowledge/RAG.
Wraps any LlamaIndex VectorStore for use with PraisonAI's interface.
Example:
from llama_index.vector_stores.chroma import ChromaVectorStore
li_store = ChromaVectorStore(chroma_collection=collection)
store = LlamaIndexKnowledgeStore(vector_store=li_store)
"""
def __init__(
self,
vector_store: Any = None,
vector_store_cls: Optional[str] = None,
vector_store_kwargs: Optional[Dict[str, Any]] = None,
):
"""
Initialize LlamaIndex adapter.
Args:
vector_store: Pre-initialized LlamaIndex VectorStore instance
vector_store_cls: Class name to import (e.g., "llama_index.vector_stores.chroma.ChromaVectorStore")
vector_store_kwargs: Kwargs for vector_store initialization
"""
self._vector_store = vector_store
self._vector_store_cls = vector_store_cls
self._vector_store_kwargs = vector_store_kwargs or {}
self._initialized = vector_store is not None
def _init_client(self):
"""Initialize vector store lazily."""
if self._initialized:
return
if not self._vector_store_cls:
raise ValueError(
"Either vector_store or vector_store_cls must be provided"
)
# Dynamic import of vector store class
parts = self._vector_store_cls.rsplit(".", 1)
if len(parts) == 2:
module_name, class_name = parts
import importlib
module = importlib.import_module(module_name)
cls = getattr(module, class_name)
self._vector_store = cls(**self._vector_store_kwargs)
else:
raise ValueError(f"Invalid vector_store_cls: {self._vector_store_cls}")
self._initialized = True
def create_collection(
self,
name: str,
dimension: int = 1536,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create collection (handled by underlying vector store)."""
self._init_client()
logger.info("Collection creation handled by LlamaIndex vector store")
def delete_collection(self, name: str) -> bool:
"""Delete collection."""
self._init_client()
try:
if hasattr(self._vector_store, 'delete'):
self._vector_store.delete(delete_all=True)
return True
logger.warning("Underlying vector store doesn't support delete")
return False
except Exception as e:
logger.error(f"Failed to delete collection: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if collection exists."""
self._init_client()
return self._vector_store is not None
def list_collections(self) -> List[str]:
"""List collections."""
self._init_client()
return ["default"]
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
self._init_client()
try:
from llama_index.core.schema import TextNode
except ImportError:
raise ImportError(
"llama-index-core is required. Install with: pip install llama-index-core"
)
nodes = []
ids = []
for doc in documents:
node = TextNode(
text=doc.content,
id_=doc.id,
metadata={
**(doc.metadata or {}),
"content_hash": doc.content_hash,
"created_at": doc.created_at,
},
embedding=doc.embedding
)
nodes.append(node)
ids.append(doc.id)
if hasattr(self._vector_store, 'add'):
self._vector_store.add(nodes)
else:
logger.warning("Vector store doesn't support add")
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Upsert documents."""
return self.insert(collection, documents)
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents."""
self._init_client()
try:
from llama_index.core.vector_stores import VectorStoreQuery
except ImportError:
raise ImportError(
"llama-index-core is required. Install with: pip install llama-index-core"
)
try:
query = VectorStoreQuery(
query_embedding=query_embedding,
similarity_top_k=limit,
)
result = self._vector_store.query(query)
documents = []
for i, node in enumerate(result.nodes or []):
score = result.similarities[i] if result.similarities else None
if score_threshold and score and score < score_threshold:
continue
metadata = node.metadata or {}
if score:
metadata["score"] = score
documents.append(KnowledgeDocument(
id=node.node_id or node.id_,
content=node.text or node.get_content(),
embedding=node.embedding,
metadata=metadata,
content_hash=metadata.get("content_hash"),
created_at=metadata.get("created_at", 0)
))
return documents
except Exception as e:
logger.error(f"Search failed: {e}")
return []
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
self._init_client()
documents = []
for doc_id in ids:
try:
if hasattr(self._vector_store, 'get'):
node = self._vector_store.get(doc_id)
if node:
metadata = node.metadata or {}
documents.append(KnowledgeDocument(
id=node.node_id or node.id_,
content=node.text or node.get_content(),
embedding=node.embedding,
metadata=metadata
))
except Exception as e:
logger.warning(f"Get by ID failed for {doc_id}: {e}")
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
self._init_client()
if ids and hasattr(self._vector_store, 'delete'):
try:
self._vector_store.delete(ref_doc_id=ids[0] if len(ids) == 1 else None)
return len(ids)
except Exception as e:
logger.error(f"Delete failed: {e}")
return 0
def count(self, collection: str) -> int:
"""Count documents."""
self._init_client()
logger.warning("LlamaIndex vector stores don't typically support counting")
return -1
def close(self) -> None:
"""Close the client."""
self._vector_store = None
self._initialized = False
@@ -0,0 +1,277 @@
"""
Milvus implementation of KnowledgeStore.
Requires: pymilvus
Install: pip install pymilvus
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class MilvusKnowledgeStore(KnowledgeStore):
"""
Milvus-based knowledge store for vector search.
Example:
store = MilvusKnowledgeStore(
host="localhost",
port=19530
)
"""
def __init__(
self,
url: Optional[str] = None,
host: str = "localhost",
port: int = 19530,
token: Optional[str] = None,
db_name: str = "default",
):
try:
from pymilvus import connections, Collection, FieldSchema, CollectionSchema, DataType, utility
except ImportError:
raise ImportError(
"pymilvus is required for Milvus support. "
"Install with: pip install pymilvus"
)
self._pymilvus = {
"connections": connections,
"Collection": Collection,
"FieldSchema": FieldSchema,
"CollectionSchema": CollectionSchema,
"DataType": DataType,
"utility": utility,
}
if url:
# Parse milvus://host:port
from urllib.parse import urlparse
parsed = urlparse(url)
host = parsed.hostname or host
port = parsed.port or port
connections.connect(
alias="default",
host=host,
port=port,
token=token,
db_name=db_name,
)
logger.info(f"Connected to Milvus at {host}:{port}")
def create_collection(
self,
name: str,
dimension: int,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create a new collection."""
FieldSchema = self._pymilvus["FieldSchema"]
CollectionSchema = self._pymilvus["CollectionSchema"]
DataType = self._pymilvus["DataType"]
Collection = self._pymilvus["Collection"]
fields = [
FieldSchema(name="id", dtype=DataType.VARCHAR, is_primary=True, max_length=256),
FieldSchema(name="content", dtype=DataType.VARCHAR, max_length=65535),
FieldSchema(name="content_hash", dtype=DataType.VARCHAR, max_length=64),
FieldSchema(name="created_at", dtype=DataType.DOUBLE),
FieldSchema(name="vector", dtype=DataType.FLOAT_VECTOR, dim=dimension),
]
schema = CollectionSchema(fields=fields, description="PraisonAI knowledge store")
collection = Collection(name=name, schema=schema)
# Create index
metric_map = {"cosine": "COSINE", "euclidean": "L2", "dot": "IP"}
index_params = {
"metric_type": metric_map.get(distance, "COSINE"),
"index_type": "HNSW",
"params": {"M": 16, "efConstruction": 256}
}
collection.create_index(field_name="vector", index_params=index_params)
logger.info(f"Created Milvus collection: {name}")
def delete_collection(self, name: str) -> bool:
"""Delete a collection."""
utility = self._pymilvus["utility"]
try:
utility.drop_collection(name)
return True
except Exception as e:
logger.warning(f"Failed to delete collection {name}: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if a collection exists."""
utility = self._pymilvus["utility"]
return utility.has_collection(name)
def list_collections(self) -> List[str]:
"""List all collections."""
utility = self._pymilvus["utility"]
return utility.list_collections()
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
Collection = self._pymilvus["Collection"]
col = Collection(collection)
col.load()
data = [
[doc.id for doc in documents],
[doc.content for doc in documents],
[doc.content_hash or "" for doc in documents],
[doc.created_at for doc in documents],
[doc.embedding for doc in documents],
]
col.insert(data)
col.flush()
return [doc.id for doc in documents]
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert or update documents."""
Collection = self._pymilvus["Collection"]
col = Collection(collection)
col.load()
data = [
[doc.id for doc in documents],
[doc.content for doc in documents],
[doc.content_hash or "" for doc in documents],
[doc.created_at for doc in documents],
[doc.embedding for doc in documents],
]
col.upsert(data)
col.flush()
return [doc.id for doc in documents]
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents."""
Collection = self._pymilvus["Collection"]
col = Collection(collection)
col.load()
search_params = {"metric_type": "COSINE", "params": {"ef": 64}}
expr = None
if filters:
conditions = [f'{k} == "{v}"' for k, v in filters.items()]
expr = " and ".join(conditions)
results = col.search(
data=[query_embedding],
anns_field="vector",
param=search_params,
limit=limit,
expr=expr,
output_fields=["id", "content", "content_hash", "created_at"],
)
documents = []
for hits in results:
for hit in hits:
if score_threshold and hit.score < score_threshold:
continue
doc = KnowledgeDocument(
id=hit.entity.get("id"),
content=hit.entity.get("content", ""),
embedding=None,
metadata={},
content_hash=hit.entity.get("content_hash"),
created_at=hit.entity.get("created_at", 0),
)
documents.append(doc)
return documents
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
Collection = self._pymilvus["Collection"]
col = Collection(collection)
col.load()
id_list = ", ".join([f'"{i}"' for i in ids])
expr = f"id in [{id_list}]"
results = col.query(
expr=expr,
output_fields=["id", "content", "content_hash", "created_at", "vector"],
)
documents = []
for row in results:
doc = KnowledgeDocument(
id=row["id"],
content=row.get("content", ""),
embedding=row.get("vector"),
metadata={},
content_hash=row.get("content_hash"),
created_at=row.get("created_at", 0),
)
documents.append(doc)
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
Collection = self._pymilvus["Collection"]
col = Collection(collection)
col.load()
if ids:
id_list = ", ".join([f'"{i}"' for i in ids])
expr = f"id in [{id_list}]"
col.delete(expr)
return len(ids)
elif filters:
conditions = [f'{k} == "{v}"' for k, v in filters.items()]
expr = " and ".join(conditions)
col.delete(expr)
return -1
return 0
def count(self, collection: str) -> int:
"""Count documents."""
Collection = self._pymilvus["Collection"]
col = Collection(collection)
return col.num_entities
def close(self) -> None:
"""Close the store."""
connections = self._pymilvus["connections"]
connections.disconnect("default")
@@ -0,0 +1,306 @@
"""
MongoDB Vector Search implementation of KnowledgeStore.
Requires: pymongo
Install: pip install pymongo
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class MongoDBVectorKnowledgeStore(KnowledgeStore):
"""
MongoDB Atlas Vector Search store for knowledge/RAG.
Uses MongoDB Atlas Vector Search capabilities.
Example:
store = MongoDBVectorKnowledgeStore(
url="mongodb+srv://...",
database="praisonai",
collection="vectors"
)
"""
def __init__(
self,
url: str = "mongodb://localhost:27017",
database: str = "praisonai",
collection: str = "vectors",
index_name: str = "vector_index",
embedding_field: str = "embedding",
text_field: str = "content",
embedding_dim: int = 1536,
):
"""
Initialize MongoDB Vector store.
Args:
url: MongoDB connection URL
database: Database name
collection: Collection name
index_name: Vector search index name
embedding_field: Field name for embeddings
text_field: Field name for text content
embedding_dim: Embedding dimension
"""
self.url = url
self.database_name = database
self.collection_name = collection
self.index_name = index_name
self.embedding_field = embedding_field
self.text_field = text_field
self.embedding_dim = embedding_dim
self._client = None
self._db = None
self._collection = None
self._initialized = False
def _init_client(self):
"""Initialize MongoDB client lazily."""
if self._initialized:
return
try:
from pymongo import MongoClient
except ImportError:
raise ImportError(
"pymongo is required for MongoDB Vector support. "
"Install with: pip install pymongo"
)
self._client = MongoClient(self.url)
self._db = self._client[self.database_name]
self._collection = self._db[self.collection_name]
self._initialized = True
def create_collection(
self,
name: str,
dimension: int = 1536,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create collection (MongoDB creates on first insert)."""
self._init_client()
# MongoDB creates collections automatically on first insert
# Vector search index must be created via Atlas UI or API
logger.info(
f"Collection '{name}' will be created on first insert. "
f"Note: Vector search index must be created via MongoDB Atlas."
)
def delete_collection(self, name: str) -> bool:
"""Delete a collection."""
self._init_client()
try:
self._db.drop_collection(name)
return True
except Exception as e:
logger.error(f"Failed to delete collection {name}: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if collection exists."""
self._init_client()
return name in self._db.list_collection_names()
def list_collections(self) -> List[str]:
"""List all collections."""
self._init_client()
return self._db.list_collection_names()
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents into collection."""
self._init_client()
coll = self._db[collection]
ids = []
for doc in documents:
mongo_doc = {
"_id": doc.id,
self.text_field: doc.content,
"metadata": doc.metadata or {},
"content_hash": doc.content_hash,
"created_at": doc.created_at,
}
if doc.embedding:
mongo_doc[self.embedding_field] = doc.embedding
try:
coll.insert_one(mongo_doc)
ids.append(doc.id)
except Exception as e:
logger.warning(f"Failed to insert document {doc.id}: {e}")
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Upsert documents into collection."""
self._init_client()
coll = self._db[collection]
ids = []
for doc in documents:
mongo_doc = {
self.text_field: doc.content,
"metadata": doc.metadata or {},
"content_hash": doc.content_hash,
"created_at": doc.created_at,
}
if doc.embedding:
mongo_doc[self.embedding_field] = doc.embedding
try:
coll.replace_one(
{"_id": doc.id},
{"_id": doc.id, **mongo_doc},
upsert=True
)
ids.append(doc.id)
except Exception as e:
logger.warning(f"Failed to upsert document {doc.id}: {e}")
return ids
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents using vector search."""
self._init_client()
coll = self._db[collection]
# Build aggregation pipeline for vector search
pipeline = [
{
"$vectorSearch": {
"index": self.index_name,
"path": self.embedding_field,
"queryVector": query_embedding,
"numCandidates": limit * 10,
"limit": limit
}
},
{
"$project": {
"_id": 1,
self.text_field: 1,
"metadata": 1,
"content_hash": 1,
"created_at": 1,
"score": {"$meta": "vectorSearchScore"}
}
}
]
# Add filter if provided
if filters:
pipeline[0]["$vectorSearch"]["filter"] = filters
try:
results = list(coll.aggregate(pipeline))
documents = []
for doc in results:
score = doc.get("score", 0)
if score_threshold and score < score_threshold:
continue
documents.append(KnowledgeDocument(
id=str(doc["_id"]),
content=doc.get(self.text_field, ""),
metadata={**(doc.get("metadata") or {}), "score": score},
content_hash=doc.get("content_hash"),
created_at=doc.get("created_at", 0)
))
return documents
except Exception as e:
logger.error(f"Vector search failed: {e}")
return []
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
self._init_client()
coll = self._db[collection]
documents = []
for doc_id in ids:
try:
doc = coll.find_one({"_id": doc_id})
if doc:
documents.append(KnowledgeDocument(
id=str(doc["_id"]),
content=doc.get(self.text_field, ""),
embedding=doc.get(self.embedding_field),
metadata=doc.get("metadata"),
content_hash=doc.get("content_hash"),
created_at=doc.get("created_at", 0)
))
except Exception as e:
logger.warning(f"Failed to get document {doc_id}: {e}")
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents by IDs or filters."""
self._init_client()
coll = self._db[collection]
if ids:
result = coll.delete_many({"_id": {"$in": ids}})
return result.deleted_count
elif filters:
result = coll.delete_many(filters)
return result.deleted_count
else:
return 0
def count(self, collection: str) -> int:
"""Count documents in collection."""
self._init_client()
return self._db[collection].count_documents({})
def close(self) -> None:
"""Close the connection."""
if self._client:
self._client.close()
self._client = None
self._db = None
self._collection = None
self._initialized = False
@@ -0,0 +1,335 @@
"""
PGVector implementation of KnowledgeStore.
Requires: pgvector, psycopg2-binary
Install: pip install pgvector psycopg2-binary
"""
import json
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class PGVectorKnowledgeStore(KnowledgeStore):
"""
PGVector-based knowledge store for vector search.
Uses PostgreSQL with pgvector extension.
Example:
store = PGVectorKnowledgeStore(
url="postgresql://localhost:5432/praisonai"
)
"""
def __init__(
self,
url: Optional[str] = None,
host: str = "localhost",
port: int = 5432,
database: str = "praisonai",
user: str = "postgres",
password: str = "",
schema: str = "public",
auto_create_extension: bool = True,
):
try:
import psycopg2
from psycopg2 import pool as pg_pool
from psycopg2.extras import RealDictCursor
except ImportError:
raise ImportError(
"psycopg2 is required for PGVector support. "
"Install with: pip install psycopg2-binary"
)
self._psycopg2 = psycopg2
self._RealDictCursor = RealDictCursor
self.schema = schema
if url:
self._pool = pg_pool.ThreadedConnectionPool(1, 5, url)
else:
self._pool = pg_pool.ThreadedConnectionPool(
1, 5,
host=host, port=port, database=database,
user=user, password=password,
)
if auto_create_extension:
self._create_extension()
def _get_conn(self):
return self._pool.getconn()
def _put_conn(self, conn):
self._pool.putconn(conn)
def _create_extension(self):
"""Create pgvector extension if not exists."""
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute("CREATE EXTENSION IF NOT EXISTS vector")
conn.commit()
finally:
self._put_conn(conn)
def _table_name(self, collection: str) -> str:
return f"{self.schema}.praison_vec_{collection}"
def create_collection(
self,
name: str,
dimension: int,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create a new collection table."""
table = self._table_name(name)
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute(f"""
CREATE TABLE IF NOT EXISTS {table} (
id VARCHAR(255) PRIMARY KEY,
content TEXT,
content_hash VARCHAR(64),
created_at DOUBLE PRECISION,
metadata JSONB,
embedding vector({dimension})
)
""")
# Create index based on distance metric
op_map = {"cosine": "vector_cosine_ops", "euclidean": "vector_l2_ops", "dot": "vector_ip_ops"}
ops = op_map.get(distance, "vector_cosine_ops")
cur.execute(f"""
CREATE INDEX IF NOT EXISTS idx_{name}_embedding
ON {table} USING hnsw (embedding {ops})
""")
conn.commit()
logger.info(f"Created PGVector table: {table}")
finally:
self._put_conn(conn)
def delete_collection(self, name: str) -> bool:
"""Delete a collection table."""
table = self._table_name(name)
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute(f"DROP TABLE IF EXISTS {table}")
conn.commit()
return True
except Exception as e:
logger.warning(f"Failed to delete table {table}: {e}")
return False
finally:
self._put_conn(conn)
def collection_exists(self, name: str) -> bool:
"""Check if a collection exists."""
table = self._table_name(name)
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute("""
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_schema = %s AND table_name = %s
)
""", (self.schema, f"praison_vec_{name}"))
return cur.fetchone()[0]
finally:
self._put_conn(conn)
def list_collections(self) -> List[str]:
"""List all collections."""
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute("""
SELECT table_name FROM information_schema.tables
WHERE table_schema = %s AND table_name LIKE 'praison_vec_%%'
""", (self.schema,))
return [row[0].replace("praison_vec_", "") for row in cur.fetchall()]
finally:
self._put_conn(conn)
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
table = self._table_name(collection)
conn = self._get_conn()
try:
with conn.cursor() as cur:
for doc in documents:
if doc.embedding is None:
raise ValueError(f"Document {doc.id} has no embedding")
cur.execute(f"""
INSERT INTO {table} (id, content, content_hash, created_at, metadata, embedding)
VALUES (%s, %s, %s, %s, %s, %s)
""", (
doc.id, doc.content, doc.content_hash, doc.created_at,
json.dumps(doc.metadata) if doc.metadata else None,
doc.embedding,
))
conn.commit()
return [doc.id for doc in documents]
finally:
self._put_conn(conn)
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert or update documents."""
table = self._table_name(collection)
conn = self._get_conn()
try:
with conn.cursor() as cur:
for doc in documents:
if doc.embedding is None:
raise ValueError(f"Document {doc.id} has no embedding")
cur.execute(f"""
INSERT INTO {table} (id, content, content_hash, created_at, metadata, embedding)
VALUES (%s, %s, %s, %s, %s, %s)
ON CONFLICT (id) DO UPDATE SET
content = EXCLUDED.content,
content_hash = EXCLUDED.content_hash,
created_at = EXCLUDED.created_at,
metadata = EXCLUDED.metadata,
embedding = EXCLUDED.embedding
""", (
doc.id, doc.content, doc.content_hash, doc.created_at,
json.dumps(doc.metadata) if doc.metadata else None,
doc.embedding,
))
conn.commit()
return [doc.id for doc in documents]
finally:
self._put_conn(conn)
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents."""
table = self._table_name(collection)
conn = self._get_conn()
try:
with conn.cursor(cursor_factory=self._RealDictCursor) as cur:
# Use cosine distance operator <=>
query = f"""
SELECT id, content, content_hash, created_at, metadata,
1 - (embedding <=> %s::vector) as score
FROM {table}
"""
params = [query_embedding]
if score_threshold:
query += " WHERE 1 - (embedding <=> %s::vector) >= %s"
params.extend([query_embedding, score_threshold])
query += " ORDER BY embedding <=> %s::vector LIMIT %s"
params.extend([query_embedding, limit])
cur.execute(query, params)
documents = []
for row in cur.fetchall():
doc = KnowledgeDocument(
id=row["id"],
content=row["content"],
embedding=None,
metadata=row["metadata"] or {},
content_hash=row["content_hash"],
created_at=row["created_at"],
)
documents.append(doc)
return documents
finally:
self._put_conn(conn)
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
table = self._table_name(collection)
conn = self._get_conn()
try:
with conn.cursor(cursor_factory=self._RealDictCursor) as cur:
placeholders = ",".join(["%s"] * len(ids))
cur.execute(f"""
SELECT id, content, content_hash, created_at, metadata, embedding::float[]
FROM {table} WHERE id IN ({placeholders})
""", ids)
documents = []
for row in cur.fetchall():
doc = KnowledgeDocument(
id=row["id"],
content=row["content"],
embedding=row["embedding"],
metadata=row["metadata"] or {},
content_hash=row["content_hash"],
created_at=row["created_at"],
)
documents.append(doc)
return documents
finally:
self._put_conn(conn)
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
table = self._table_name(collection)
conn = self._get_conn()
try:
with conn.cursor() as cur:
if ids:
placeholders = ",".join(["%s"] * len(ids))
cur.execute(f"DELETE FROM {table} WHERE id IN ({placeholders})", ids)
deleted = cur.rowcount
else:
deleted = 0
conn.commit()
return deleted
finally:
self._put_conn(conn)
def count(self, collection: str) -> int:
"""Count documents."""
table = self._table_name(collection)
conn = self._get_conn()
try:
with conn.cursor() as cur:
cur.execute(f"SELECT COUNT(*) FROM {table}")
return cur.fetchone()[0]
finally:
self._put_conn(conn)
def close(self) -> None:
"""Close the store."""
if self._pool:
self._pool.closeall()
@@ -0,0 +1,253 @@
"""
Pinecone implementation of KnowledgeStore.
Requires: pinecone-client
Install: pip install pinecone-client
"""
import logging
import os
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class PineconeKnowledgeStore(KnowledgeStore):
"""
Pinecone-based knowledge store for vector search.
Cloud-only managed vector database.
Example:
store = PineconeKnowledgeStore(
api_key="your-api-key",
index_name="praisonai"
)
"""
def __init__(
self,
api_key: Optional[str] = None,
index_name: Optional[str] = None,
environment: Optional[str] = None,
namespace: str = "",
):
try:
from pinecone import Pinecone
except ImportError:
raise ImportError(
"pinecone-client is required for Pinecone support. "
"Install with: pip install pinecone-client"
)
api_key = api_key or os.getenv("PINECONE_API_KEY")
if not api_key:
raise ValueError("Pinecone API key is required")
self._pc = Pinecone(api_key=api_key)
self._index_name = index_name or os.getenv("PINECONE_INDEX", "praisonai")
self._namespace = namespace
self._index = None
# Try to connect to existing index
try:
self._index = self._pc.Index(self._index_name)
logger.info(f"Connected to Pinecone index: {self._index_name}")
except Exception as e:
logger.warning(f"Index {self._index_name} not found: {e}")
def create_collection(
self,
name: str,
dimension: int,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create a new index (Pinecone calls them indexes, not collections)."""
from pinecone import ServerlessSpec
metric_map = {"cosine": "cosine", "euclidean": "euclidean", "dot": "dotproduct"}
self._pc.create_index(
name=name,
dimension=dimension,
metric=metric_map.get(distance, "cosine"),
spec=ServerlessSpec(cloud="aws", region="us-east-1")
)
self._index_name = name
self._index = self._pc.Index(name)
logger.info(f"Created Pinecone index: {name}")
def delete_collection(self, name: str) -> bool:
"""Delete an index."""
try:
self._pc.delete_index(name)
if name == self._index_name:
self._index = None
return True
except Exception as e:
logger.warning(f"Failed to delete index {name}: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if an index exists."""
try:
indexes = self._pc.list_indexes()
return any(idx.name == name for idx in indexes)
except Exception:
return False
def list_collections(self) -> List[str]:
"""List all indexes."""
indexes = self._pc.list_indexes()
return [idx.name for idx in indexes]
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
if collection != self._index_name:
self._index = self._pc.Index(collection)
self._index_name = collection
vectors = []
ids = []
for doc in documents:
if doc.embedding is None:
raise ValueError(f"Document {doc.id} has no embedding")
vectors.append({
"id": doc.id,
"values": doc.embedding,
"metadata": {
"content": doc.content,
"content_hash": doc.content_hash or "",
"created_at": doc.created_at,
**(doc.metadata or {})
}
})
ids.append(doc.id)
self._index.upsert(vectors=vectors, namespace=self._namespace)
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert or update documents."""
return self.insert(collection, documents)
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents."""
if collection != self._index_name:
self._index = self._pc.Index(collection)
self._index_name = collection
results = self._index.query(
vector=query_embedding,
top_k=limit,
filter=filters,
include_metadata=True,
namespace=self._namespace,
)
documents = []
for match in results.matches:
if score_threshold and match.score < score_threshold:
continue
metadata = match.metadata or {}
content = metadata.pop("content", "")
content_hash = metadata.pop("content_hash", None)
created_at = metadata.pop("created_at", 0)
doc = KnowledgeDocument(
id=match.id,
content=content,
embedding=None,
metadata=metadata,
content_hash=content_hash,
created_at=created_at,
)
documents.append(doc)
return documents
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
if collection != self._index_name:
self._index = self._pc.Index(collection)
self._index_name = collection
results = self._index.fetch(ids=ids, namespace=self._namespace)
documents = []
for doc_id, vector in results.vectors.items():
metadata = vector.metadata or {}
content = metadata.pop("content", "")
content_hash = metadata.pop("content_hash", None)
created_at = metadata.pop("created_at", 0)
doc = KnowledgeDocument(
id=doc_id,
content=content,
embedding=list(vector.values),
metadata=metadata,
content_hash=content_hash,
created_at=created_at,
)
documents.append(doc)
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
if collection != self._index_name:
self._index = self._pc.Index(collection)
self._index_name = collection
if ids:
self._index.delete(ids=ids, namespace=self._namespace)
return len(ids)
elif filters:
self._index.delete(filter=filters, namespace=self._namespace)
return -1
return 0
def count(self, collection: str) -> int:
"""Count documents."""
if collection != self._index_name:
self._index = self._pc.Index(collection)
self._index_name = collection
stats = self._index.describe_index_stats()
if self._namespace:
return stats.namespaces.get(self._namespace, {}).get("vector_count", 0)
return stats.total_vector_count
def close(self) -> None:
"""Close the store."""
self._index = None
@@ -0,0 +1,301 @@
"""
Qdrant implementation of KnowledgeStore.
Requires: qdrant-client
Install: pip install qdrant-client
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class QdrantKnowledgeStore(KnowledgeStore):
"""
Qdrant-based knowledge store for vector search.
Example:
store = QdrantKnowledgeStore(
url="http://localhost:6333"
)
"""
def __init__(
self,
url: Optional[str] = None,
host: str = "localhost",
port: int = 6333,
grpc_port: int = 6334,
api_key: Optional[str] = None,
prefer_grpc: bool = False,
timeout: int = 30,
):
"""
Initialize Qdrant knowledge store.
Args:
url: Full URL (overrides host/port)
host: Qdrant host
port: Qdrant HTTP port
grpc_port: Qdrant gRPC port
api_key: API key for Qdrant Cloud
prefer_grpc: Use gRPC instead of HTTP
timeout: Request timeout in seconds
"""
try:
from qdrant_client import QdrantClient
from qdrant_client.models import (
Distance, VectorParams, PointStruct,
Filter, FieldCondition, MatchValue
)
except ImportError:
raise ImportError(
"qdrant-client is required for Qdrant support. "
"Install with: pip install qdrant-client"
)
self._QdrantClient = QdrantClient
self._Distance = Distance
self._VectorParams = VectorParams
self._PointStruct = PointStruct
self._Filter = Filter
self._FieldCondition = FieldCondition
self._MatchValue = MatchValue
if url:
self._client = QdrantClient(url=url, api_key=api_key, timeout=timeout)
else:
self._client = QdrantClient(
host=host,
port=port,
grpc_port=grpc_port,
api_key=api_key,
prefer_grpc=prefer_grpc,
timeout=timeout,
)
logger.info(f"Connected to Qdrant at {url or f'{host}:{port}'}")
def _get_distance(self, distance: str) -> Any:
"""Convert distance string to Qdrant Distance enum."""
distance_map = {
"cosine": self._Distance.COSINE,
"euclidean": self._Distance.EUCLID,
"dot": self._Distance.DOT,
"manhattan": self._Distance.MANHATTAN,
}
return distance_map.get(distance.lower(), self._Distance.COSINE)
def create_collection(
self,
name: str,
dimension: int,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create a new collection."""
self._client.create_collection(
collection_name=name,
vectors_config=self._VectorParams(
size=dimension,
distance=self._get_distance(distance)
)
)
logger.info(f"Created Qdrant collection: {name} (dim={dimension}, distance={distance})")
def delete_collection(self, name: str) -> bool:
"""Delete a collection."""
try:
self._client.delete_collection(collection_name=name)
logger.info(f"Deleted Qdrant collection: {name}")
return True
except Exception as e:
logger.warning(f"Failed to delete collection {name}: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if a collection exists."""
try:
collections = self._client.get_collections()
return any(c.name == name for c in collections.collections)
except Exception:
return False
def list_collections(self) -> List[str]:
"""List all collections."""
collections = self._client.get_collections()
return [c.name for c in collections.collections]
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents into a collection."""
points = []
ids = []
for i, doc in enumerate(documents):
if doc.embedding is None:
raise ValueError(f"Document {doc.id} has no embedding")
point = self._PointStruct(
id=i, # Qdrant prefers numeric IDs for performance
vector=doc.embedding,
payload={
"doc_id": doc.id,
"content": doc.content,
"content_hash": doc.content_hash,
"created_at": doc.created_at,
**(doc.metadata or {})
}
)
points.append(point)
ids.append(doc.id)
self._client.upsert(collection_name=collection, points=points)
logger.debug(f"Inserted {len(documents)} documents into {collection}")
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert or update documents."""
return self.insert(collection, documents)
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents."""
search_filter = None
if filters:
conditions = [
self._FieldCondition(
key=k,
match=self._MatchValue(value=v)
)
for k, v in filters.items()
]
search_filter = self._Filter(must=conditions)
results = self._client.query_points(
collection_name=collection,
query=query_embedding,
limit=limit,
query_filter=search_filter,
score_threshold=score_threshold,
)
documents = []
for point in results.points:
payload = point.payload or {}
doc = KnowledgeDocument(
id=payload.get("doc_id", str(point.id)),
content=payload.get("content", ""),
embedding=None, # Don't return embeddings by default
metadata={
k: v for k, v in payload.items()
if k not in ("doc_id", "content", "content_hash", "created_at")
},
content_hash=payload.get("content_hash"),
created_at=payload.get("created_at", 0),
)
documents.append(doc)
return documents
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
# Qdrant uses numeric IDs internally, so we search by doc_id in payload
documents = []
for doc_id in ids:
results = self._client.scroll(
collection_name=collection,
scroll_filter=self._Filter(
must=[
self._FieldCondition(
key="doc_id",
match=self._MatchValue(value=doc_id)
)
]
),
limit=1,
)
for point in results[0]:
payload = point.payload or {}
doc = KnowledgeDocument(
id=payload.get("doc_id", str(point.id)),
content=payload.get("content", ""),
embedding=list(point.vector) if point.vector else None,
metadata={
k: v for k, v in payload.items()
if k not in ("doc_id", "content", "content_hash", "created_at")
},
content_hash=payload.get("content_hash"),
created_at=payload.get("created_at", 0),
)
documents.append(doc)
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents by IDs or filters."""
if ids:
# Delete by doc_id in payload
for doc_id in ids:
self._client.delete(
collection_name=collection,
points_selector=self._Filter(
must=[
self._FieldCondition(
key="doc_id",
match=self._MatchValue(value=doc_id)
)
]
)
)
return len(ids)
elif filters:
conditions = [
self._FieldCondition(
key=k,
match=self._MatchValue(value=v)
)
for k, v in filters.items()
]
self._client.delete(
collection_name=collection,
points_selector=self._Filter(must=conditions)
)
return -1 # Unknown count
return 0
def count(self, collection: str) -> int:
"""Count documents in a collection."""
info = self._client.get_collection(collection_name=collection)
return info.points_count or 0
def close(self) -> None:
"""Close the store."""
if self._client:
self._client.close()
self._client = None
@@ -0,0 +1,291 @@
"""
Redis Vector implementation of KnowledgeStore.
Requires: redis
Install: pip install redis
"""
import json
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class RedisVectorKnowledgeStore(KnowledgeStore):
"""
Redis-based knowledge store using RediSearch for vector search.
Requires Redis Stack or Redis with RediSearch module.
Example:
store = RedisVectorKnowledgeStore(
url="redis://localhost:6379"
)
"""
def __init__(
self,
url: Optional[str] = None,
host: str = "localhost",
port: int = 6379,
password: Optional[str] = None,
prefix: str = "praison:vec:",
):
try:
import redis
from redis.commands.search.field import VectorField, TextField, NumericField
from redis.commands.search.indexDefinition import IndexDefinition, IndexType
from redis.commands.search.query import Query
except ImportError:
raise ImportError(
"redis is required for Redis Vector support. "
"Install with: pip install redis"
)
self._redis = redis
self._VectorField = VectorField
self._TextField = TextField
self._NumericField = NumericField
self._IndexDefinition = IndexDefinition
self._IndexType = IndexType
self._Query = Query
self.prefix = prefix
if url:
self._client = redis.from_url(url, decode_responses=False)
else:
self._client = redis.Redis(
host=host, port=port, password=password, decode_responses=False
)
self._client.ping()
logger.info(f"Connected to Redis at {url or f'{host}:{port}'}")
def _index_name(self, collection: str) -> str:
return f"{self.prefix}{collection}:idx"
def _doc_key(self, collection: str, doc_id: str) -> str:
return f"{self.prefix}{collection}:{doc_id}"
def create_collection(
self,
name: str,
dimension: int,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create a new index."""
index_name = self._index_name(name)
distance_map = {"cosine": "COSINE", "euclidean": "L2", "dot": "IP"}
schema = (
self._TextField("content"),
self._TextField("content_hash"),
self._NumericField("created_at"),
self._VectorField(
"embedding",
"HNSW",
{
"TYPE": "FLOAT32",
"DIM": dimension,
"DISTANCE_METRIC": distance_map.get(distance, "COSINE"),
}
),
)
definition = self._IndexDefinition(
prefix=[f"{self.prefix}{name}:"],
index_type=self._IndexType.HASH,
)
try:
self._client.ft(index_name).create_index(schema, definition=definition)
logger.info(f"Created Redis index: {index_name}")
except Exception as e:
if "Index already exists" not in str(e):
raise
def delete_collection(self, name: str) -> bool:
"""Delete an index and all documents."""
index_name = self._index_name(name)
try:
self._client.ft(index_name).dropindex(delete_documents=True)
return True
except Exception as e:
logger.warning(f"Failed to delete index {index_name}: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if an index exists."""
index_name = self._index_name(name)
try:
self._client.ft(index_name).info()
return True
except Exception:
return False
def list_collections(self) -> List[str]:
"""List all collections."""
try:
indexes = self._client.execute_command("FT._LIST")
prefix = f"{self.prefix}"
return [
idx.decode().replace(prefix, "").replace(":idx", "")
for idx in indexes
if idx.decode().startswith(prefix)
]
except Exception:
return []
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
import numpy as np
ids = []
pipe = self._client.pipeline()
for doc in documents:
if doc.embedding is None:
raise ValueError(f"Document {doc.id} has no embedding")
key = self._doc_key(collection, doc.id)
embedding_bytes = np.array(doc.embedding, dtype=np.float32).tobytes()
pipe.hset(key, mapping={
"content": doc.content,
"content_hash": doc.content_hash or "",
"created_at": doc.created_at,
"embedding": embedding_bytes,
})
ids.append(doc.id)
pipe.execute()
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert or update documents."""
return self.insert(collection, documents)
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents."""
import numpy as np
index_name = self._index_name(collection)
query_bytes = np.array(query_embedding, dtype=np.float32).tobytes()
query_str = f"*=>[KNN {limit} @embedding $vec AS score]"
query = (
self._Query(query_str)
.return_fields("content", "content_hash", "created_at", "score")
.sort_by("score")
.dialect(2)
)
results = self._client.ft(index_name).search(
query, query_params={"vec": query_bytes}
)
documents = []
for doc in results.docs:
score = float(doc.score) if hasattr(doc, "score") else 0
# Redis returns distance, convert to similarity for cosine
similarity = 1 - score
if score_threshold and similarity < score_threshold:
continue
doc_id = doc.id.decode() if isinstance(doc.id, bytes) else doc.id
doc_id = doc_id.split(":")[-1] # Remove prefix
knowledge_doc = KnowledgeDocument(
id=doc_id,
content=doc.content.decode() if isinstance(doc.content, bytes) else doc.content,
embedding=None,
metadata={},
content_hash=doc.content_hash.decode() if isinstance(doc.content_hash, bytes) else doc.content_hash,
created_at=float(doc.created_at) if doc.created_at else 0,
)
documents.append(knowledge_doc)
return documents
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
import numpy as np
documents = []
for doc_id in ids:
key = self._doc_key(collection, doc_id)
data = self._client.hgetall(key)
if data:
embedding = None
if b"embedding" in data:
embedding = np.frombuffer(data[b"embedding"], dtype=np.float32).tolist()
doc = KnowledgeDocument(
id=doc_id,
content=data.get(b"content", b"").decode(),
embedding=embedding,
metadata={},
content_hash=data.get(b"content_hash", b"").decode(),
created_at=float(data.get(b"created_at", 0)),
)
documents.append(doc)
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
if ids:
pipe = self._client.pipeline()
for doc_id in ids:
key = self._doc_key(collection, doc_id)
pipe.delete(key)
pipe.execute()
return len(ids)
return 0
def count(self, collection: str) -> int:
"""Count documents."""
index_name = self._index_name(collection)
try:
info = self._client.ft(index_name).info()
return int(info.get("num_docs", 0))
except Exception:
return 0
def close(self) -> None:
"""Close the store."""
if self._client:
self._client.close()
@@ -0,0 +1,299 @@
"""
SingleStore Vector implementation of KnowledgeStore.
Requires: singlestoredb
Install: pip install singlestoredb
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class SingleStoreVectorKnowledgeStore(KnowledgeStore):
"""
SingleStore vector store for knowledge/RAG.
Uses SingleStore's native vector capabilities.
Example:
store = SingleStoreVectorKnowledgeStore(
url="singlestoredb://user:pass@host:3306/db"
)
"""
def __init__(
self,
url: Optional[str] = None,
host: str = "localhost",
port: int = 3306,
database: str = "praisonai",
user: str = "root",
password: str = "",
table_prefix: str = "praisonai_",
embedding_dim: int = 1536,
):
"""
Initialize SingleStore Vector store.
Args:
url: SingleStore connection URL
host: Database host
port: Database port
database: Database name
user: Database user
password: Database password
table_prefix: Prefix for table names
embedding_dim: Embedding dimension
"""
self.url = url
self.host = host
self.port = port
self.database = database
self.user = user
self.password = password
self.table_prefix = table_prefix
self.embedding_dim = embedding_dim
self._conn = None
self._initialized = False
def _init_client(self):
"""Initialize SingleStore client lazily."""
if self._initialized:
return
try:
import singlestoredb as s2
except ImportError:
raise ImportError(
"singlestoredb is required for SingleStore Vector support. "
"Install with: pip install singlestoredb"
)
if self.url:
self._conn = s2.connect(self.url)
else:
self._conn = s2.connect(
host=self.host,
port=self.port,
database=self.database,
user=self.user,
password=self.password
)
self._initialized = True
def create_collection(
self,
name: str,
dimension: int = 1536,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create a vector table."""
self._init_client()
table_name = f"{self.table_prefix}{name}"
with self._conn.cursor() as cur:
cur.execute(f"""
CREATE TABLE IF NOT EXISTS {table_name} (
id VARCHAR(255) PRIMARY KEY,
content TEXT,
embedding VECTOR({dimension}),
metadata JSON,
content_hash VARCHAR(64),
created_at DOUBLE
)
""")
def delete_collection(self, name: str) -> bool:
"""Delete a vector table."""
self._init_client()
table_name = f"{self.table_prefix}{name}"
try:
with self._conn.cursor() as cur:
cur.execute(f"DROP TABLE IF EXISTS {table_name}")
return True
except Exception as e:
logger.error(f"Failed to delete collection {name}: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if table exists."""
self._init_client()
table_name = f"{self.table_prefix}{name}"
with self._conn.cursor() as cur:
cur.execute(f"""
SELECT COUNT(*) FROM information_schema.tables
WHERE table_schema = %s AND table_name = %s
""", (self.database, table_name))
return cur.fetchone()[0] > 0
def list_collections(self) -> List[str]:
"""List all vector tables."""
self._init_client()
with self._conn.cursor() as cur:
cur.execute(f"""
SELECT table_name FROM information_schema.tables
WHERE table_schema = %s AND table_name LIKE %s
""", (self.database, f"{self.table_prefix}%"))
return [row[0][len(self.table_prefix):] for row in cur.fetchall()]
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
self._init_client()
table_name = f"{self.table_prefix}{collection}"
ids = []
with self._conn.cursor() as cur:
for doc in documents:
embedding_str = str(doc.embedding) if doc.embedding else None
cur.execute(f"""
INSERT INTO {table_name} (id, content, embedding, metadata, content_hash, created_at)
VALUES (%s, %s, %s, %s, %s, %s)
""", (doc.id, doc.content, embedding_str,
str(doc.metadata) if doc.metadata else None,
doc.content_hash, doc.created_at))
ids.append(doc.id)
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Upsert documents."""
self._init_client()
table_name = f"{self.table_prefix}{collection}"
ids = []
with self._conn.cursor() as cur:
for doc in documents:
embedding_str = str(doc.embedding) if doc.embedding else None
cur.execute(f"""
REPLACE INTO {table_name} (id, content, embedding, metadata, content_hash, created_at)
VALUES (%s, %s, %s, %s, %s, %s)
""", (doc.id, doc.content, embedding_str,
str(doc.metadata) if doc.metadata else None,
doc.content_hash, doc.created_at))
ids.append(doc.id)
return ids
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents using vector search."""
self._init_client()
table_name = f"{self.table_prefix}{collection}"
embedding_str = str(query_embedding)
with self._conn.cursor() as cur:
cur.execute(f"""
SELECT id, content, metadata, content_hash, created_at,
DOT_PRODUCT(embedding, %s) as score
FROM {table_name}
ORDER BY score DESC
LIMIT %s
""", (embedding_str, limit))
documents = []
for row in cur.fetchall():
score = row[5] if len(row) > 5 else 0
if score_threshold and score < score_threshold:
continue
documents.append(KnowledgeDocument(
id=row[0],
content=row[1],
metadata=eval(row[2]) if row[2] else None,
content_hash=row[3],
created_at=row[4]
))
return documents
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
self._init_client()
table_name = f"{self.table_prefix}{collection}"
placeholders = ','.join(['%s'] * len(ids))
with self._conn.cursor() as cur:
cur.execute(f"""
SELECT id, content, metadata, content_hash, created_at
FROM {table_name}
WHERE id IN ({placeholders})
""", ids)
return [
KnowledgeDocument(
id=row[0],
content=row[1],
metadata=eval(row[2]) if row[2] else None,
content_hash=row[3],
created_at=row[4]
)
for row in cur.fetchall()
]
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
self._init_client()
table_name = f"{self.table_prefix}{collection}"
with self._conn.cursor() as cur:
if ids:
placeholders = ','.join(['%s'] * len(ids))
cur.execute(f"DELETE FROM {table_name} WHERE id IN ({placeholders})", ids)
else:
cur.execute(f"DELETE FROM {table_name}")
return cur.rowcount
def count(self, collection: str) -> int:
"""Count documents."""
self._init_client()
table_name = f"{self.table_prefix}{collection}"
with self._conn.cursor() as cur:
cur.execute(f"SELECT COUNT(*) FROM {table_name}")
return cur.fetchone()[0]
def close(self) -> None:
"""Close the connection."""
if self._conn:
self._conn.close()
self._conn = None
self._initialized = False
@@ -0,0 +1,309 @@
"""
SurrealDB Vector implementation of KnowledgeStore.
Requires: surrealdb
Install: pip install surrealdb
"""
import logging
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class SurrealDBVectorKnowledgeStore(KnowledgeStore):
"""
SurrealDB vector store for knowledge/RAG.
Uses SurrealDB's vector capabilities.
Example:
store = SurrealDBVectorKnowledgeStore(
url="ws://localhost:8000/rpc",
namespace="praisonai",
database="vectors"
)
"""
def __init__(
self,
url: str = "ws://localhost:8000/rpc",
namespace: str = "praisonai",
database: str = "vectors",
username: str = "root",
password: str = "root",
embedding_dim: int = 1536,
):
"""
Initialize SurrealDB Vector store.
Args:
url: SurrealDB WebSocket URL
namespace: SurrealDB namespace
database: SurrealDB database
username: Authentication username
password: Authentication password
embedding_dim: Embedding dimension
"""
self.url = url
self.namespace = namespace
self.database = database
self.username = username
self.password = password
self.embedding_dim = embedding_dim
self._client = None
self._initialized = False
def _init_client(self):
"""Initialize SurrealDB client lazily."""
if self._initialized:
return
try:
from surrealdb import Surreal
except ImportError:
raise ImportError(
"surrealdb is required for SurrealDB Vector support. "
"Install with: pip install surrealdb"
)
import asyncio
async def connect():
client = Surreal(self.url)
await client.connect()
await client.signin({"user": self.username, "pass": self.password})
await client.use(self.namespace, self.database)
return client
loop = asyncio.get_event_loop()
self._client = loop.run_until_complete(connect())
self._initialized = True
def _run_async(self, coro):
"""Run async coroutine synchronously."""
import asyncio
return asyncio.get_event_loop().run_until_complete(coro)
def create_collection(
self,
name: str,
dimension: int = 1536,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create a collection (SurrealDB creates tables on insert)."""
self._init_client()
# SurrealDB creates tables automatically
logger.info(f"Collection '{name}' will be created on first insert")
def delete_collection(self, name: str) -> bool:
"""Delete a collection."""
self._init_client()
try:
self._run_async(self._client.query(f"REMOVE TABLE {name}"))
return True
except Exception as e:
logger.error(f"Failed to delete collection {name}: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if collection exists."""
self._init_client()
try:
result = self._run_async(self._client.query(f"INFO FOR TABLE {name}"))
return bool(result)
except Exception:
return False
def list_collections(self) -> List[str]:
"""List all collections."""
self._init_client()
try:
result = self._run_async(self._client.query("INFO FOR DB"))
if result and len(result) > 0:
tables = result[0].get("result", {}).get("tables", {})
return list(tables.keys())
return []
except Exception as e:
logger.error(f"Failed to list collections: {e}")
return []
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
self._init_client()
ids = []
for doc in documents:
record = {
"content": doc.content,
"embedding": doc.embedding,
"metadata": doc.metadata or {},
"content_hash": doc.content_hash,
"created_at": doc.created_at,
}
try:
result = self._run_async(
self._client.create(f"{collection}:{doc.id}", record)
)
ids.append(doc.id)
except Exception as e:
logger.warning(f"Failed to insert document {doc.id}: {e}")
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Upsert documents."""
self._init_client()
ids = []
for doc in documents:
record = {
"content": doc.content,
"embedding": doc.embedding,
"metadata": doc.metadata or {},
"content_hash": doc.content_hash,
"created_at": doc.created_at,
}
try:
self._run_async(
self._client.update(f"{collection}:{doc.id}", record)
)
ids.append(doc.id)
except Exception as e:
logger.warning(f"Failed to upsert document {doc.id}: {e}")
return ids
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents using vector search."""
self._init_client()
# SurrealDB vector search query
embedding_str = str(query_embedding)
query = f"""
SELECT *, vector::similarity::cosine(embedding, {embedding_str}) AS score
FROM {collection}
ORDER BY score DESC
LIMIT {limit}
"""
try:
result = self._run_async(self._client.query(query))
documents = []
if result and len(result) > 0:
for row in result[0].get("result", []):
score = row.get("score", 0)
if score_threshold and score < score_threshold:
continue
doc_id = str(row.get("id", "")).split(":")[-1]
documents.append(KnowledgeDocument(
id=doc_id,
content=row.get("content", ""),
embedding=row.get("embedding"),
metadata={**(row.get("metadata") or {}), "score": score},
content_hash=row.get("content_hash"),
created_at=row.get("created_at", 0)
))
return documents
except Exception as e:
logger.error(f"Vector search failed: {e}")
return []
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
self._init_client()
documents = []
for doc_id in ids:
try:
result = self._run_async(self._client.select(f"{collection}:{doc_id}"))
if result:
row = result if isinstance(result, dict) else result[0]
documents.append(KnowledgeDocument(
id=doc_id,
content=row.get("content", ""),
embedding=row.get("embedding"),
metadata=row.get("metadata"),
content_hash=row.get("content_hash"),
created_at=row.get("created_at", 0)
))
except Exception as e:
logger.warning(f"Failed to get document {doc_id}: {e}")
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
self._init_client()
count = 0
if ids:
for doc_id in ids:
try:
self._run_async(self._client.delete(f"{collection}:{doc_id}"))
count += 1
except Exception as e:
logger.warning(f"Failed to delete {doc_id}: {e}")
else:
try:
self._run_async(self._client.query(f"DELETE FROM {collection}"))
count = -1 # Unknown count
except Exception as e:
logger.error(f"Failed to delete all: {e}")
return count
def count(self, collection: str) -> int:
"""Count documents."""
self._init_client()
try:
result = self._run_async(
self._client.query(f"SELECT count() FROM {collection} GROUP ALL")
)
if result and len(result) > 0:
return result[0].get("result", [{}])[0].get("count", 0)
return 0
except Exception as e:
logger.error(f"Count failed: {e}")
return 0
def close(self) -> None:
"""Close the connection."""
if self._client:
self._run_async(self._client.close())
self._client = None
self._initialized = False
@@ -0,0 +1,266 @@
"""
Upstash Vector implementation of KnowledgeStore.
Requires: upstash-vector
Install: pip install upstash-vector
"""
import logging
import os
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class UpstashVectorKnowledgeStore(KnowledgeStore):
"""
Upstash Vector store for knowledge/RAG.
Uses Upstash's serverless vector database.
Example:
store = UpstashVectorKnowledgeStore(
url="https://xxx.upstash.io",
token="your-token"
)
"""
def __init__(
self,
url: Optional[str] = None,
token: Optional[str] = None,
embedding_dim: int = 1536,
):
"""
Initialize Upstash Vector store.
Args:
url: Upstash Vector REST URL (or UPSTASH_VECTOR_REST_URL env)
token: Upstash Vector REST token (or UPSTASH_VECTOR_REST_TOKEN env)
embedding_dim: Embedding dimension
"""
self.url = url or os.getenv("UPSTASH_VECTOR_REST_URL")
self.token = token or os.getenv("UPSTASH_VECTOR_REST_TOKEN")
self.embedding_dim = embedding_dim
self._index = None
self._initialized = False
def _init_client(self):
"""Initialize Upstash client lazily."""
if self._initialized:
return
if not self.url or not self.token:
raise ValueError(
"Upstash Vector requires url and token. "
"Set UPSTASH_VECTOR_REST_URL and UPSTASH_VECTOR_REST_TOKEN env vars."
)
try:
from upstash_vector import Index
except ImportError:
raise ImportError(
"upstash-vector is required for Upstash Vector support. "
"Install with: pip install upstash-vector"
)
self._index = Index(url=self.url, token=self.token)
self._initialized = True
def create_collection(
self,
name: str,
dimension: int = 1536,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create collection (Upstash uses namespaces)."""
self._init_client()
# Upstash Vector uses a single index with namespaces
logger.info(f"Using namespace '{name}' in Upstash Vector index")
def delete_collection(self, name: str) -> bool:
"""Delete all vectors in namespace."""
self._init_client()
try:
self._index.delete_namespace(name)
return True
except Exception as e:
logger.error(f"Failed to delete namespace {name}: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if namespace has vectors."""
self._init_client()
try:
info = self._index.info()
namespaces = info.namespaces if hasattr(info, 'namespaces') else {}
return name in namespaces
except Exception:
return False
def list_collections(self) -> List[str]:
"""List all namespaces."""
self._init_client()
try:
info = self._index.info()
namespaces = info.namespaces if hasattr(info, 'namespaces') else {}
return list(namespaces.keys())
except Exception as e:
logger.error(f"Failed to list namespaces: {e}")
return []
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
self._init_client()
vectors = []
for doc in documents:
if not doc.embedding:
logger.warning(f"Document {doc.id} has no embedding, skipping")
continue
vectors.append({
"id": doc.id,
"vector": doc.embedding,
"metadata": {
"content": doc.content,
**(doc.metadata or {}),
"content_hash": doc.content_hash,
"created_at": doc.created_at,
}
})
if vectors:
self._index.upsert(vectors, namespace=collection)
return [v["id"] for v in vectors]
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Upsert documents (same as insert for Upstash)."""
return self.insert(collection, documents)
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents."""
self._init_client()
try:
results = self._index.query(
vector=query_embedding,
top_k=limit,
include_metadata=True,
namespace=collection,
filter=filters
)
documents = []
for result in results:
score = result.score if hasattr(result, 'score') else 0
if score_threshold and score < score_threshold:
continue
metadata = result.metadata if hasattr(result, 'metadata') else {}
content = metadata.pop("content", "") if metadata else ""
documents.append(KnowledgeDocument(
id=result.id,
content=content,
metadata={**metadata, "score": score},
content_hash=metadata.get("content_hash") if metadata else None,
created_at=metadata.get("created_at", 0) if metadata else 0
))
return documents
except Exception as e:
logger.error(f"Search failed: {e}")
return []
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
self._init_client()
documents = []
try:
results = self._index.fetch(ids, include_metadata=True, namespace=collection)
for result in results:
if result:
metadata = result.metadata if hasattr(result, 'metadata') else {}
content = metadata.pop("content", "") if metadata else ""
documents.append(KnowledgeDocument(
id=result.id,
content=content,
embedding=result.vector if hasattr(result, 'vector') else None,
metadata=metadata,
content_hash=metadata.get("content_hash") if metadata else None,
created_at=metadata.get("created_at", 0) if metadata else 0
))
except Exception as e:
logger.error(f"Fetch failed: {e}")
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
self._init_client()
try:
if ids:
self._index.delete(ids, namespace=collection)
return len(ids)
elif filters:
# Upstash doesn't support filter-based delete directly
logger.warning("Filter-based delete not supported, use IDs")
return 0
else:
self._index.delete_namespace(collection)
return -1 # Unknown count
except Exception as e:
logger.error(f"Delete failed: {e}")
return 0
def count(self, collection: str) -> int:
"""Count documents in namespace."""
self._init_client()
try:
info = self._index.info()
namespaces = info.namespaces if hasattr(info, 'namespaces') else {}
ns_info = namespaces.get(collection, {})
return ns_info.get("vector_count", 0) if isinstance(ns_info, dict) else 0
except Exception as e:
logger.error(f"Count failed: {e}")
return 0
def close(self) -> None:
"""Close the client."""
self._index = None
self._initialized = False
@@ -0,0 +1,223 @@
"""
Weaviate implementation of KnowledgeStore.
Requires: weaviate-client
Install: pip install weaviate-client
"""
import logging
import os
from typing import Any, Dict, List, Optional
from .base import KnowledgeStore, KnowledgeDocument
logger = logging.getLogger(__name__)
class WeaviateKnowledgeStore(KnowledgeStore):
"""
Weaviate-based knowledge store for vector search.
Example:
store = WeaviateKnowledgeStore(
url="http://localhost:8080"
)
"""
def __init__(
self,
url: Optional[str] = None,
api_key: Optional[str] = None,
additional_headers: Optional[Dict[str, str]] = None,
):
try:
import weaviate
except ImportError:
raise ImportError(
"weaviate-client is required for Weaviate support. "
"Install with: pip install weaviate-client"
)
url = url or os.getenv("WEAVIATE_URL", "http://localhost:8080")
api_key = api_key or os.getenv("WEAVIATE_API_KEY")
auth = weaviate.auth.AuthApiKey(api_key) if api_key else None
self._client = weaviate.connect_to_custom(
http_host=url.replace("http://", "").replace("https://", "").split(":")[0],
http_port=int(url.split(":")[-1]) if ":" in url.split("/")[-1] else 8080,
http_secure=url.startswith("https"),
auth_credentials=auth,
additional_headers=additional_headers,
)
logger.info(f"Connected to Weaviate at {url}")
def create_collection(
self,
name: str,
dimension: int,
distance: str = "cosine",
metadata: Optional[Dict[str, Any]] = None
) -> None:
"""Create a new collection (class in Weaviate)."""
from weaviate.classes.config import Configure, Property, DataType
distance_map = {"cosine": "cosine", "euclidean": "l2-squared", "dot": "dot"}
self._client.collections.create(
name=name,
vectorizer_config=Configure.Vectorizer.none(),
vector_index_config=Configure.VectorIndex.hnsw(
distance_metric=distance_map.get(distance, "cosine")
),
properties=[
Property(name="content", data_type=DataType.TEXT),
Property(name="content_hash", data_type=DataType.TEXT),
Property(name="created_at", data_type=DataType.NUMBER),
]
)
logger.info(f"Created Weaviate collection: {name}")
def delete_collection(self, name: str) -> bool:
"""Delete a collection."""
try:
self._client.collections.delete(name)
return True
except Exception as e:
logger.warning(f"Failed to delete collection {name}: {e}")
return False
def collection_exists(self, name: str) -> bool:
"""Check if a collection exists."""
return self._client.collections.exists(name)
def list_collections(self) -> List[str]:
"""List all collections."""
return [c.name for c in self._client.collections.list_all().values()]
def insert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert documents."""
col = self._client.collections.get(collection)
ids = []
with col.batch.dynamic() as batch:
for doc in documents:
if doc.embedding is None:
raise ValueError(f"Document {doc.id} has no embedding")
properties = {
"content": doc.content,
"content_hash": doc.content_hash or "",
"created_at": doc.created_at,
**(doc.metadata or {})
}
batch.add_object(
properties=properties,
vector=doc.embedding,
uuid=doc.id,
)
ids.append(doc.id)
return ids
def upsert(
self,
collection: str,
documents: List[KnowledgeDocument]
) -> List[str]:
"""Insert or update documents."""
return self.insert(collection, documents)
def search(
self,
collection: str,
query_embedding: List[float],
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
score_threshold: Optional[float] = None
) -> List[KnowledgeDocument]:
"""Search for similar documents."""
col = self._client.collections.get(collection)
query = col.query.near_vector(
near_vector=query_embedding,
limit=limit,
return_metadata=["distance"],
)
documents = []
for obj in query.objects:
if score_threshold:
score = 1 - (obj.metadata.distance or 0)
if score < score_threshold:
continue
props = obj.properties
doc = KnowledgeDocument(
id=str(obj.uuid),
content=props.get("content", ""),
embedding=None,
metadata={k: v for k, v in props.items() if k not in ("content", "content_hash", "created_at")},
content_hash=props.get("content_hash"),
created_at=props.get("created_at", 0),
)
documents.append(doc)
return documents
def get(
self,
collection: str,
ids: List[str]
) -> List[KnowledgeDocument]:
"""Get documents by IDs."""
col = self._client.collections.get(collection)
documents = []
for doc_id in ids:
try:
obj = col.query.fetch_object_by_id(doc_id, include_vector=True)
if obj:
props = obj.properties
doc = KnowledgeDocument(
id=str(obj.uuid),
content=props.get("content", ""),
embedding=list(obj.vector) if obj.vector else None,
metadata={k: v for k, v in props.items() if k not in ("content", "content_hash", "created_at")},
content_hash=props.get("content_hash"),
created_at=props.get("created_at", 0),
)
documents.append(doc)
except Exception:
pass
return documents
def delete(
self,
collection: str,
ids: Optional[List[str]] = None,
filters: Optional[Dict[str, Any]] = None
) -> int:
"""Delete documents."""
col = self._client.collections.get(collection)
if ids:
for doc_id in ids:
col.data.delete_by_id(doc_id)
return len(ids)
return 0
def count(self, collection: str) -> int:
"""Count documents."""
col = self._client.collections.get(collection)
return col.aggregate.over_all(total_count=True).total_count or 0
def close(self) -> None:
"""Close the store."""
if self._client:
self._client.close()
@@ -0,0 +1,10 @@
"""
Schema versioning and migrations for PraisonAI persistence.
This module provides tools for managing database schema versions
and applying migrations when upgrading.
"""
from .manager import MigrationManager, SchemaVersion
__all__ = ["MigrationManager", "SchemaVersion"]
@@ -0,0 +1,251 @@
"""
Migration manager for PraisonAI persistence schema versioning.
"""
import logging
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, List, Optional, Tuple
from packaging import version as pkg_version
logger = logging.getLogger(__name__)
@dataclass
class SchemaVersion:
"""Represents a schema version."""
version: str
description: str = ""
applied_at: Optional[float] = None
def __lt__(self, other: "SchemaVersion") -> bool:
return pkg_version.parse(self.version) < pkg_version.parse(other.version)
def __eq__(self, other: object) -> bool:
if not isinstance(other, SchemaVersion):
return False
return self.version == other.version
@dataclass
class Migration:
"""Represents a single migration."""
version: str
description: str
up: Callable[[Any], None]
down: Optional[Callable[[Any], None]] = None
class MigrationManager:
"""
Manages database schema versions and migrations.
Example:
manager = MigrationManager(db_adapter)
manager.register_migration(
version="1.0.0",
description="Initial schema",
up=lambda db: db.execute("CREATE TABLE ..."),
down=lambda db: db.execute("DROP TABLE ...")
)
manager.migrate_up() # Apply all pending migrations
"""
# Current schema version
CURRENT_VERSION = "1.0.0"
def __init__(self, db_adapter: Any = None):
"""
Initialize migration manager.
Args:
db_adapter: Database adapter (PraisonDB or similar)
"""
self._db = db_adapter
self._migrations: List[Migration] = []
self._versions_table = "praison_schema_versions"
# Register built-in migrations
self._register_builtin_migrations()
def _register_builtin_migrations(self):
"""Register built-in migrations."""
self.register_migration(
version="1.0.0",
description="Initial schema with sessions, messages, runs",
up=self._migrate_1_0_0_up,
down=self._migrate_1_0_0_down
)
def register_migration(
self,
version: str,
description: str,
up: Callable[[Any], None],
down: Optional[Callable[[Any], None]] = None
):
"""
Register a new migration.
Args:
version: Version string (e.g., "1.0.0")
description: Human-readable description
up: Function to apply migration
down: Optional function to rollback migration
"""
migration = Migration(
version=version,
description=description,
up=up,
down=down
)
self._migrations.append(migration)
self._migrations.sort(key=lambda m: pkg_version.parse(m.version))
def get_current_version(self) -> Optional[str]:
"""Get the current schema version from database."""
if not self._db:
return None
try:
# Try to get version from state store
if hasattr(self._db, '_state_store') and self._db._state_store:
self._db._init_stores()
version_data = self._db._state_store.get(f"{self._versions_table}:current")
if version_data:
return version_data.get("version")
except Exception as e:
logger.warning(f"Failed to get schema version: {e}")
return None
def set_current_version(self, version: str):
"""Set the current schema version in database."""
if not self._db:
return
try:
if hasattr(self._db, '_state_store') and self._db._state_store:
self._db._init_stores()
import time
self._db._state_store.set(f"{self._versions_table}:current", {
"version": version,
"applied_at": time.time()
})
except Exception as e:
logger.warning(f"Failed to set schema version: {e}")
def get_pending_migrations(self) -> List[Migration]:
"""Get list of migrations that haven't been applied."""
current = self.get_current_version()
if not current:
return self._migrations.copy()
current_ver = pkg_version.parse(current)
return [
m for m in self._migrations
if pkg_version.parse(m.version) > current_ver
]
def migrate_up(self, target_version: Optional[str] = None) -> List[str]:
"""
Apply pending migrations up to target version.
Args:
target_version: Optional target version (default: latest)
Returns:
List of applied migration versions
"""
pending = self.get_pending_migrations()
if not pending:
logger.info("No pending migrations")
return []
target = pkg_version.parse(target_version) if target_version else None
applied = []
for migration in pending:
if target and pkg_version.parse(migration.version) > target:
break
logger.info(f"Applying migration {migration.version}: {migration.description}")
try:
migration.up(self._db)
self.set_current_version(migration.version)
applied.append(migration.version)
logger.info(f"Successfully applied migration {migration.version}")
except Exception as e:
logger.error(f"Failed to apply migration {migration.version}: {e}")
raise
return applied
def migrate_down(self, target_version: str) -> List[str]:
"""
Rollback migrations down to target version.
Args:
target_version: Target version to rollback to
Returns:
List of rolled back migration versions
"""
current = self.get_current_version()
if not current:
logger.info("No migrations to rollback")
return []
current_ver = pkg_version.parse(current)
target_ver = pkg_version.parse(target_version)
if target_ver >= current_ver:
logger.info("Target version is not lower than current")
return []
# Get migrations to rollback (in reverse order)
to_rollback = [
m for m in reversed(self._migrations)
if pkg_version.parse(m.version) > target_ver and pkg_version.parse(m.version) <= current_ver
]
rolled_back = []
for migration in to_rollback:
if not migration.down:
logger.warning(f"Migration {migration.version} has no down function, skipping")
continue
logger.info(f"Rolling back migration {migration.version}")
try:
migration.down(self._db)
rolled_back.append(migration.version)
logger.info(f"Successfully rolled back migration {migration.version}")
except Exception as e:
logger.error(f"Failed to rollback migration {migration.version}: {e}")
raise
self.set_current_version(target_version)
return rolled_back
def get_migration_status(self) -> Dict[str, Any]:
"""Get current migration status."""
current = self.get_current_version()
pending = self.get_pending_migrations()
return {
"current_version": current,
"latest_version": self._migrations[-1].version if self._migrations else None,
"pending_count": len(pending),
"pending_versions": [m.version for m in pending],
"all_versions": [m.version for m in self._migrations]
}
# --- Built-in Migrations ---
def _migrate_1_0_0_up(self, db: Any):
"""Initial schema migration."""
# This is a no-op since tables are created on first use
logger.info("Initial schema (1.0.0) - tables created on first use")
def _migrate_1_0_0_down(self, db: Any):
"""Rollback initial schema."""
logger.info("Rolling back initial schema (1.0.0)")
@@ -0,0 +1,406 @@
"""
PersistenceOrchestrator - Central coordinator for all persistence operations.
Hooks into agent lifecycle to provide automatic conversation persistence,
knowledge retrieval, and state management.
"""
import logging
import time
import uuid
from typing import Any, Dict, List, Optional, TYPE_CHECKING
from .conversation.base import ConversationStore, ConversationSession, ConversationMessage
from .knowledge.base import KnowledgeStore, KnowledgeDocument
from .state.base import StateStore
from .config import PersistenceConfig
from .factory import create_stores_from_config
if TYPE_CHECKING:
pass
logger = logging.getLogger(__name__)
class PersistenceOrchestrator:
"""
Central coordinator for all persistence operations.
Lives in wrapper layer, hooks into agent lifecycle to provide:
- Automatic session loading/creation at agent start
- Message persistence during conversation
- Session metadata updates at agent end
- Knowledge retrieval for RAG
- State caching
Example:
orchestrator = PersistenceOrchestrator(
conversation_store=PostgresConversationStore(...),
knowledge_store=QdrantKnowledgeStore(...),
state_store=RedisStateStore(...),
)
# Hook into agent lifecycle
history = orchestrator.on_agent_start(agent, session_id="sess-123")
# ... agent runs ...
orchestrator.on_message(session_id, user_message)
orchestrator.on_message(session_id, assistant_message)
orchestrator.on_agent_end(agent, session_id)
"""
def __init__(
self,
conversation_store: Optional[ConversationStore] = None,
knowledge_store: Optional[KnowledgeStore] = None,
state_store: Optional[StateStore] = None,
config: Optional[PersistenceConfig] = None,
):
"""
Initialize the orchestrator.
Args:
conversation_store: Store for session/message persistence
knowledge_store: Store for vector embeddings
state_store: Store for key-value state
config: Configuration (alternative to passing stores directly)
"""
if config:
stores = create_stores_from_config(config)
self.conversation = stores["conversation"] or conversation_store
self.knowledge = stores["knowledge"] or knowledge_store
self.state = stores["state"] or state_store
self._config = config
else:
self.conversation = conversation_store
self.knowledge = knowledge_store
self.state = state_store
self._config = None
self._current_session: Optional[ConversationSession] = None
self._session_cache: Dict[str, ConversationSession] = {}
@classmethod
def from_config(cls, config: PersistenceConfig) -> "PersistenceOrchestrator":
"""Create orchestrator from configuration."""
return cls(config=config)
@classmethod
def from_env(cls) -> "PersistenceOrchestrator":
"""Create orchestrator from environment variables."""
config = PersistenceConfig.from_env()
return cls(config=config)
# =========================================================================
# Agent Lifecycle Hooks
# =========================================================================
def on_agent_start(
self,
agent: Any,
session_id: Optional[str] = None,
user_id: Optional[str] = None,
resume: bool = True,
) -> List[ConversationMessage]:
"""
Called before agent run. Loads or creates session.
Args:
agent: The agent instance
session_id: Session ID (generated if not provided)
user_id: User ID for session
resume: Whether to load existing session history
Returns:
List of previous messages if resuming, empty list otherwise
"""
if not session_id:
session_id = str(uuid.uuid4())
if not self.conversation:
logger.debug("No conversation store configured, skipping session load")
return []
# Try to load existing session
session = None
if resume:
session = self.conversation.get_session(session_id)
if session:
logger.info(f"Resuming session: {session_id}")
self._current_session = session
self._session_cache[session_id] = session
# Load previous messages
messages = self.conversation.get_messages(session_id)
return messages
else:
# Create new session
agent_id = getattr(agent, "name", None) or getattr(agent, "agent_id", None)
session = ConversationSession(
session_id=session_id,
user_id=user_id,
agent_id=agent_id,
name=f"Session {session_id[:8]}",
metadata={"agent_type": type(agent).__name__},
)
self.conversation.create_session(session)
logger.info(f"Created new session: {session_id}")
self._current_session = session
self._session_cache[session_id] = session
return []
def on_message(
self,
session_id: str,
role: str,
content: str,
tool_calls: Optional[List[Dict]] = None,
tool_call_id: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None,
) -> Optional[ConversationMessage]:
"""
Called after each message (user or assistant).
Args:
session_id: Session ID
role: Message role (user, assistant, system, tool)
content: Message content
tool_calls: Tool calls (for assistant messages)
tool_call_id: Tool call ID (for tool response messages)
metadata: Additional metadata
Returns:
The persisted message, or None if no store configured
"""
if not self.conversation:
return None
message = ConversationMessage(
id=str(uuid.uuid4()),
session_id=session_id,
role=role,
content=content,
tool_calls=tool_calls,
tool_call_id=tool_call_id,
metadata=metadata,
)
self.conversation.add_message(session_id, message)
logger.debug(f"Persisted {role} message to session {session_id}")
return message
def on_agent_end(
self,
agent: Any,
session_id: str,
metadata: Optional[Dict[str, Any]] = None,
) -> None:
"""
Called after agent run. Updates session metadata.
Args:
agent: The agent instance
session_id: Session ID
metadata: Additional metadata to store
"""
if not self.conversation:
return
session = self._session_cache.get(session_id) or self.conversation.get_session(session_id)
if session:
session.updated_at = time.time()
if metadata:
session.metadata = {**(session.metadata or {}), **metadata}
self.conversation.update_session(session)
logger.debug(f"Updated session metadata: {session_id}")
# =========================================================================
# Knowledge Retrieval
# =========================================================================
def retrieve_knowledge(
self,
query_embedding: List[float],
collection: str = "default",
limit: int = 5,
filters: Optional[Dict[str, Any]] = None,
) -> List[KnowledgeDocument]:
"""
Retrieve relevant documents for RAG.
Args:
query_embedding: Query vector embedding
collection: Collection name
limit: Max documents to return
filters: Metadata filters
Returns:
List of relevant documents
"""
if not self.knowledge:
logger.debug("No knowledge store configured")
return []
return self.knowledge.search(
collection=collection,
query_embedding=query_embedding,
limit=limit,
filters=filters,
)
def add_knowledge(
self,
documents: List[KnowledgeDocument],
collection: str = "default",
) -> List[str]:
"""
Add documents to knowledge store.
Args:
documents: Documents with embeddings
collection: Collection name
Returns:
List of document IDs
"""
if not self.knowledge:
raise ValueError("No knowledge store configured")
return self.knowledge.upsert(collection, documents)
# =========================================================================
# State Management
# =========================================================================
def get_state(self, key: str) -> Optional[Any]:
"""Get state value."""
if not self.state:
return None
return self.state.get(key)
def set_state(self, key: str, value: Any, ttl: Optional[int] = None) -> None:
"""Set state value with optional TTL."""
if not self.state:
return
self.state.set(key, value, ttl)
def delete_state(self, key: str) -> bool:
"""Delete state value."""
if not self.state:
return False
return self.state.delete(key)
# =========================================================================
# Session Management
# =========================================================================
def get_session(self, session_id: str) -> Optional[ConversationSession]:
"""Get a session by ID."""
if not self.conversation:
return None
return self.conversation.get_session(session_id)
def list_sessions(
self,
user_id: Optional[str] = None,
limit: int = 100,
) -> List[ConversationSession]:
"""List sessions for a user."""
if not self.conversation:
return []
return self.conversation.list_sessions(user_id=user_id, limit=limit)
def delete_session(self, session_id: str) -> bool:
"""Delete a session and all its messages."""
if not self.conversation:
return False
if session_id in self._session_cache:
del self._session_cache[session_id]
return self.conversation.delete_session(session_id)
def get_messages(
self,
session_id: str,
limit: Optional[int] = None,
) -> List[ConversationMessage]:
"""Get messages from a session."""
if not self.conversation:
return []
return self.conversation.get_messages(session_id, limit=limit)
# =========================================================================
# Context Building
# =========================================================================
def build_context(
self,
session_id: str,
query_embedding: Optional[List[float]] = None,
history_limit: int = 20,
knowledge_limit: int = 5,
knowledge_collection: str = "default",
) -> Dict[str, Any]:
"""
Build context for agent including history and knowledge.
Args:
session_id: Session ID
query_embedding: Query embedding for knowledge retrieval
history_limit: Max history messages
knowledge_limit: Max knowledge documents
knowledge_collection: Knowledge collection name
Returns:
Dict with 'history' and 'knowledge' keys
"""
context = {
"history": [],
"knowledge": [],
}
# Get conversation history
if self.conversation:
messages = self.conversation.get_messages(session_id, limit=history_limit)
context["history"] = [
{"role": m.role, "content": m.content}
for m in messages
]
# Get relevant knowledge
if self.knowledge and query_embedding:
docs = self.knowledge.search(
collection=knowledge_collection,
query_embedding=query_embedding,
limit=knowledge_limit,
)
context["knowledge"] = [
{"content": d.content, "metadata": d.metadata}
for d in docs
]
return context
# =========================================================================
# Cleanup
# =========================================================================
def close(self) -> None:
"""Close all stores and release resources."""
if self.conversation:
self.conversation.close()
if self.knowledge:
self.knowledge.close()
if self.state:
self.state.close()
self._session_cache.clear()
logger.info("Persistence orchestrator closed")
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
return False
@@ -0,0 +1,21 @@
"""
StateStore implementations for fast key-value state and caching.
Supported backends:
- Redis (redis)
- DynamoDB (dynamodb)
- Firestore (firestore)
- MongoDB (mongodb)
- Upstash (upstash)
- In-memory / JSON file (memory)
"""
__all__ = [
"StateStore",
]
def __getattr__(name: str):
if name == "StateStore":
from .base import StateStore
return StateStore
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -0,0 +1,200 @@
"""
Async MongoDB implementation of StateStore.
Requires: motor
Install: pip install motor
"""
import asyncio
import logging
import time
from typing import Any, Dict, List, Optional
from .base import StateStore
logger = logging.getLogger(__name__)
class AsyncMongoDBStateStore(StateStore):
"""
Async MongoDB state store using motor.
Provides high-performance async database operations.
Example:
store = AsyncMongoDBStateStore(
url="mongodb://localhost:27017",
database="praisonai"
)
await store.init()
"""
def __init__(
self,
url: str = "mongodb://localhost:27017",
database: str = "praisonai",
collection: str = "state",
):
"""
Initialize async MongoDB store.
Args:
url: MongoDB connection URL
database: Database name
collection: Collection name
"""
self.url = url
self.database_name = database
self.collection_name = collection
self._client = None
self._db = None
self._collection = None
self._initialized = False
async def init(self):
"""Initialize connection."""
if self._initialized:
return
try:
from motor.motor_asyncio import AsyncIOMotorClient
except ImportError:
raise ImportError(
"motor is required for async MongoDB support. "
"Install with: pip install motor"
)
self._client = AsyncIOMotorClient(self.url)
self._db = self._client[self.database_name]
self._collection = self._db[self.collection_name]
# Create TTL index for expiration
await self._collection.create_index("expires_at", expireAfterSeconds=0)
self._initialized = True
async def async_get(self, key: str) -> Optional[Dict[str, Any]]:
"""Get state by key asynchronously."""
if not self._initialized:
await self.init()
doc = await self._collection.find_one({"_id": key})
if doc:
# Check TTL manually (in case index hasn't cleaned up yet)
if "expires_at" in doc and doc["expires_at"]:
if time.time() > doc["expires_at"]:
await self.async_delete(key)
return None
# Remove internal fields
result = dict(doc)
result.pop("_id", None)
result.pop("expires_at", None)
return result.get("value", result)
return None
def get(self, key: str) -> Optional[Dict[str, Any]]:
"""Sync wrapper for get."""
return asyncio.get_event_loop().run_until_complete(self.async_get(key))
async def async_set(self, key: str, value: Dict[str, Any], ttl: Optional[int] = None) -> bool:
"""Set state by key with optional TTL asynchronously."""
if not self._initialized:
await self.init()
doc = {
"_id": key,
"value": value,
"updated_at": time.time(),
}
if ttl:
doc["expires_at"] = time.time() + ttl
try:
await self._collection.replace_one(
{"_id": key},
doc,
upsert=True
)
return True
except Exception as e:
logger.error(f"Error setting state {key}: {e}")
return False
def set(self, key: str, value: Dict[str, Any], ttl: Optional[int] = None) -> bool:
"""Sync wrapper for set."""
return asyncio.get_event_loop().run_until_complete(self.async_set(key, value, ttl))
async def async_delete(self, key: str) -> bool:
"""Delete state by key asynchronously."""
if not self._initialized:
await self.init()
result = await self._collection.delete_one({"_id": key})
return result.deleted_count > 0
def delete(self, key: str) -> bool:
"""Sync wrapper for delete."""
return asyncio.get_event_loop().run_until_complete(self.async_delete(key))
async def async_exists(self, key: str) -> bool:
"""Check if key exists asynchronously."""
if not self._initialized:
await self.init()
count = await self._collection.count_documents({"_id": key}, limit=1)
return count > 0
def exists(self, key: str) -> bool:
"""Sync wrapper for exists."""
return asyncio.get_event_loop().run_until_complete(self.async_exists(key))
async def async_list_keys(self, prefix: Optional[str] = None) -> List[str]:
"""List all keys with optional prefix filter asynchronously."""
if not self._initialized:
await self.init()
query = {}
if prefix:
query["_id"] = {"$regex": f"^{prefix}"}
cursor = self._collection.find(query, {"_id": 1})
keys = []
async for doc in cursor:
keys.append(doc["_id"])
return keys
def list_keys(self, prefix: Optional[str] = None) -> List[str]:
"""Sync wrapper for list_keys."""
return asyncio.get_event_loop().run_until_complete(self.async_list_keys(prefix))
async def async_clear(self, prefix: Optional[str] = None) -> int:
"""Clear all keys with optional prefix filter asynchronously."""
if not self._initialized:
await self.init()
query = {}
if prefix:
query["_id"] = {"$regex": f"^{prefix}"}
result = await self._collection.delete_many(query)
return result.deleted_count
def clear(self, prefix: Optional[str] = None) -> int:
"""Sync wrapper for clear."""
return asyncio.get_event_loop().run_until_complete(self.async_clear(prefix))
async def async_close(self) -> None:
"""Close the connection."""
if self._client:
self._client.close()
self._client = None
self._db = None
self._collection = None
self._initialized = False
def close(self) -> None:
"""Sync wrapper for close."""
if self._client:
asyncio.get_event_loop().run_until_complete(self.async_close())
@@ -0,0 +1,107 @@
"""
Base interfaces for StateStore.
StateStore handles fast key-value state and caching.
"""
from abc import ABC, abstractmethod
from typing import Any, Dict, List, Optional
import json
class StateStore(ABC):
"""
Abstract base class for key-value state persistence.
Implementations handle fast state/cache storage for different backends:
- Redis (dedicated KV store)
- DynamoDB, Firestore (serverless document stores)
- MongoDB (document store)
- Upstash (serverless Redis)
- In-memory / JSON file (zero-dependency fallback)
"""
@abstractmethod
def get(self, key: str) -> Optional[Any]:
"""Get a value by key. Returns None if not found."""
raise NotImplementedError
@abstractmethod
def set(
self,
key: str,
value: Any,
ttl: Optional[int] = None
) -> None:
"""Set a value. TTL is in seconds."""
raise NotImplementedError
@abstractmethod
def delete(self, key: str) -> bool:
"""Delete a key. Returns True if key existed."""
raise NotImplementedError
@abstractmethod
def exists(self, key: str) -> bool:
"""Check if a key exists."""
raise NotImplementedError
@abstractmethod
def keys(self, pattern: str = "*") -> List[str]:
"""List keys matching pattern."""
raise NotImplementedError
@abstractmethod
def ttl(self, key: str) -> Optional[int]:
"""Get remaining TTL in seconds. Returns None if no TTL or key doesn't exist."""
raise NotImplementedError
@abstractmethod
def expire(self, key: str, ttl: int) -> bool:
"""Set TTL on existing key. Returns True if key exists."""
raise NotImplementedError
def get_json(self, key: str) -> Optional[Any]:
"""Get and deserialize JSON value."""
value = self.get(key)
if value is None:
return None
if isinstance(value, str):
return json.loads(value)
return value
def set_json(self, key: str, value: Any, ttl: Optional[int] = None) -> None:
"""Serialize and set JSON value."""
self.set(key, json.dumps(value), ttl)
@abstractmethod
def hget(self, key: str, field: str) -> Optional[Any]:
"""Get a field from a hash."""
raise NotImplementedError
@abstractmethod
def hset(self, key: str, field: str, value: Any) -> None:
"""Set a field in a hash."""
raise NotImplementedError
@abstractmethod
def hgetall(self, key: str) -> Dict[str, Any]:
"""Get all fields from a hash."""
raise NotImplementedError
@abstractmethod
def hdel(self, key: str, *fields: str) -> int:
"""Delete fields from a hash. Returns count deleted."""
raise NotImplementedError
@abstractmethod
def close(self) -> None:
"""Close the store and release resources."""
raise NotImplementedError
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
return False
@@ -0,0 +1,226 @@
"""
DynamoDB implementation of StateStore.
Requires: boto3
Install: pip install boto3
"""
import json
import logging
import os
import time
from typing import Any, Dict, List, Optional
from .base import StateStore
logger = logging.getLogger(__name__)
class DynamoDBStateStore(StateStore):
"""
DynamoDB-based state store.
Example:
store = DynamoDBStateStore(
table_name="praisonai_state",
region="us-east-1"
)
"""
def __init__(
self,
table_name: str = "praisonai_state",
region: Optional[str] = None,
endpoint_url: Optional[str] = None,
auto_create_table: bool = True,
):
try:
import boto3
except ImportError:
raise ImportError(
"boto3 is required for DynamoDB support. "
"Install with: pip install boto3"
)
region = region or os.getenv("AWS_REGION", "us-east-1")
self._dynamodb = boto3.resource(
"dynamodb",
region_name=region,
endpoint_url=endpoint_url,
)
self._client = boto3.client(
"dynamodb",
region_name=region,
endpoint_url=endpoint_url,
)
self.table_name = table_name
if auto_create_table:
self._create_table()
self._table = self._dynamodb.Table(table_name)
logger.info(f"Connected to DynamoDB table: {table_name}")
def _create_table(self) -> None:
"""Create table if not exists."""
try:
self._client.describe_table(TableName=self.table_name)
except self._client.exceptions.ResourceNotFoundException:
self._client.create_table(
TableName=self.table_name,
KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}],
AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}],
BillingMode="PAY_PER_REQUEST",
)
waiter = self._client.get_waiter("table_exists")
waiter.wait(TableName=self.table_name)
# Enable TTL
self._client.update_time_to_live(
TableName=self.table_name,
TimeToLiveSpecification={"Enabled": True, "AttributeName": "ttl"}
)
logger.info(f"Created DynamoDB table: {self.table_name}")
def get(self, key: str) -> Optional[Any]:
"""Get a value by key."""
response = self._table.get_item(Key={"pk": key})
item = response.get("Item")
if not item:
return None
# Check TTL
if item.get("ttl") and item["ttl"] <= int(time.time()):
return None
value = item.get("value")
if isinstance(value, str):
try:
return json.loads(value)
except json.JSONDecodeError:
return value
return value
def set(
self,
key: str,
value: Any,
ttl: Optional[int] = None
) -> None:
"""Set a value with optional TTL."""
item = {
"pk": key,
"value": json.dumps(value) if not isinstance(value, str) else value,
"updated_at": int(time.time()),
}
if ttl:
item["ttl"] = int(time.time()) + ttl
self._table.put_item(Item=item)
def delete(self, key: str) -> bool:
"""Delete a key."""
response = self._table.delete_item(
Key={"pk": key},
ReturnValues="ALL_OLD"
)
return "Attributes" in response
def exists(self, key: str) -> bool:
"""Check if a key exists."""
response = self._table.get_item(
Key={"pk": key},
ProjectionExpression="pk,#t",
ExpressionAttributeNames={"#t": "ttl"}
)
item = response.get("Item")
if not item:
return False
if item.get("ttl") and item["ttl"] <= int(time.time()):
return False
return True
def keys(self, pattern: str = "*") -> List[str]:
"""List keys matching pattern."""
# DynamoDB scan is expensive, use with caution
response = self._table.scan(ProjectionExpression="pk")
keys = [item["pk"] for item in response.get("Items", [])]
if pattern != "*":
import fnmatch
keys = [k for k in keys if fnmatch.fnmatch(k, pattern)]
return keys
def ttl(self, key: str) -> Optional[int]:
"""Get remaining TTL in seconds."""
response = self._table.get_item(
Key={"pk": key},
ProjectionExpression="#t",
ExpressionAttributeNames={"#t": "ttl"}
)
item = response.get("Item")
if not item or "ttl" not in item:
return None
remaining = item["ttl"] - int(time.time())
if remaining <= 0:
return None
return remaining
def expire(self, key: str, ttl: int) -> bool:
"""Set TTL on existing key."""
try:
self._table.update_item(
Key={"pk": key},
UpdateExpression="SET #t = :ttl",
ExpressionAttributeNames={"#t": "ttl"},
ExpressionAttributeValues={":ttl": int(time.time()) + ttl},
ConditionExpression="attribute_exists(pk)"
)
return True
except self._client.exceptions.ConditionalCheckFailedException:
return False
def hget(self, key: str, field: str) -> Optional[Any]:
"""Get a field from a hash."""
value = self.get(key)
if not isinstance(value, dict):
return None
return value.get(field)
def hset(self, key: str, field: str, value: Any) -> None:
"""Set a field in a hash."""
current = self.get(key)
if not isinstance(current, dict):
current = {}
current[field] = value
self.set(key, current)
def hgetall(self, key: str) -> Dict[str, Any]:
"""Get all fields from a hash."""
value = self.get(key)
if not isinstance(value, dict):
return {}
return value
def hdel(self, key: str, *fields: str) -> int:
"""Delete fields from a hash."""
current = self.get(key)
if not isinstance(current, dict):
return 0
count = 0
for field in fields:
if field in current:
del current[field]
count += 1
if count > 0:
self.set(key, current)
return count
def close(self) -> None:
"""Close the store."""
pass # boto3 handles cleanup
@@ -0,0 +1,175 @@
"""
Firestore implementation of StateStore.
Requires: google-cloud-firestore
Install: pip install google-cloud-firestore
"""
import json
import logging
import os
import time
from typing import Any, Dict, List, Optional
from .base import StateStore
logger = logging.getLogger(__name__)
class FirestoreStateStore(StateStore):
"""
Firestore-based state store.
Example:
store = FirestoreStateStore(
project="my-project",
collection="praisonai_state"
)
"""
def __init__(
self,
project: Optional[str] = None,
collection: str = "praisonai_state",
credentials_path: Optional[str] = None,
):
try:
from google.cloud import firestore
except ImportError:
raise ImportError(
"google-cloud-firestore is required for Firestore support. "
"Install with: pip install google-cloud-firestore"
)
if credentials_path:
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = credentials_path
project = project or os.getenv("FIRESTORE_PROJECT")
self._client = firestore.Client(project=project)
self._collection = self._client.collection(collection)
logger.info(f"Connected to Firestore collection: {collection}")
def get(self, key: str) -> Optional[Any]:
"""Get a value by key."""
doc = self._collection.document(key).get()
if not doc.exists:
return None
data = doc.to_dict()
# Check TTL
if data.get("expires_at") and data["expires_at"] <= time.time():
self._collection.document(key).delete()
return None
return data.get("value")
def set(
self,
key: str,
value: Any,
ttl: Optional[int] = None
) -> None:
"""Set a value with optional TTL."""
data = {"value": value, "updated_at": time.time()}
if ttl:
data["expires_at"] = time.time() + ttl
self._collection.document(key).set(data)
def delete(self, key: str) -> bool:
"""Delete a key."""
doc = self._collection.document(key)
if doc.get().exists:
doc.delete()
return True
return False
def exists(self, key: str) -> bool:
"""Check if a key exists."""
doc = self._collection.document(key).get()
if not doc.exists:
return False
data = doc.to_dict()
if data.get("expires_at") and data["expires_at"] <= time.time():
return False
return True
def keys(self, pattern: str = "*") -> List[str]:
"""List keys matching pattern."""
docs = self._collection.stream()
keys = [doc.id for doc in docs]
if pattern != "*":
import fnmatch
keys = [k for k in keys if fnmatch.fnmatch(k, pattern)]
return keys
def ttl(self, key: str) -> Optional[int]:
"""Get remaining TTL in seconds."""
doc = self._collection.document(key).get()
if not doc.exists:
return None
data = doc.to_dict()
if "expires_at" not in data:
return None
remaining = data["expires_at"] - time.time()
if remaining <= 0:
return None
return int(remaining)
def expire(self, key: str, ttl: int) -> bool:
"""Set TTL on existing key."""
doc = self._collection.document(key)
if not doc.get().exists:
return False
doc.update({"expires_at": time.time() + ttl})
return True
def hget(self, key: str, field: str) -> Optional[Any]:
"""Get a field from a hash."""
value = self.get(key)
if not isinstance(value, dict):
return None
return value.get(field)
def hset(self, key: str, field: str, value: Any) -> None:
"""Set a field in a hash."""
current = self.get(key)
if not isinstance(current, dict):
current = {}
current[field] = value
self.set(key, current)
def hgetall(self, key: str) -> Dict[str, Any]:
"""Get all fields from a hash."""
value = self.get(key)
if not isinstance(value, dict):
return {}
return value
def hdel(self, key: str, *fields: str) -> int:
"""Delete fields from a hash."""
current = self.get(key)
if not isinstance(current, dict):
return 0
count = 0
for field in fields:
if field in current:
del current[field]
count += 1
if count > 0:
self.set(key, current)
return count
def close(self) -> None:
"""Close the store."""
pass # Firestore client handles cleanup
@@ -0,0 +1,155 @@
"""
Google Cloud Storage implementation of StateStore.
Requires: google-cloud-storage
Install: pip install google-cloud-storage
"""
import json
import logging
import time
from typing import Any, Dict, List, Optional
from .base import StateStore
logger = logging.getLogger(__name__)
class GCSStateStore(StateStore):
"""
Google Cloud Storage-based state store.
Stores state as JSON objects in GCS buckets.
Example:
store = GCSStateStore(
bucket_name="my-praisonai-state",
prefix="state/"
)
"""
def __init__(
self,
bucket_name: str,
prefix: str = "praisonai_state/",
project: Optional[str] = None,
credentials_path: Optional[str] = None,
):
"""
Initialize GCS state store.
Args:
bucket_name: GCS bucket name
prefix: Object prefix for state keys
project: GCP project ID (optional)
credentials_path: Path to service account JSON (optional)
"""
try:
from google.cloud import storage
if credentials_path:
self._client = storage.Client.from_service_account_json(
credentials_path, project=project
)
else:
self._client = storage.Client(project=project)
except ImportError:
raise ImportError(
"google-cloud-storage is required for GCS support. "
"Install with: pip install google-cloud-storage"
)
self.bucket_name = bucket_name
self.prefix = prefix
self._bucket = self._client.bucket(bucket_name)
def _key_to_path(self, key: str) -> str:
"""Convert key to GCS object path."""
return f"{self.prefix}{key}.json"
def get(self, key: str) -> Optional[Dict[str, Any]]:
"""Get state by key."""
try:
blob = self._bucket.blob(self._key_to_path(key))
if blob.exists():
data = json.loads(blob.download_as_text())
# Check TTL
if "_ttl_expires" in data:
if time.time() > data["_ttl_expires"]:
self.delete(key)
return None
del data["_ttl_expires"]
return data
return None
except Exception as e:
logger.error(f"Error getting state {key}: {e}")
return None
def set(self, key: str, value: Dict[str, Any], ttl: Optional[int] = None) -> bool:
"""Set state by key with optional TTL in seconds."""
try:
data = value.copy()
if ttl:
data["_ttl_expires"] = time.time() + ttl
blob = self._bucket.blob(self._key_to_path(key))
blob.upload_from_string(
json.dumps(data, default=str),
content_type="application/json"
)
return True
except Exception as e:
logger.error(f"Error setting state {key}: {e}")
return False
def delete(self, key: str) -> bool:
"""Delete state by key."""
try:
blob = self._bucket.blob(self._key_to_path(key))
if blob.exists():
blob.delete()
return True
return False
except Exception as e:
logger.error(f"Error deleting state {key}: {e}")
return False
def exists(self, key: str) -> bool:
"""Check if key exists."""
try:
blob = self._bucket.blob(self._key_to_path(key))
return blob.exists()
except Exception:
return False
def list_keys(self, prefix: Optional[str] = None) -> List[str]:
"""List all keys with optional prefix filter."""
try:
search_prefix = self.prefix
if prefix:
search_prefix = f"{self.prefix}{prefix}"
blobs = self._client.list_blobs(self.bucket_name, prefix=search_prefix)
keys = []
for blob in blobs:
# Remove prefix and .json suffix
key = blob.name[len(self.prefix):]
if key.endswith(".json"):
key = key[:-5]
keys.append(key)
return keys
except Exception as e:
logger.error(f"Error listing keys: {e}")
return []
def clear(self, prefix: Optional[str] = None) -> int:
"""Clear all keys with optional prefix filter."""
keys = self.list_keys(prefix)
count = 0
for key in keys:
if self.delete(key):
count += 1
return count
def close(self) -> None:
"""Close the store."""
self._client.close()
@@ -0,0 +1,245 @@
"""
In-memory / JSON file implementation of StateStore.
Zero external dependencies - uses Python built-ins.
"""
import json
import logging
import os
import threading
import time
from typing import Any, Dict, List, Optional
from .base import StateStore
logger = logging.getLogger(__name__)
class MemoryStateStore(StateStore):
"""
In-memory state store with optional JSON file persistence.
Zero external dependencies - ideal for development and testing.
Example:
# In-memory only
store = MemoryStateStore()
# With file persistence
store = MemoryStateStore(path="./state.json")
"""
def __init__(
self,
path: Optional[str] = None,
auto_save: bool = True,
save_interval: int = 60,
):
"""
Initialize memory state store.
Args:
path: Path to JSON file for persistence (None = in-memory only)
auto_save: Auto-save to file periodically
save_interval: Save interval in seconds
"""
self.path = path
self.auto_save = auto_save
self.save_interval = save_interval
self._data: Dict[str, Any] = {}
self._ttls: Dict[str, float] = {} # key -> expiry timestamp
self._lock = threading.RLock()
self._last_save = time.time()
# Load from file if exists
if path and os.path.exists(path):
self._load()
logger.info(f"MemoryStateStore initialized (path={path})")
def _load(self) -> None:
"""Load state from JSON file."""
if not self.path:
return
try:
with open(self.path, "r") as f:
saved = json.load(f)
self._data = saved.get("data", {})
self._ttls = saved.get("ttls", {})
# Clean expired keys
now = time.time()
expired = [k for k, exp in self._ttls.items() if exp <= now]
for k in expired:
self._data.pop(k, None)
self._ttls.pop(k, None)
logger.debug(f"Loaded {len(self._data)} keys from {self.path}")
except Exception as e:
logger.warning(f"Failed to load state from {self.path}: {e}")
def _save(self) -> None:
"""Save state to JSON file."""
if not self.path:
return
try:
os.makedirs(os.path.dirname(self.path) or ".", exist_ok=True)
with open(self.path, "w") as f:
json.dump({"data": self._data, "ttls": self._ttls}, f)
self._last_save = time.time()
except Exception as e:
logger.warning(f"Failed to save state to {self.path}: {e}")
def _maybe_save(self) -> None:
"""Save if auto_save is enabled and interval has passed."""
if self.auto_save and self.path:
if time.time() - self._last_save >= self.save_interval:
self._save()
def _check_ttl(self, key: str) -> bool:
"""Check if key is expired. Returns True if valid, False if expired."""
if key not in self._ttls:
return True
if self._ttls[key] <= time.time():
self._data.pop(key, None)
self._ttls.pop(key, None)
return False
return True
def get(self, key: str) -> Optional[Any]:
"""Get a value by key."""
with self._lock:
if not self._check_ttl(key):
return None
return self._data.get(key)
def set(
self,
key: str,
value: Any,
ttl: Optional[int] = None
) -> None:
"""Set a value with optional TTL."""
with self._lock:
self._data[key] = value
if ttl:
self._ttls[key] = time.time() + ttl
elif key in self._ttls:
del self._ttls[key]
self._maybe_save()
def delete(self, key: str) -> bool:
"""Delete a key."""
with self._lock:
existed = key in self._data
self._data.pop(key, None)
self._ttls.pop(key, None)
self._maybe_save()
return existed
def exists(self, key: str) -> bool:
"""Check if a key exists."""
with self._lock:
if not self._check_ttl(key):
return False
return key in self._data
def keys(self, pattern: str = "*") -> List[str]:
"""List keys matching pattern."""
import fnmatch
with self._lock:
# Clean expired keys first
now = time.time()
expired = [k for k, exp in self._ttls.items() if exp <= now]
for k in expired:
self._data.pop(k, None)
self._ttls.pop(k, None)
if pattern == "*":
return list(self._data.keys())
return [k for k in self._data.keys() if fnmatch.fnmatch(k, pattern)]
def ttl(self, key: str) -> Optional[int]:
"""Get remaining TTL in seconds."""
with self._lock:
if key not in self._ttls:
return None
remaining = self._ttls[key] - time.time()
if remaining <= 0:
self._data.pop(key, None)
self._ttls.pop(key, None)
return None
return int(remaining)
def expire(self, key: str, ttl: int) -> bool:
"""Set TTL on existing key."""
with self._lock:
if key not in self._data:
return False
self._ttls[key] = time.time() + ttl
self._maybe_save()
return True
def hget(self, key: str, field: str) -> Optional[Any]:
"""Get a field from a hash."""
with self._lock:
if not self._check_ttl(key):
return None
data = self._data.get(key)
if not isinstance(data, dict):
return None
return data.get(field)
def hset(self, key: str, field: str, value: Any) -> None:
"""Set a field in a hash."""
with self._lock:
if key not in self._data or not isinstance(self._data[key], dict):
self._data[key] = {}
self._data[key][field] = value
self._maybe_save()
def hgetall(self, key: str) -> Dict[str, Any]:
"""Get all fields from a hash."""
with self._lock:
if not self._check_ttl(key):
return {}
data = self._data.get(key)
if not isinstance(data, dict):
return {}
return dict(data)
def hdel(self, key: str, *fields: str) -> int:
"""Delete fields from a hash."""
with self._lock:
if key not in self._data or not isinstance(self._data[key], dict):
return 0
count = 0
for field in fields:
if field in self._data[key]:
del self._data[key][field]
count += 1
self._maybe_save()
return count
def flush(self) -> None:
"""Force save to file."""
with self._lock:
self._save()
def clear(self) -> None:
"""Clear all data."""
with self._lock:
self._data.clear()
self._ttls.clear()
self._maybe_save()
def close(self) -> None:
"""Close the store and save."""
with self._lock:
if self.path:
self._save()
Loaded 100 of 108 files, more files were not shown because too many files have changed in this diff. Show more