mirror of
https://github.com/MervinPraison/PraisonAI.git
synced 2026-11-10 03:31:28 -10:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
de9d4bd3c6 | ||
|
|
aba2265e5e | ||
|
|
ebb9b24bff | ||
|
|
759ef2c655 | ||
|
|
cb4318f6e8 | ||
|
|
d9bb869d5e |
No files matched your search
@@ -261,6 +261,12 @@ npm install praisonai
|
||||
| ↳ 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
|
||||
| ↳ Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
|
||||
| ↳ Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
|
||||
| ↳ Shadow Git Checkpoints | [Example](#10-shadow-git-checkpoints) | [📖](https://docs.praison.ai/features/checkpoints) |
|
||||
| ↳ Background Tasks | [Example](examples/background/basic_background.py) | [📖](https://docs.praison.ai/features/background-tasks) |
|
||||
| ↳ Policy Engine | [Example](examples/policy/basic_policy.py) | [📖](https://docs.praison.ai/features/policy-engine) |
|
||||
| ↳ Thinking Budgets | [Example](examples/thinking/basic_thinking.py) | [📖](https://docs.praison.ai/features/thinking-budgets) |
|
||||
| ↳ Output Styles | [Example](examples/output/basic_output.py) | [📖](https://docs.praison.ai/features/output-styles) |
|
||||
| ↳ Context Compaction | [Example](examples/compaction/basic_compaction.py) | [📖](https://docs.praison.ai/features/context-compaction) |
|
||||
| **📊 Monitoring & Management** | | |
|
||||
| ↳ Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
|
||||
| ↳ Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
@@ -930,8 +936,191 @@ praisonai checkpoint diff # Show changes
|
||||
praisonai checkpoint restore abc123 # Restore to checkpoint
|
||||
```
|
||||
|
||||
**Links:**
|
||||
- [📖 Coding Docs](https://docs.praison.ai/features/checkpoints)
|
||||
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/checkpoint)
|
||||
- [💻 Example](examples/checkpoints/basic_checkpoints.py)
|
||||
|
||||
### 11. Field Names Reference (A-I-G-S)
|
||||
---
|
||||
|
||||
### 11. Background Tasks
|
||||
|
||||
Run agent tasks asynchronously without blocking:
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
from praisonaiagents.background import BackgroundRunner, BackgroundConfig
|
||||
|
||||
async def main():
|
||||
config = BackgroundConfig(max_concurrent_tasks=3)
|
||||
runner = BackgroundRunner(config=config)
|
||||
|
||||
async def my_task(name: str) -> str:
|
||||
await asyncio.sleep(2)
|
||||
return f"Task {name} completed"
|
||||
|
||||
task = await runner.submit(my_task, args=("example",), name="my_task")
|
||||
await task.wait(timeout=10.0)
|
||||
print(task.result)
|
||||
|
||||
asyncio.run(main())
|
||||
```
|
||||
|
||||
**CLI Commands:**
|
||||
```bash
|
||||
praisonai background list # List running tasks
|
||||
praisonai background status <id> # Check task status
|
||||
praisonai background cancel <id> # Cancel a task
|
||||
praisonai background clear # Clear completed tasks
|
||||
```
|
||||
|
||||
**Links:**
|
||||
- [📖 Coding Docs](https://docs.praison.ai/features/background-tasks)
|
||||
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/background)
|
||||
- [💻 Example](examples/background/basic_background.py)
|
||||
|
||||
---
|
||||
|
||||
### 12. Policy Engine
|
||||
|
||||
Control what agents can and cannot do with policy-based execution:
|
||||
|
||||
```python
|
||||
from praisonaiagents.policy import (
|
||||
PolicyEngine, Policy, PolicyRule, PolicyAction
|
||||
)
|
||||
|
||||
engine = PolicyEngine()
|
||||
|
||||
policy = Policy(
|
||||
name="no_delete",
|
||||
rules=[
|
||||
PolicyRule(
|
||||
action=PolicyAction.DENY,
|
||||
resource="tool:delete_*",
|
||||
reason="Delete operations blocked"
|
||||
)
|
||||
]
|
||||
)
|
||||
engine.add_policy(policy)
|
||||
|
||||
result = engine.check("tool:delete_file", {})
|
||||
print(f"Allowed: {result.allowed}")
|
||||
```
|
||||
|
||||
**CLI Commands:**
|
||||
```bash
|
||||
praisonai policy list # List policies
|
||||
praisonai policy check "tool:name" # Check if allowed
|
||||
praisonai policy init # Create template
|
||||
```
|
||||
|
||||
**Links:**
|
||||
- [📖 Coding Docs](https://docs.praison.ai/features/policy-engine)
|
||||
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/policy)
|
||||
- [💻 Example](examples/policy/basic_policy.py)
|
||||
|
||||
---
|
||||
|
||||
### 13. Thinking Budgets
|
||||
|
||||
Configure token budgets for extended thinking:
|
||||
|
||||
```python
|
||||
from praisonaiagents.thinking import ThinkingBudget, ThinkingTracker
|
||||
|
||||
# Use predefined levels
|
||||
budget = ThinkingBudget.high() # 16,000 tokens
|
||||
|
||||
# Track usage
|
||||
tracker = ThinkingTracker()
|
||||
session = tracker.start_session(budget_tokens=16000)
|
||||
tracker.end_session(session, tokens_used=12000)
|
||||
|
||||
summary = tracker.get_summary()
|
||||
print(f"Utilization: {summary['average_utilization']:.1%}")
|
||||
```
|
||||
|
||||
**CLI Commands:**
|
||||
```bash
|
||||
praisonai thinking status # Show current budget
|
||||
praisonai thinking set high # Set budget level
|
||||
praisonai thinking stats # Show usage statistics
|
||||
```
|
||||
|
||||
**Links:**
|
||||
- [📖 Coding Docs](https://docs.praison.ai/features/thinking-budgets)
|
||||
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/thinking)
|
||||
- [💻 Example](examples/thinking/basic_thinking.py)
|
||||
|
||||
---
|
||||
|
||||
### 14. Output Styles
|
||||
|
||||
Configure how agents format their responses:
|
||||
|
||||
```python
|
||||
from praisonaiagents.output import OutputStyle, OutputFormatter
|
||||
|
||||
# Use preset styles
|
||||
style = OutputStyle.concise()
|
||||
formatter = OutputFormatter(style)
|
||||
|
||||
# Format output
|
||||
text = "# Hello\n\nThis is **bold** text."
|
||||
plain = formatter.format(text)
|
||||
print(plain)
|
||||
```
|
||||
|
||||
**CLI Commands:**
|
||||
```bash
|
||||
praisonai output status # Show current style
|
||||
praisonai output set concise # Set output style
|
||||
```
|
||||
|
||||
**Links:**
|
||||
- [📖 Coding Docs](https://docs.praison.ai/features/output-styles)
|
||||
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/output-style)
|
||||
- [💻 Example](examples/output/basic_output.py)
|
||||
|
||||
---
|
||||
|
||||
### 15. Context Compaction
|
||||
|
||||
Automatically manage context window size:
|
||||
|
||||
```python
|
||||
from praisonaiagents.compaction import (
|
||||
ContextCompactor, CompactionStrategy
|
||||
)
|
||||
|
||||
compactor = ContextCompactor(
|
||||
max_tokens=4000,
|
||||
strategy=CompactionStrategy.SLIDING,
|
||||
preserve_recent=3
|
||||
)
|
||||
|
||||
messages = [...] # Your conversation history
|
||||
compacted, result = compactor.compact(messages)
|
||||
|
||||
print(f"Compression: {result.compression_ratio:.1%}")
|
||||
```
|
||||
|
||||
**CLI Commands:**
|
||||
```bash
|
||||
praisonai compaction status # Show settings
|
||||
praisonai compaction set sliding # Set strategy
|
||||
praisonai compaction stats # Show statistics
|
||||
```
|
||||
|
||||
**Links:**
|
||||
- [📖 Coding Docs](https://docs.praison.ai/features/context-compaction)
|
||||
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/compaction)
|
||||
- [💻 Example](examples/compaction/basic_compaction.py)
|
||||
|
||||
---
|
||||
|
||||
### 16. Field Names Reference (A-I-G-S)
|
||||
|
||||
PraisonAI accepts both old (agents.yaml) and new (workflow.yaml) field names. Use the **canonical names** for new projects:
|
||||
|
||||
@@ -2535,6 +2724,103 @@ PraisonAI provides zero-dependency persistent memory for agents. For detailed ex
|
||||
|
||||
---
|
||||
|
||||
## 💾 Persistence (Databases)
|
||||
|
||||
Enable automatic conversation persistence with 2 lines of code:
|
||||
|
||||
```python
|
||||
from praisonaiagents import Agent, db
|
||||
|
||||
agent = Agent(
|
||||
name="Assistant",
|
||||
db=db(database_url="postgresql://localhost/mydb"), # db(...) shortcut
|
||||
session_id="my-session" # Optional: defaults to per-hour ID (YYYYMMDDHH)
|
||||
)
|
||||
agent.chat("Hello!") # Auto-persists messages, runs, traces
|
||||
```
|
||||
|
||||
### Persistence CLI Commands
|
||||
|
||||
| Command | Description |
|
||||
|---------|-------------|
|
||||
| `praisonai persistence doctor` | Validate DB connectivity |
|
||||
| `praisonai persistence run` | Run agent with persistence |
|
||||
| `praisonai persistence resume` | Resume existing session |
|
||||
| `praisonai persistence export` | Export session to JSONL |
|
||||
| `praisonai persistence import` | Import session from JSONL |
|
||||
| `praisonai persistence migrate` | Apply schema migrations |
|
||||
| `praisonai persistence status` | Show schema status |
|
||||
|
||||
### Databases Table
|
||||
|
||||
| Database | Store Type | Install | Example | Docs |
|
||||
|----------|------------|---------|---------|------|
|
||||
| PostgreSQL | Conversation | `pip install "praisonai[tools]"` | [simple_db_agent.py](examples/persistence/simple_db_agent.py) | [docs](https://docs.praison.ai/docs/databases/postgres) |
|
||||
| MySQL | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| SQLite | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| SingleStore | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| Supabase | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| SurrealDB | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| Qdrant | Knowledge | `pip install "praisonai[tools]"` | [knowledge_qdrant.py](examples/persistence/knowledge_qdrant.py) | [docs](https://docs.praison.ai/docs/databases/qdrant) |
|
||||
| ChromaDB | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| Pinecone | Knowledge | `pip install pinecone` | [pinecone_wow.py](examples/vector/pinecone_wow.py) | [docs](https://docs.praison.ai/docs/databases/pinecone) |
|
||||
| Weaviate | Knowledge | `pip install weaviate-client` | [weaviate_wow.py](examples/vector/weaviate_wow.py) | [docs](https://docs.praison.ai/docs/databases/weaviate) |
|
||||
| LanceDB | Knowledge | `pip install lancedb` | [lancedb_real_wow.py](examples/vector/lancedb_real_wow.py) | [docs](https://docs.praison.ai/docs/databases/lancedb) |
|
||||
| Milvus | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| PGVector | Knowledge | `pip install psycopg2-binary` | [pgvector_real_wow.py](examples/vector/pgvector_real_wow.py) | [docs](https://docs.praison.ai/docs/databases/pgvector) |
|
||||
| Redis Vector | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| Cassandra | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| ClickHouse | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| Redis | State | `pip install "praisonai[tools]"` | [state_redis.py](examples/persistence/state_redis.py) | [docs](https://docs.praison.ai/docs/databases/redis) |
|
||||
| MongoDB | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| DynamoDB | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| Firestore | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| Upstash | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| Memory | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
|
||||
---
|
||||
|
||||
## 🔧 Tools Table
|
||||
|
||||
Install all tools with: `pip install "praisonai[tools]"`
|
||||
|
||||
| Tool | Category | Import | Docs |
|
||||
|------|----------|--------|------|
|
||||
| Tavily | Web Search | `from praisonai_tools import TavilyTool` | [docs](https://docs.praison.ai/docs/tools/external/tavily) |
|
||||
| DuckDuckGo | Web Search | `from praisonai_tools import DuckDuckGoTool` | [docs](https://docs.praison.ai/docs/tools/external/duckduckgo) |
|
||||
| Exa | Web Search | `from praisonai_tools import ExaTool` | [docs](https://docs.praison.ai/docs/tools/external/exa) |
|
||||
| Serper | Web Search | `from praisonai_tools import SerperTool` | [docs](https://docs.praison.ai/docs/tools/external/serper) |
|
||||
| Jina | Web Reader | `from praisonai_tools import JinaTool` | [docs](https://docs.praison.ai/docs/tools/external/jina) |
|
||||
| Firecrawl | Web Scraping | `from praisonai_tools import FirecrawlTool` | [docs](https://docs.praison.ai/docs/tools/external/firecrawl) |
|
||||
| Crawl4AI | Web Scraping | `from praisonai_tools import Crawl4AITool` | [docs](https://docs.praison.ai/docs/tools/external/crawl4ai) |
|
||||
| Wikipedia | Knowledge | `from praisonai_tools import WikipediaTool` | [docs](https://docs.praison.ai/docs/tools/external/wikipedia) |
|
||||
| ArXiv | Research | `from praisonai_tools import ArxivTool` | [docs](https://docs.praison.ai/docs/tools/external/arxiv) |
|
||||
| HackerNews | News | `from praisonai_tools import HackerNewsTool` | [docs](https://docs.praison.ai/docs/tools/external/hackernews) |
|
||||
| YouTube | Media | `from praisonai_tools import YouTubeTool` | [docs](https://docs.praison.ai/docs/tools/external/youtube) |
|
||||
| Weather | Data | `from praisonai_tools import WeatherTool` | [docs](https://docs.praison.ai/docs/tools/external/weather) |
|
||||
| PostgreSQL | Database | `from praisonai_tools import PostgresTool` | [docs](https://docs.praison.ai/docs/tools/external/postgres) |
|
||||
| MySQL | Database | `from praisonai_tools import MySQLTool` | [docs](https://docs.praison.ai/docs/tools/external/mysql) |
|
||||
| SQLite | Database | `from praisonai_tools import SQLiteTool` | [docs](https://docs.praison.ai/docs/tools/external/sqlite) |
|
||||
| MongoDB | Database | `from praisonai_tools import MongoDBTool` | [docs](https://docs.praison.ai/docs/tools/external/mongodb) |
|
||||
| Redis | Database | `from praisonai_tools import RedisTool` | [docs](https://docs.praison.ai/docs/tools/external/redis) |
|
||||
| Qdrant | Vector DB | `from praisonai_tools import QdrantTool` | [docs](https://docs.praison.ai/docs/tools/external/qdrant) |
|
||||
| GitHub | DevOps | `from praisonai_tools import GitHubTool` | [docs](https://docs.praison.ai/docs/tools/external/github) |
|
||||
| Slack | Communication | `from praisonai_tools import SlackTool` | [docs](https://docs.praison.ai/docs/tools/external/slack) |
|
||||
| Discord | Communication | `from praisonai_tools import DiscordTool` | [docs](https://docs.praison.ai/docs/tools/external/discord) |
|
||||
| Telegram | Communication | `from praisonai_tools import TelegramTool` | [docs](https://docs.praison.ai/docs/tools/external/telegram) |
|
||||
| Email | Communication | `from praisonai_tools import EmailTool` | [docs](https://docs.praison.ai/docs/tools/external/email) |
|
||||
| Notion | Productivity | `from praisonai_tools import NotionTool` | [docs](https://docs.praison.ai/docs/tools/external/notion) |
|
||||
| File | File System | `from praisonai_tools import FileTool` | [docs](https://docs.praison.ai/docs/tools/external/file) |
|
||||
| Shell | System | `from praisonai_tools import ShellTool` | [docs](https://docs.praison.ai/docs/tools/external/shell) |
|
||||
| Python | Code | `from praisonai_tools import PythonTool` | [docs](https://docs.praison.ai/docs/tools/external/python) |
|
||||
| JSON | Data | `from praisonai_tools import JSONTool` | [docs](https://docs.praison.ai/docs/tools/external/json) |
|
||||
| CSV | Data | `from praisonai_tools import CSVTool` | [docs](https://docs.praison.ai/docs/tools/external/csv) |
|
||||
| Calculator | Math | `from praisonai_tools import CalculatorTool` | [docs](https://docs.praison.ai/docs/tools/external/calculator) |
|
||||
|
||||
> See [full tools documentation](https://docs.praison.ai/docs/tools/tools) for all 100+ available tools.
|
||||
|
||||
---
|
||||
|
||||
## 🎓 Video Tutorials
|
||||
|
||||
Learn PraisonAI through our comprehensive video series:
|
||||
|
||||
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=2.4.0" \
|
||||
"praisonai>=2.4.4" \
|
||||
"praisonai[chat]" \
|
||||
"embedchain[github,youtube]"
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=2.4.0" \
|
||||
"praisonai>=2.4.4" \
|
||||
"praisonai[ui]" \
|
||||
"praisonai[chat]" \
|
||||
"praisonai[realtime]" \
|
||||
|
||||
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
|
||||
# Install Python packages (using latest versions)
|
||||
RUN pip install --no-cache-dir \
|
||||
praisonai_tools \
|
||||
"praisonai>=2.4.0" \
|
||||
"praisonai>=2.4.4" \
|
||||
"praisonai[ui]" \
|
||||
"praisonai[crewai]"
|
||||
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
# Persistence CLI Commands
|
||||
|
||||
## Quick Reference
|
||||
|
||||
```bash
|
||||
# Check database connectivity
|
||||
praisonai persistence doctor \
|
||||
--conversation-url postgresql://localhost:5432/praisonai \
|
||||
--state-url redis://localhost:6379
|
||||
|
||||
# Run agent with persistence
|
||||
praisonai persistence run \
|
||||
--conversation-url postgresql://localhost:5432/praisonai \
|
||||
--session-id my-session
|
||||
|
||||
# Resume a session
|
||||
praisonai persistence resume \
|
||||
--conversation-url postgresql://localhost:5432/praisonai \
|
||||
--session-id my-session
|
||||
|
||||
# Export session to file
|
||||
praisonai persistence export \
|
||||
--conversation-url postgresql://localhost:5432/praisonai \
|
||||
--session-id my-session \
|
||||
--output session_backup.jsonl
|
||||
|
||||
# Import session from file
|
||||
praisonai persistence import \
|
||||
--conversation-url postgresql://localhost:5432/praisonai \
|
||||
--file session_backup.jsonl
|
||||
|
||||
# Check schema status
|
||||
praisonai persistence status \
|
||||
--conversation-url postgresql://localhost:5432/praisonai \
|
||||
--state-url redis://localhost:6379
|
||||
|
||||
# Apply migrations
|
||||
praisonai persistence migrate \
|
||||
--conversation-url postgresql://localhost:5432/praisonai \
|
||||
--state-url redis://localhost:6379
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
Instead of CLI flags, you can use environment variables:
|
||||
|
||||
```bash
|
||||
export PRAISON_CONVERSATION_URL=postgresql://localhost:5432/praisonai
|
||||
export PRAISON_STATE_URL=redis://localhost:6379
|
||||
export PRAISON_KNOWLEDGE_URL=http://localhost:6333
|
||||
|
||||
# Then run without URL flags
|
||||
praisonai persistence doctor
|
||||
praisonai persistence status
|
||||
```
|
||||
|
||||
## Docker Setup
|
||||
|
||||
```bash
|
||||
# Start all services
|
||||
docker run -d --name praison-postgres \
|
||||
-e POSTGRES_PASSWORD=praison123 \
|
||||
-e POSTGRES_DB=praisonai \
|
||||
-p 5432:5432 postgres:16
|
||||
|
||||
docker run -d --name praison-redis \
|
||||
-p 6379:6379 redis:7
|
||||
|
||||
docker run -d --name praison-qdrant \
|
||||
-p 6333:6333 -p 6334:6334 qdrant/qdrant
|
||||
|
||||
# Verify
|
||||
docker ps
|
||||
```
|
||||
|
||||
## Example Workflow
|
||||
|
||||
```bash
|
||||
# 1. Start services
|
||||
docker-compose up -d
|
||||
|
||||
# 2. Check connectivity
|
||||
praisonai persistence doctor
|
||||
|
||||
# 3. Run agent
|
||||
praisonai persistence run --session-id demo-session
|
||||
|
||||
# 4. Export for backup
|
||||
praisonai persistence export --session-id demo-session --output backup.jsonl
|
||||
|
||||
# 5. Import to another environment
|
||||
praisonai persistence import --file backup.jsonl
|
||||
```
|
||||
@@ -0,0 +1,124 @@
|
||||
# PraisonAI Persistence CLI - End-to-End Guide
|
||||
|
||||
This guide demonstrates the CLI commands for database persistence.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
```bash
|
||||
pip install "praisonai[tools]"
|
||||
|
||||
# Start Docker containers
|
||||
docker run -d --name praison-postgres -p 5432:5432 \
|
||||
-e POSTGRES_PASSWORD=praison123 \
|
||||
-e POSTGRES_DB=praisonai \
|
||||
postgres:16
|
||||
|
||||
docker run -d --name praison-qdrant -p 6333:6333 qdrant/qdrant
|
||||
docker run -d --name praison-redis -p 6379:6379 redis:7
|
||||
```
|
||||
|
||||
## 1. Doctor Command - Validate Connectivity
|
||||
|
||||
```bash
|
||||
# Check all stores
|
||||
praisonai persistence doctor \
|
||||
--conversation-url "postgresql://postgres:praison123@localhost:5432/praisonai" \
|
||||
--knowledge-url "http://localhost:6333" \
|
||||
--state-url "redis://localhost:6379"
|
||||
```
|
||||
|
||||
**Expected Output:**
|
||||
```
|
||||
==================================================
|
||||
PraisonAI Persistence Doctor
|
||||
==================================================
|
||||
|
||||
[Conversation Store] Testing: postgresql://postgres:****@localhost:5432/praisonai
|
||||
✅ Connected (postgres)
|
||||
|
||||
[Knowledge Store] Testing: http://localhost:6333
|
||||
✅ Connected (qdrant)
|
||||
|
||||
[State Store] Testing: redis://localhost:6379
|
||||
✅ Connected (redis)
|
||||
|
||||
==================================================
|
||||
Results: 3/3 stores connected successfully
|
||||
==================================================
|
||||
```
|
||||
|
||||
## 2. Run Command - Execute Agent with Persistence
|
||||
|
||||
```bash
|
||||
# Run agent with persistence (dry-run first)
|
||||
praisonai persistence run \
|
||||
--session-id "cli-demo-001" \
|
||||
--conversation-url "postgresql://postgres:praison123@localhost:5432/praisonai" \
|
||||
--dry-run \
|
||||
"Hello, my name is Bob"
|
||||
|
||||
# Actually run it
|
||||
praisonai persistence run \
|
||||
--session-id "cli-demo-001" \
|
||||
--conversation-url "postgresql://postgres:praison123@localhost:5432/praisonai" \
|
||||
"Hello, my name is Bob"
|
||||
```
|
||||
|
||||
**Expected Output:**
|
||||
```
|
||||
[Session: cli-demo-001]
|
||||
Agent: Hello Bob! Nice to meet you. How can I help you today?
|
||||
```
|
||||
|
||||
## 3. Resume Command - Continue a Session
|
||||
|
||||
```bash
|
||||
# Show history
|
||||
praisonai persistence resume \
|
||||
--session-id "cli-demo-001" \
|
||||
--conversation-url "postgresql://postgres:praison123@localhost:5432/praisonai" \
|
||||
--show-history
|
||||
|
||||
# Continue conversation
|
||||
praisonai persistence resume \
|
||||
--session-id "cli-demo-001" \
|
||||
--conversation-url "postgresql://postgres:praison123@localhost:5432/praisonai" \
|
||||
--continue "What's my name?"
|
||||
```
|
||||
|
||||
**Expected Output:**
|
||||
```
|
||||
[Session: cli-demo-001]
|
||||
Messages in history: 2
|
||||
|
||||
--- Conversation History ---
|
||||
[USER] Hello, my name is Bob
|
||||
[ASSISTANT] Hello Bob! Nice to meet you...
|
||||
--- End History ---
|
||||
|
||||
Agent: Your name is Bob!
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
Instead of passing URLs on command line, use environment variables:
|
||||
|
||||
```bash
|
||||
export PRAISON_CONVERSATION_URL="postgresql://postgres:praison123@localhost:5432/praisonai"
|
||||
export PRAISON_KNOWLEDGE_URL="http://localhost:6333"
|
||||
export PRAISON_STATE_URL="redis://localhost:6379"
|
||||
export OPENAI_API_KEY="your-key"
|
||||
|
||||
# Now commands are simpler
|
||||
praisonai persistence doctor --all
|
||||
praisonai persistence run --session-id "my-session" "Hello!"
|
||||
```
|
||||
|
||||
## Full CLI Reference
|
||||
|
||||
```
|
||||
praisonai persistence --help
|
||||
praisonai persistence doctor --help
|
||||
praisonai persistence run --help
|
||||
praisonai persistence resume --help
|
||||
```
|
||||
@@ -0,0 +1,122 @@
|
||||
"""
|
||||
Multi-Agent Persistence Example
|
||||
|
||||
Demonstrates how multiple agents can share or isolate conversation
|
||||
history using the persistence layer.
|
||||
"""
|
||||
|
||||
from praisonai.persistence.factory import create_conversation_store, create_state_store
|
||||
from praisonai.persistence.conversation.base import ConversationSession, ConversationMessage
|
||||
import uuid
|
||||
|
||||
|
||||
def main():
|
||||
# Create stores
|
||||
conv_store = create_conversation_store("sqlite", path=":memory:")
|
||||
state_store = create_state_store("memory")
|
||||
|
||||
print("=== Multi-Agent Persistence Demo ===\n")
|
||||
|
||||
# Scenario 1: Shared session (agents collaborate)
|
||||
print("--- Scenario 1: Shared Session ---")
|
||||
shared_session_id = f"shared-{uuid.uuid4().hex[:8]}"
|
||||
|
||||
session = ConversationSession(
|
||||
session_id=shared_session_id,
|
||||
user_id="user-123",
|
||||
name="Shared Conversation"
|
||||
)
|
||||
conv_store.create_session(session)
|
||||
|
||||
# Agent 1 (Researcher) adds context
|
||||
conv_store.add_message(shared_session_id, ConversationMessage(
|
||||
role="assistant",
|
||||
content="[Researcher] I found relevant information about the topic.",
|
||||
metadata={"agent_id": "researcher", "agent_role": "research"}
|
||||
))
|
||||
|
||||
# Agent 2 (Writer) uses the context
|
||||
conv_store.add_message(shared_session_id, ConversationMessage(
|
||||
role="assistant",
|
||||
content="[Writer] Based on the research, here's a summary...",
|
||||
metadata={"agent_id": "writer", "agent_role": "writing"}
|
||||
))
|
||||
|
||||
# Agent 3 (Editor) reviews
|
||||
conv_store.add_message(shared_session_id, ConversationMessage(
|
||||
role="assistant",
|
||||
content="[Editor] I've reviewed and polished the content.",
|
||||
metadata={"agent_id": "editor", "agent_role": "editing"}
|
||||
))
|
||||
|
||||
messages = conv_store.get_messages(shared_session_id)
|
||||
print(f"Shared session has {len(messages)} messages from different agents")
|
||||
for msg in messages:
|
||||
agent = msg.metadata.get("agent_id", "unknown") if msg.metadata else "unknown"
|
||||
print(f" - {agent}: {msg.content[:50]}...")
|
||||
|
||||
# Scenario 2: Isolated sessions (agents work independently)
|
||||
print("\n--- Scenario 2: Isolated Sessions ---")
|
||||
|
||||
agent_sessions = {}
|
||||
for agent_name in ["planner", "executor", "validator"]:
|
||||
session_id = f"{agent_name}-{uuid.uuid4().hex[:8]}"
|
||||
agent_sessions[agent_name] = session_id
|
||||
|
||||
conv_store.create_session(ConversationSession(
|
||||
session_id=session_id,
|
||||
agent_id=agent_name,
|
||||
name=f"{agent_name.title()} Session"
|
||||
))
|
||||
|
||||
# Each agent has its own conversation
|
||||
conv_store.add_message(session_id, ConversationMessage(
|
||||
role="user",
|
||||
content=f"Task for {agent_name}"
|
||||
))
|
||||
conv_store.add_message(session_id, ConversationMessage(
|
||||
role="assistant",
|
||||
content=f"[{agent_name.title()}] Processing task..."
|
||||
))
|
||||
|
||||
print("Created isolated sessions for each agent:")
|
||||
for agent, sid in agent_sessions.items():
|
||||
msgs = conv_store.get_messages(sid)
|
||||
print(f" - {agent}: {len(msgs)} messages")
|
||||
|
||||
# Scenario 3: Shared state across agents
|
||||
print("\n--- Scenario 3: Shared State ---")
|
||||
|
||||
# Agents share state for coordination
|
||||
state_store.set("workflow:status", {"phase": "research", "progress": 0})
|
||||
state_store.set("workflow:context", {"topic": "AI Agents", "deadline": "2024-12-31"})
|
||||
|
||||
# Agent 1 updates progress
|
||||
state = state_store.get("workflow:status")
|
||||
state["progress"] = 33
|
||||
state["phase"] = "writing"
|
||||
state_store.set("workflow:status", state)
|
||||
print(f"Agent 1 updated state: {state}")
|
||||
|
||||
# Agent 2 reads and updates
|
||||
state = state_store.get("workflow:status")
|
||||
state["progress"] = 66
|
||||
state["phase"] = "editing"
|
||||
state_store.set("workflow:status", state)
|
||||
print(f"Agent 2 updated state: {state}")
|
||||
|
||||
# Agent 3 completes
|
||||
state = state_store.get("workflow:status")
|
||||
state["progress"] = 100
|
||||
state["phase"] = "complete"
|
||||
state_store.set("workflow:status", state)
|
||||
print(f"Agent 3 completed: {state}")
|
||||
|
||||
# Cleanup
|
||||
conv_store.close()
|
||||
state_store.close()
|
||||
print("\n✅ Multi-agent persistence demo complete!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Multi-Agent Shared Session - DB Persistence Test"""
|
||||
import sys
|
||||
sys.path.insert(0, 'src/praisonai')
|
||||
from praisonai.persistence import create_conversation_store
|
||||
from praisonai.persistence.conversation.base import ConversationSession, ConversationMessage
|
||||
|
||||
# Create shared session for multiple agents
|
||||
store = create_conversation_store("sqlite", path="/tmp/multi_agent.db")
|
||||
session_id = "shared-session-123"
|
||||
|
||||
# Create session
|
||||
session = ConversationSession(session_id=session_id, agent_id="multi")
|
||||
try:
|
||||
store.create_session(session)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Agent 1 writes
|
||||
msg1 = ConversationMessage(session_id=session_id, role="user", content="Agent1: Hello")
|
||||
store.add_message(session_id, msg1)
|
||||
msg2 = ConversationMessage(session_id=session_id, role="assistant", content="Agent1: Hi there!")
|
||||
store.add_message(session_id, msg2)
|
||||
|
||||
# Agent 2 writes to same session
|
||||
msg3 = ConversationMessage(session_id=session_id, role="user", content="Agent2: What did Agent1 say?")
|
||||
store.add_message(session_id, msg3)
|
||||
msg4 = ConversationMessage(session_id=session_id, role="assistant", content="Agent2: Agent1 said Hello")
|
||||
store.add_message(session_id, msg4)
|
||||
|
||||
# Verify both agents' messages are in session
|
||||
messages = store.get_messages(session_id)
|
||||
print(f"Total messages in shared session: {len(messages)}")
|
||||
assert len(messages) >= 4, "Should have messages from both agents"
|
||||
|
||||
# Verify content
|
||||
contents = [m.content for m in messages]
|
||||
assert any("Agent1" in c for c in contents), "Should have Agent1 messages"
|
||||
assert any("Agent2" in c for c in contents), "Should have Agent2 messages"
|
||||
|
||||
print("PASSED: Multi-agent shared session persistence")
|
||||
@@ -0,0 +1,16 @@
|
||||
"""Multi-Agent Shared Session Demo"""
|
||||
from praisonaiagents import Agent
|
||||
from praisonai.persistence import create_conversation_store
|
||||
|
||||
store = create_conversation_store("sqlite", path="/tmp/multi_agent.db")
|
||||
|
||||
agent1 = Agent(name="Researcher", llm="gpt-4o-mini", db=store, session_id="shared-session")
|
||||
agent2 = Agent(name="Writer", llm="gpt-4o-mini", db=store, session_id="shared-session")
|
||||
|
||||
r1 = agent1.chat("Research topic: AI in healthcare. Key finding: reduces diagnosis time by 50%")
|
||||
print(f"Researcher: {r1[:50]}...")
|
||||
|
||||
r2 = agent2.chat("Based on the research, what was the key finding?")
|
||||
print(f"Writer: {r2[:50]}...")
|
||||
assert "50" in r2 or "diagnosis" in r2.lower(), "Writer should see Researcher's context"
|
||||
print("PASSED: Multi-agent shared session works")
|
||||
@@ -0,0 +1,48 @@
|
||||
"""
|
||||
Basic Observability Usage Example
|
||||
|
||||
This example demonstrates how to use the observability module
|
||||
with PraisonAI agents.
|
||||
|
||||
Usage:
|
||||
# Set your provider's API key first
|
||||
export LANGFUSE_PUBLIC_KEY=pk-lf-xxx
|
||||
export LANGFUSE_SECRET_KEY=sk-lf-xxx
|
||||
|
||||
python basic_usage.py
|
||||
"""
|
||||
|
||||
from praisonai_tools.observability import obs
|
||||
from praisonaiagents import Agent
|
||||
|
||||
# Initialize observability (auto-detects provider from env vars)
|
||||
obs.init()
|
||||
|
||||
# Or specify a provider explicitly
|
||||
# obs.init(provider="langfuse", project_name="my-project")
|
||||
|
||||
# Create an agent
|
||||
agent = Agent(
|
||||
instructions="You are a helpful assistant.",
|
||||
model="gpt-4o-mini",
|
||||
)
|
||||
|
||||
# Use trace context manager for complete workflow tracing
|
||||
with obs.trace("chat-session", session_id="user-123"):
|
||||
# Use span context manager for individual operations
|
||||
with obs.span("user-query", kind=obs.SpanKind.AGENT):
|
||||
response = agent.chat("What is the capital of France?")
|
||||
print(response)
|
||||
|
||||
# Log LLM calls manually if needed
|
||||
obs.log_llm_call(
|
||||
model="gpt-4o-mini",
|
||||
input_messages="What is 2+2?",
|
||||
output="4",
|
||||
input_tokens=10,
|
||||
output_tokens=1,
|
||||
)
|
||||
|
||||
# Check diagnostics
|
||||
print("\nDiagnostics:")
|
||||
print(obs.doctor())
|
||||
@@ -0,0 +1,48 @@
|
||||
"""
|
||||
Langfuse Integration Example
|
||||
|
||||
This example shows how to use Langfuse for LLM observability.
|
||||
|
||||
Setup:
|
||||
1. Sign up at https://langfuse.com/
|
||||
2. Get your API keys from the project settings
|
||||
3. Set environment variables:
|
||||
export LANGFUSE_PUBLIC_KEY=pk-lf-xxx
|
||||
export LANGFUSE_SECRET_KEY=sk-lf-xxx
|
||||
4. Install dependencies:
|
||||
pip install opentelemetry-sdk opentelemetry-exporter-otlp
|
||||
|
||||
Usage:
|
||||
python langfuse_example.py
|
||||
"""
|
||||
|
||||
import os
|
||||
from praisonai_tools.observability import obs
|
||||
from praisonaiagents import Agent
|
||||
|
||||
# Initialize Langfuse
|
||||
success = obs.init(
|
||||
provider="langfuse",
|
||||
project_name="praisonai-demo",
|
||||
)
|
||||
|
||||
if not success:
|
||||
print("Failed to initialize Langfuse. Check your API keys.")
|
||||
print("Required: LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY")
|
||||
exit(1)
|
||||
|
||||
print("Langfuse initialized successfully!")
|
||||
print(f"View traces at: https://cloud.langfuse.com/")
|
||||
|
||||
# Create agent
|
||||
agent = Agent(
|
||||
instructions="You are a helpful coding assistant.",
|
||||
model="gpt-4o-mini",
|
||||
)
|
||||
|
||||
# Run with tracing
|
||||
with obs.trace("coding-session", user_id="developer-1"):
|
||||
response = agent.chat("Write a Python function to calculate fibonacci numbers")
|
||||
print(response)
|
||||
|
||||
print("\nCheck Langfuse dashboard for traces!")
|
||||
@@ -0,0 +1,22 @@
|
||||
"""MLflow Local Observability Demo (no API key needed)"""
|
||||
import mlflow
|
||||
import time
|
||||
|
||||
mlflow.set_tracking_uri("file:///tmp/mlflow_demo")
|
||||
mlflow.set_experiment("praisonai-demo")
|
||||
|
||||
with mlflow.start_run(run_name="agent-trace"):
|
||||
mlflow.log_param("agent_name", "Assistant")
|
||||
mlflow.log_param("model", "gpt-4o-mini")
|
||||
|
||||
# Simulate agent work
|
||||
start = time.time()
|
||||
time.sleep(0.1)
|
||||
latency = time.time() - start
|
||||
|
||||
mlflow.log_metric("latency_ms", latency * 1000)
|
||||
mlflow.log_metric("tokens_used", 150)
|
||||
|
||||
print(f"Logged run to MLflow: latency={latency*1000:.1f}ms")
|
||||
|
||||
print("PASSED: MLflow local observability works")
|
||||
@@ -0,0 +1,53 @@
|
||||
"""
|
||||
Multi-Agent Tracing Example
|
||||
|
||||
This example shows how to trace multi-agent workflows with
|
||||
parent/child span relationships.
|
||||
|
||||
Usage:
|
||||
# Set your provider's API key
|
||||
export LANGFUSE_PUBLIC_KEY=pk-lf-xxx
|
||||
export LANGFUSE_SECRET_KEY=sk-lf-xxx
|
||||
|
||||
python multi_agent_tracing.py
|
||||
"""
|
||||
|
||||
from praisonai_tools.observability import obs
|
||||
from praisonai_tools.observability.base import SpanKind
|
||||
from praisonaiagents import Agent, PraisonAIAgents
|
||||
|
||||
# Initialize observability
|
||||
obs.init()
|
||||
|
||||
# Create agents
|
||||
researcher = Agent(
|
||||
name="Researcher",
|
||||
instructions="You research topics and provide detailed information.",
|
||||
model="gpt-4o-mini",
|
||||
)
|
||||
|
||||
writer = Agent(
|
||||
name="Writer",
|
||||
instructions="You write clear, engaging content based on research.",
|
||||
model="gpt-4o-mini",
|
||||
)
|
||||
|
||||
# Trace the multi-agent workflow
|
||||
with obs.trace("content-creation", metadata={"workflow": "research-write"}):
|
||||
|
||||
# Research phase
|
||||
with obs.span("research-phase", kind=SpanKind.AGENT) as research_span:
|
||||
research_span.attributes["agent"] = "Researcher"
|
||||
research_result = researcher.chat("Research the benefits of AI in healthcare")
|
||||
|
||||
# Writing phase
|
||||
with obs.span("writing-phase", kind=SpanKind.AGENT) as write_span:
|
||||
write_span.attributes["agent"] = "Writer"
|
||||
write_span.attributes["input_length"] = len(str(research_result))
|
||||
final_content = writer.chat(f"Write a blog post based on: {research_result}")
|
||||
|
||||
print("Final Content:")
|
||||
print(final_content)
|
||||
|
||||
# Check trace info
|
||||
print("\nDiagnostics:", obs.doctor())
|
||||
@@ -0,0 +1,79 @@
|
||||
"""
|
||||
Qdrant Knowledge Store Example for PraisonAI.
|
||||
|
||||
Demonstrates vector insert + search through the knowledge store.
|
||||
|
||||
Requirements:
|
||||
pip install "praisonai[tools]"
|
||||
|
||||
Docker Setup:
|
||||
docker run -d --name praison-qdrant -p 6333:6333 qdrant/qdrant
|
||||
|
||||
Run:
|
||||
python knowledge_qdrant.py
|
||||
|
||||
Expected Output:
|
||||
=== Qdrant Knowledge Store Demo ===
|
||||
Created collection: demo_knowledge
|
||||
Inserted 3 documents
|
||||
Search results: 2 documents found
|
||||
"""
|
||||
|
||||
from praisonai.persistence.factory import create_knowledge_store
|
||||
from praisonai.persistence.knowledge.base import KnowledgeDocument
|
||||
import random
|
||||
|
||||
# Create Qdrant knowledge store
|
||||
store = create_knowledge_store("qdrant", url="http://localhost:6333")
|
||||
|
||||
collection_name = "demo_knowledge"
|
||||
dimension = 384 # Embedding dimension
|
||||
|
||||
# Create collection
|
||||
print("=== Qdrant Knowledge Store Demo ===")
|
||||
if store.collection_exists(collection_name):
|
||||
store.delete_collection(collection_name)
|
||||
|
||||
store.create_collection(collection_name, dimension=dimension)
|
||||
print(f"Created collection: {collection_name}")
|
||||
|
||||
# Create sample documents with embeddings
|
||||
random.seed(42)
|
||||
documents = [
|
||||
KnowledgeDocument(
|
||||
id="doc-1",
|
||||
content="Python is a versatile programming language used for web development, data science, and AI.",
|
||||
embedding=[random.random() for _ in range(dimension)],
|
||||
metadata={"category": "programming", "language": "python"}
|
||||
),
|
||||
KnowledgeDocument(
|
||||
id="doc-2",
|
||||
content="Machine learning enables computers to learn patterns from data without explicit programming.",
|
||||
embedding=[random.random() for _ in range(dimension)],
|
||||
metadata={"category": "ai", "topic": "ml"}
|
||||
),
|
||||
KnowledgeDocument(
|
||||
id="doc-3",
|
||||
content="PostgreSQL is a powerful open-source relational database system.",
|
||||
embedding=[random.random() for _ in range(dimension)],
|
||||
metadata={"category": "database", "type": "sql"}
|
||||
),
|
||||
]
|
||||
|
||||
# Insert documents
|
||||
ids = store.insert(collection_name, documents)
|
||||
print(f"Inserted {len(ids)} documents")
|
||||
|
||||
# Search for similar documents
|
||||
query_embedding = [random.random() for _ in range(dimension)]
|
||||
results = store.search(collection_name, query_embedding, limit=2)
|
||||
print(f"Search results: {len(results)} documents found")
|
||||
|
||||
for doc in results:
|
||||
print(f" - {doc.id}: {doc.content[:50]}...")
|
||||
|
||||
# Cleanup
|
||||
store.delete_collection(collection_name)
|
||||
store.close()
|
||||
|
||||
print("\n=== Demo Complete ===")
|
||||
@@ -0,0 +1,43 @@
|
||||
"""
|
||||
Minimal Agent with Database Persistence
|
||||
|
||||
This is the simplest way to add persistence to your agent.
|
||||
Messages, runs, and traces are automatically saved.
|
||||
|
||||
Run:
|
||||
python minimal_agent_db.py
|
||||
|
||||
Expected output:
|
||||
- Agent responds to your question
|
||||
- Data is persisted to PostgreSQL
|
||||
- Session can be resumed later with same session_id
|
||||
"""
|
||||
|
||||
from praisonaiagents import Agent, db
|
||||
|
||||
# Create database connection (PostgreSQL + Redis)
|
||||
my_db = db(
|
||||
database_url="postgresql://postgres:praison123@localhost:5432/praisonai",
|
||||
state_url="redis://localhost:6379" # Optional: for state/tracing
|
||||
)
|
||||
|
||||
# Create agent with persistence
|
||||
agent = Agent(
|
||||
name="Assistant",
|
||||
instructions="You are a helpful assistant. Be concise.",
|
||||
db=my_db,
|
||||
# session_id is optional - defaults to per-hour ID (YYYYMMDDHH-hash)
|
||||
# Set explicitly for session continuity across runs:
|
||||
session_id="my-persistent-session"
|
||||
)
|
||||
|
||||
# Chat - messages are automatically persisted
|
||||
response = agent.chat("What is the capital of France?")
|
||||
print(f"Response: {response}")
|
||||
|
||||
# Verify persistence
|
||||
data = my_db.export_session("my-persistent-session")
|
||||
print(f"\nMessages stored: {len(data.get('messages', []))}")
|
||||
|
||||
# Clean up
|
||||
my_db.close()
|
||||
@@ -0,0 +1,61 @@
|
||||
"""
|
||||
PostgreSQL with Runs and Traces
|
||||
|
||||
Demonstrates full persistence with run tracking and tracing.
|
||||
|
||||
Run:
|
||||
python postgres_runs_traces.py
|
||||
|
||||
Expected output:
|
||||
- Agent responds with tool usage
|
||||
- Runs and traces are persisted
|
||||
- Export shows full session data
|
||||
"""
|
||||
|
||||
from praisonaiagents import Agent, db
|
||||
|
||||
def get_weather(city: str) -> str:
|
||||
"""Get weather for a city."""
|
||||
return f"Weather in {city}: Sunny, 72°F"
|
||||
|
||||
# Create database with PostgreSQL + Redis
|
||||
my_db = db(
|
||||
database_url="postgresql://postgres:praison123@localhost:5432/praisonai",
|
||||
state_url="redis://localhost:6379"
|
||||
)
|
||||
|
||||
# Create agent with tools
|
||||
agent = Agent(
|
||||
name="WeatherBot",
|
||||
instructions="You help with weather. Use the get_weather tool when asked.",
|
||||
tools=[get_weather],
|
||||
db=my_db,
|
||||
session_id="weather-session-001"
|
||||
)
|
||||
|
||||
# First chat - triggers run tracking
|
||||
print("=== First Question ===")
|
||||
response1 = agent.chat("What's the weather in Tokyo?")
|
||||
print(f"Response: {response1}\n")
|
||||
|
||||
# Second chat - same session, new run
|
||||
print("=== Second Question ===")
|
||||
response2 = agent.chat("How about New York?")
|
||||
print(f"Response: {response2}\n")
|
||||
|
||||
# Export session data
|
||||
print("=== Session Data ===")
|
||||
data = my_db.export_session("weather-session-001")
|
||||
print(f"Session ID: {data.get('session_id')}")
|
||||
print(f"Messages: {len(data.get('messages', []))}")
|
||||
print(f"Agent: {data.get('agent_name')}")
|
||||
|
||||
# Show messages
|
||||
print("\n=== Messages ===")
|
||||
for msg in data.get('messages', []):
|
||||
role = msg.get('role', 'unknown')
|
||||
content = msg.get('content', '')[:60]
|
||||
print(f" [{role}] {content}...")
|
||||
|
||||
my_db.close()
|
||||
print("\n✅ Done")
|
||||
@@ -0,0 +1,83 @@
|
||||
"""
|
||||
Redis State Store
|
||||
|
||||
Use Redis for state management, tracing, and caching.
|
||||
|
||||
Run:
|
||||
python redis_state.py
|
||||
|
||||
Expected output:
|
||||
- State operations (set/get/delete)
|
||||
- Tracing operations
|
||||
"""
|
||||
|
||||
from praisonaiagents import db
|
||||
|
||||
# Create Redis-only database (for state/tracing)
|
||||
redis_db = db.RedisDB(host="localhost", port=6379)
|
||||
|
||||
print("=== Redis State Store ===")
|
||||
|
||||
# Test state operations
|
||||
print("\n1. State Operations:")
|
||||
redis_db._init_stores() # Initialize stores
|
||||
|
||||
# Set a value
|
||||
redis_db._state_store.set("user:123:preferences", {
|
||||
"theme": "dark",
|
||||
"language": "en"
|
||||
})
|
||||
print(" Set user preferences")
|
||||
|
||||
# Get the value
|
||||
prefs = redis_db._state_store.get("user:123:preferences")
|
||||
print(f" Got preferences: {prefs}")
|
||||
|
||||
# Set with TTL (expires in 60 seconds)
|
||||
redis_db._state_store.set("temp:token", "abc123", ttl=60)
|
||||
print(" Set temporary token (60s TTL)")
|
||||
|
||||
# Delete
|
||||
redis_db._state_store.delete("user:123:preferences")
|
||||
print(" Deleted preferences")
|
||||
|
||||
# Test tracing
|
||||
print("\n2. Tracing Operations:")
|
||||
|
||||
# Start a trace
|
||||
trace_id = "trace-example-001"
|
||||
redis_db.on_trace_start(
|
||||
trace_id=trace_id,
|
||||
session_id="example-session",
|
||||
agent_name="ExampleAgent",
|
||||
metadata={"purpose": "demo"}
|
||||
)
|
||||
print(f" Started trace: {trace_id}")
|
||||
|
||||
# Add spans
|
||||
span_id = "span-llm-001"
|
||||
redis_db.on_span_start(
|
||||
span_id=span_id,
|
||||
trace_id=trace_id,
|
||||
name="llm_call",
|
||||
attributes={"model": "gpt-4", "temperature": 0.7}
|
||||
)
|
||||
print(f" Started span: {span_id}")
|
||||
|
||||
redis_db.on_span_end(
|
||||
span_id=span_id,
|
||||
status="ok",
|
||||
attributes={"tokens": 150, "latency_ms": 450}
|
||||
)
|
||||
print(f" Ended span: {span_id}")
|
||||
|
||||
# End trace
|
||||
redis_db.on_trace_end(
|
||||
trace_id=trace_id,
|
||||
status="ok",
|
||||
metadata={"total_spans": 1}
|
||||
)
|
||||
print(f" Ended trace: {trace_id}")
|
||||
|
||||
redis_db.close()
|
||||
print("\n✅ Done")
|
||||
@@ -0,0 +1,66 @@
|
||||
"""
|
||||
Session Resume Example
|
||||
|
||||
Demonstrates how to resume a conversation from a previous session.
|
||||
|
||||
Run:
|
||||
python session_resume.py
|
||||
|
||||
Expected output:
|
||||
- First run: Creates new session
|
||||
- Second run: Resumes with history
|
||||
"""
|
||||
|
||||
from praisonaiagents import Agent, db
|
||||
|
||||
# Create database
|
||||
my_db = db(
|
||||
database_url="postgresql://postgres:praison123@localhost:5432/praisonai",
|
||||
state_url="redis://localhost:6379"
|
||||
)
|
||||
|
||||
SESSION_ID = "resume-demo-session"
|
||||
|
||||
# Create agent with explicit session_id
|
||||
agent = Agent(
|
||||
name="MemoryBot",
|
||||
instructions="You remember our conversation. Be helpful and concise.",
|
||||
db=my_db,
|
||||
session_id=SESSION_ID
|
||||
)
|
||||
|
||||
# Check if this is a new or resumed session
|
||||
data = my_db.export_session(SESSION_ID)
|
||||
existing_messages = len(data.get('messages', []))
|
||||
|
||||
if existing_messages == 0:
|
||||
print("=== NEW SESSION ===")
|
||||
print("This is a new conversation.\n")
|
||||
|
||||
response = agent.chat("My name is Alice and I love Python programming.")
|
||||
print(f"Response: {response}\n")
|
||||
|
||||
response = agent.chat("What's my favorite programming language?")
|
||||
print(f"Response: {response}\n")
|
||||
else:
|
||||
print(f"=== RESUMED SESSION ({existing_messages} previous messages) ===")
|
||||
print("Continuing previous conversation.\n")
|
||||
|
||||
# Show previous messages
|
||||
print("Previous messages:")
|
||||
for msg in data.get('messages', [])[:4]:
|
||||
role = msg.get('role', 'unknown')
|
||||
content = msg.get('content', '')[:60]
|
||||
print(f" [{role}] {content}...")
|
||||
print()
|
||||
|
||||
# Continue conversation
|
||||
response = agent.chat("What's my name and what do I like?")
|
||||
print(f"Response: {response}\n")
|
||||
|
||||
# Show final state
|
||||
final_data = my_db.export_session(SESSION_ID)
|
||||
print(f"Total messages now: {len(final_data.get('messages', []))}")
|
||||
|
||||
my_db.close()
|
||||
print("✅ Done")
|
||||
@@ -0,0 +1,63 @@
|
||||
"""
|
||||
Simple Database Persistence Example for PraisonAI Agents.
|
||||
|
||||
This example shows how to enable automatic conversation persistence
|
||||
with just 2 lines of code.
|
||||
|
||||
Requirements:
|
||||
pip install "praisonai[tools]"
|
||||
|
||||
Docker Setup:
|
||||
docker run -d --name praison-postgres -p 5432:5432 \
|
||||
-e POSTGRES_PASSWORD=praison123 \
|
||||
-e POSTGRES_DB=praisonai \
|
||||
postgres:16
|
||||
|
||||
Environment:
|
||||
export OPENAI_API_KEY=your-key
|
||||
|
||||
Run:
|
||||
python simple_agent_postgres.py
|
||||
|
||||
Expected Output:
|
||||
=== First Run ===
|
||||
Agent: Nice to meet you, Alice! Python is a great choice...
|
||||
Session ID: demo-session-001
|
||||
Messages persisted: 2
|
||||
"""
|
||||
|
||||
from praisonaiagents import Agent
|
||||
from praisonai.db import PostgresDB
|
||||
|
||||
# Create database adapter
|
||||
db = PostgresDB(
|
||||
host="localhost",
|
||||
port=5432,
|
||||
database="praisonai",
|
||||
user="postgres",
|
||||
password="praison123"
|
||||
)
|
||||
|
||||
# Create agent with persistence enabled
|
||||
agent = Agent(
|
||||
name="Assistant",
|
||||
instructions="You are a helpful assistant. Keep responses brief.",
|
||||
db=db,
|
||||
session_id="demo-session-001", # Same session_id = resume conversation
|
||||
verbose=False
|
||||
)
|
||||
|
||||
# Chat with persistence
|
||||
print("=== First Run ===")
|
||||
response = agent.chat("My name is Alice and I love Python programming.")
|
||||
print(f"Agent: {response}\n")
|
||||
|
||||
# Show session info
|
||||
print(f"Session ID: {agent.session_id}")
|
||||
print(f"Messages in history: {len(agent.chat_history)}")
|
||||
|
||||
# Close DB connection
|
||||
db.close()
|
||||
|
||||
print("\n=== Session persisted! ===")
|
||||
print("Run again with same session_id to resume.")
|
||||
@@ -0,0 +1,41 @@
|
||||
"""
|
||||
SQLite Local Database
|
||||
|
||||
Use SQLite for simple local persistence without external services.
|
||||
|
||||
Run:
|
||||
python sqlite_local.py
|
||||
|
||||
Expected output:
|
||||
- Agent responds
|
||||
- Data persisted to local SQLite file
|
||||
"""
|
||||
|
||||
from praisonaiagents import Agent, db
|
||||
import tempfile
|
||||
import os
|
||||
|
||||
# Create SQLite database (local file)
|
||||
db_path = os.path.join(tempfile.gettempdir(), "my_agent.db")
|
||||
my_db = db.SQLiteDB(path=db_path)
|
||||
|
||||
print(f"Database: {db_path}")
|
||||
|
||||
# Create agent
|
||||
agent = Agent(
|
||||
name="LocalBot",
|
||||
instructions="You are a helpful assistant.",
|
||||
db=my_db,
|
||||
session_id="local-session"
|
||||
)
|
||||
|
||||
# Chat
|
||||
response = agent.chat("Hello! What can you help me with?")
|
||||
print(f"Response: {response}")
|
||||
|
||||
# Verify
|
||||
data = my_db.export_session("local-session")
|
||||
print(f"\nMessages stored: {len(data.get('messages', []))}")
|
||||
|
||||
my_db.close()
|
||||
print("✅ Done")
|
||||
@@ -0,0 +1,67 @@
|
||||
"""
|
||||
Redis State Store Example for PraisonAI.
|
||||
|
||||
Demonstrates set/get/delete + TTL operations.
|
||||
|
||||
Requirements:
|
||||
pip install "praisonai[tools]"
|
||||
|
||||
Docker Setup:
|
||||
docker run -d --name praison-redis -p 6379:6379 redis:7
|
||||
|
||||
Run:
|
||||
python state_redis.py
|
||||
|
||||
Expected Output:
|
||||
=== Redis State Store Demo ===
|
||||
Set key: session:user123:preferences
|
||||
Get value: {'theme': 'dark', 'language': 'en'}
|
||||
TTL remaining: 3600 seconds
|
||||
Hash operations: {'name': 'Alice', 'role': 'admin'}
|
||||
"""
|
||||
|
||||
from praisonai.persistence.factory import create_state_store
|
||||
|
||||
# Create Redis state store
|
||||
store = create_state_store("redis", url="redis://localhost:6379")
|
||||
|
||||
print("=== Redis State Store Demo ===")
|
||||
|
||||
# Basic set/get
|
||||
key = "session:user123:preferences"
|
||||
value = {"theme": "dark", "language": "en", "notifications": True}
|
||||
|
||||
store.set(key, value)
|
||||
print(f"Set key: {key}")
|
||||
|
||||
retrieved = store.get(key)
|
||||
print(f"Get value: {retrieved}")
|
||||
|
||||
# Set with TTL (expires in 1 hour)
|
||||
ttl_key = "session:user123:token"
|
||||
store.set(ttl_key, "abc123xyz", ttl=3600)
|
||||
remaining = store.ttl(ttl_key)
|
||||
print(f"TTL remaining: {remaining} seconds")
|
||||
|
||||
# Hash operations (for structured data)
|
||||
hash_key = "user:123:profile"
|
||||
store.hset(hash_key, "name", "Alice")
|
||||
store.hset(hash_key, "role", "admin")
|
||||
store.hset(hash_key, "last_login", "2024-12-24")
|
||||
|
||||
profile = store.hgetall(hash_key)
|
||||
print(f"Hash operations: {profile}")
|
||||
|
||||
# Check existence
|
||||
exists = store.exists(key)
|
||||
print(f"Key exists: {exists}")
|
||||
|
||||
# Delete keys
|
||||
store.delete(key)
|
||||
store.delete(ttl_key)
|
||||
store.delete(hash_key)
|
||||
print("Cleaned up test keys")
|
||||
|
||||
store.close()
|
||||
|
||||
print("\n=== Demo Complete ===")
|
||||
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run all examples and collect PASS/SKIP/FAIL results."""
|
||||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
EXAMPLES_DIR = Path(__file__).parent
|
||||
RESULTS = []
|
||||
|
||||
def run_example(path: Path) -> tuple:
|
||||
"""Run an example and return (name, status, reason)."""
|
||||
name = f"{path.parent.name}/{path.name}"
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(path)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
env={**os.environ, "PYTHONPATH": str(EXAMPLES_DIR.parent / "src")}
|
||||
)
|
||||
output = result.stdout + result.stderr
|
||||
if "PASSED" in output:
|
||||
return (name, "PASS", "")
|
||||
elif "SKIPPED" in output:
|
||||
return (name, "SKIP", output.split("SKIPPED:")[-1].strip()[:50])
|
||||
elif result.returncode != 0:
|
||||
error = output.strip().split("\n")[-1][:80]
|
||||
return (name, "FAIL", error)
|
||||
else:
|
||||
return (name, "PASS", "")
|
||||
except subprocess.TimeoutExpired:
|
||||
return (name, "FAIL", "Timeout")
|
||||
except Exception as e:
|
||||
return (name, "FAIL", str(e)[:50])
|
||||
|
||||
def main():
|
||||
print("=" * 70)
|
||||
print("EXAMPLE RUNNER - PASS/SKIP/FAIL SUMMARY")
|
||||
print("=" * 70)
|
||||
|
||||
# Find all example files
|
||||
examples = []
|
||||
for subdir in ["db", "vector", "observability", "multi_agent", "workflows"]:
|
||||
subpath = EXAMPLES_DIR / subdir
|
||||
if subpath.exists():
|
||||
examples.extend(sorted(subpath.glob("*_wow.py")))
|
||||
|
||||
print(f"\nFound {len(examples)} examples to run\n")
|
||||
|
||||
for ex in examples:
|
||||
name, status, reason = run_example(ex)
|
||||
RESULTS.append((name, status, reason))
|
||||
icon = {"PASS": "✅", "SKIP": "⏭️", "FAIL": "❌"}.get(status, "?")
|
||||
reason_str = f" ({reason})" if reason else ""
|
||||
print(f"{icon} {status:4} | {name:45} {reason_str}")
|
||||
|
||||
# Summary
|
||||
print("\n" + "=" * 70)
|
||||
passed = sum(1 for _, s, _ in RESULTS if s == "PASS")
|
||||
skipped = sum(1 for _, s, _ in RESULTS if s == "SKIP")
|
||||
failed = sum(1 for _, s, _ in RESULTS if s == "FAIL")
|
||||
print(f"SUMMARY: {passed} PASS | {skipped} SKIP | {failed} FAIL | {len(RESULTS)} TOTAL")
|
||||
print("=" * 70)
|
||||
|
||||
return 0 if failed == 0 else 1
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
"""ArXiv Tool Example.
|
||||
|
||||
This example demonstrates how to use the ArXiv tool with PraisonAI agents.
|
||||
|
||||
Requirements:
|
||||
pip install "praisonai[tools]"
|
||||
|
||||
Usage:
|
||||
python tool.py
|
||||
|
||||
Note: No API key required!
|
||||
"""
|
||||
|
||||
from praisonai_tools import ArxivTool
|
||||
|
||||
|
||||
def main():
|
||||
# Initialize ArXiv tool (no API key needed)
|
||||
arxiv = ArxivTool()
|
||||
|
||||
# Example 1: Search ArXiv
|
||||
print("=" * 60)
|
||||
print("Example 1: Search ArXiv")
|
||||
print("=" * 60)
|
||||
|
||||
results = arxiv.search("transformer neural networks", max_results=3)
|
||||
|
||||
if results and "error" in results[0]:
|
||||
print(f"Error: {results[0]['error']}")
|
||||
else:
|
||||
print(f"Found {len(results)} papers:")
|
||||
for paper in results:
|
||||
print(f"\n Title: {paper.get('title', 'N/A')[:60]}...")
|
||||
authors = paper.get('authors', [])
|
||||
if authors:
|
||||
print(f" Authors: {', '.join(authors[:2])}...")
|
||||
print(f" Published: {paper.get('published', 'N/A')}")
|
||||
print(f" PDF: {paper.get('pdf_url', 'N/A')}")
|
||||
|
||||
print("\n✅ ArXiv tool working correctly!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
"""Calculator Tool Example.
|
||||
|
||||
This example demonstrates how to use the Calculator tool with PraisonAI agents.
|
||||
|
||||
Requirements:
|
||||
pip install "praisonai[tools]"
|
||||
|
||||
Usage:
|
||||
python tool.py
|
||||
|
||||
Note: No API key required!
|
||||
"""
|
||||
|
||||
from praisonai_tools import CalculatorTool
|
||||
|
||||
|
||||
def main():
|
||||
# Initialize Calculator tool
|
||||
calc = CalculatorTool()
|
||||
|
||||
# Example 1: Basic arithmetic
|
||||
print("=" * 60)
|
||||
print("Example 1: Basic Arithmetic")
|
||||
print("=" * 60)
|
||||
|
||||
expressions = [
|
||||
"2 + 2",
|
||||
"10 * 5",
|
||||
"100 / 4",
|
||||
"2 ** 10",
|
||||
"(10 + 5) * 2 - 3"
|
||||
]
|
||||
|
||||
for expr in expressions:
|
||||
result = calc.evaluate(expr)
|
||||
print(f" {expr} = {result.get('result', result)}")
|
||||
|
||||
# Example 2: Percentage calculation
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 2: Percentage Calculation")
|
||||
print("=" * 60)
|
||||
|
||||
result = calc.evaluate("250 * 0.15")
|
||||
print(f" 15% of 250 = {result.get('result', result)}")
|
||||
|
||||
result = calc.evaluate("1000 * (1 + 0.05) ** 3")
|
||||
print(f" $1000 at 5% for 3 years = ${result.get('result', 0):.2f}")
|
||||
|
||||
print("\n✅ Calculator tool working correctly!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
"""DuckDuckGo Search Tool Example.
|
||||
|
||||
This example demonstrates how to use the DuckDuckGo search tool with PraisonAI agents.
|
||||
|
||||
Requirements:
|
||||
pip install "praisonai[tools]"
|
||||
|
||||
Usage:
|
||||
python tool.py
|
||||
|
||||
Note: No API key required!
|
||||
"""
|
||||
|
||||
from praisonai_tools import DuckDuckGoTool
|
||||
|
||||
|
||||
def main():
|
||||
# Initialize DuckDuckGo tool (no API key needed)
|
||||
ddg = DuckDuckGoTool()
|
||||
|
||||
# Example 1: Web search
|
||||
print("=" * 60)
|
||||
print("Example 1: Web Search")
|
||||
print("=" * 60)
|
||||
|
||||
results = ddg.search("Python programming best practices", max_results=3)
|
||||
|
||||
if results and "error" in results[0]:
|
||||
print(f"Error: {results[0]['error']}")
|
||||
else:
|
||||
print(f"Found {len(results)} results:")
|
||||
for r in results:
|
||||
print(f" - {r['title']}")
|
||||
print(f" URL: {r['url']}")
|
||||
print(f" Snippet: {r['snippet'][:100]}...")
|
||||
print()
|
||||
|
||||
# Example 2: News search
|
||||
print("=" * 60)
|
||||
print("Example 2: News Search")
|
||||
print("=" * 60)
|
||||
|
||||
news = ddg.news("artificial intelligence", max_results=3)
|
||||
|
||||
if news and "error" in news[0]:
|
||||
print(f"Error: {news[0]['error']}")
|
||||
else:
|
||||
print(f"Found {len(news)} news articles:")
|
||||
for n in news:
|
||||
print(f" - {n['title']}")
|
||||
print(f" Source: {n.get('source', 'N/A')}")
|
||||
print(f" Date: {n.get('date', 'N/A')}")
|
||||
print()
|
||||
|
||||
# Example 3: Image search
|
||||
print("=" * 60)
|
||||
print("Example 3: Image Search")
|
||||
print("=" * 60)
|
||||
|
||||
images = ddg.images("sunset landscape", max_results=3)
|
||||
|
||||
if images and "error" in images[0]:
|
||||
print(f"Error: {images[0]['error']}")
|
||||
else:
|
||||
print(f"Found {len(images)} images:")
|
||||
for img in images:
|
||||
print(f" - {img['title']}")
|
||||
print(f" Image URL: {img['image_url'][:60]}...")
|
||||
print()
|
||||
|
||||
print("✅ DuckDuckGo tool working correctly!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Vendored
+48
@@ -0,0 +1,48 @@
|
||||
"""Exa Search Tool Example.
|
||||
|
||||
This example demonstrates how to use the Exa search tool with PraisonAI agents.
|
||||
|
||||
Requirements:
|
||||
pip install "praisonai[tools]"
|
||||
export EXA_API_KEY=your_api_key
|
||||
|
||||
Usage:
|
||||
python tool.py
|
||||
"""
|
||||
|
||||
import os
|
||||
from praisonai_tools import ExaTool
|
||||
|
||||
|
||||
def main():
|
||||
# Check for API key
|
||||
if not os.getenv("EXA_API_KEY"):
|
||||
print("Error: EXA_API_KEY environment variable not set")
|
||||
print("Get your API key from https://exa.ai/")
|
||||
return
|
||||
|
||||
# Initialize Exa tool
|
||||
exa = ExaTool()
|
||||
|
||||
# Example 1: Neural search
|
||||
print("=" * 60)
|
||||
print("Example 1: Neural Search")
|
||||
print("=" * 60)
|
||||
|
||||
results = exa.search("best practices for Python development", num_results=3)
|
||||
|
||||
if results and "error" in results[0]:
|
||||
print(f"Error: {results[0]['error']}")
|
||||
else:
|
||||
print(f"Found {len(results)} results:")
|
||||
for r in results:
|
||||
print(f" - {r.get('title', 'N/A')[:50]}...")
|
||||
print(f" URL: {r.get('url', 'N/A')[:60]}...")
|
||||
print(f" Score: {r.get('score', 'N/A')}")
|
||||
print()
|
||||
|
||||
print("✅ Exa tool working correctly!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
"""Firecrawl Tool Example.
|
||||
|
||||
This example demonstrates how to use the Firecrawl tool with PraisonAI agents.
|
||||
|
||||
Requirements:
|
||||
pip install "praisonai[tools]"
|
||||
export FIRECRAWL_API_KEY=your_api_key
|
||||
|
||||
Usage:
|
||||
python tool.py
|
||||
"""
|
||||
|
||||
import os
|
||||
from praisonai_tools import FirecrawlTool
|
||||
|
||||
|
||||
def main():
|
||||
# Check for API key
|
||||
if not os.getenv("FIRECRAWL_API_KEY"):
|
||||
print("Error: FIRECRAWL_API_KEY environment variable not set")
|
||||
print("Get your API key from https://firecrawl.dev/")
|
||||
return
|
||||
|
||||
# Initialize Firecrawl tool
|
||||
firecrawl = FirecrawlTool()
|
||||
|
||||
# Example 1: Scrape a page
|
||||
print("=" * 60)
|
||||
print("Example 1: Scrape a Page")
|
||||
print("=" * 60)
|
||||
|
||||
result = firecrawl.scrape("https://example.com")
|
||||
|
||||
if "error" in result:
|
||||
print(f"Error: {result['error']}")
|
||||
else:
|
||||
print(f"URL: {result.get('url', 'N/A')}")
|
||||
markdown = result.get('markdown', '')
|
||||
print(f"Content preview: {markdown[:300]}...")
|
||||
|
||||
print("\n✅ Firecrawl tool working correctly!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
"""Hacker News Tool Example.
|
||||
|
||||
This example demonstrates how to use the Hacker News tool with PraisonAI agents.
|
||||
|
||||
Requirements:
|
||||
pip install "praisonai[tools]"
|
||||
|
||||
Usage:
|
||||
python tool.py
|
||||
|
||||
Note: No API key required!
|
||||
"""
|
||||
|
||||
from praisonai_tools import HackerNewsTool
|
||||
|
||||
|
||||
def main():
|
||||
# Initialize Hacker News tool
|
||||
hn = HackerNewsTool()
|
||||
|
||||
# Example 1: Get top stories
|
||||
print("=" * 60)
|
||||
print("Example 1: Top Stories")
|
||||
print("=" * 60)
|
||||
|
||||
stories = hn.get_top_stories(limit=5)
|
||||
|
||||
if stories and "error" in stories[0]:
|
||||
print(f"Error: {stories[0]['error']}")
|
||||
else:
|
||||
print(f"Found {len(stories)} top stories:")
|
||||
for story in stories:
|
||||
print(f" - {story.get('title', 'N/A')[:50]}...")
|
||||
print(f" Score: {story.get('score', 'N/A')} | Comments: {story.get('descendants', 'N/A')}")
|
||||
print(f" URL: {story.get('url', 'N/A')[:60]}...")
|
||||
print()
|
||||
|
||||
print("✅ Hacker News tool working correctly!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Vendored
+40
@@ -0,0 +1,40 @@
|
||||
"""Jina Reader Tool Example.
|
||||
|
||||
This example demonstrates how to use the Jina Reader tool with PraisonAI agents.
|
||||
|
||||
Requirements:
|
||||
pip install "praisonai[tools]"
|
||||
|
||||
Usage:
|
||||
python tool.py
|
||||
|
||||
Note: Works without API key (with rate limits)
|
||||
"""
|
||||
|
||||
from praisonai_tools import JinaTool
|
||||
|
||||
|
||||
def main():
|
||||
# Initialize Jina tool (works without API key)
|
||||
jina = JinaTool()
|
||||
|
||||
# Example 1: Read a URL
|
||||
print("=" * 60)
|
||||
print("Example 1: Read URL Content")
|
||||
print("=" * 60)
|
||||
|
||||
result = jina.read("https://example.com")
|
||||
|
||||
if "error" in result:
|
||||
print(f"Error: {result['error']}")
|
||||
else:
|
||||
print(f"URL: {result.get('url', 'N/A')}")
|
||||
print(f"Title: {result.get('title', 'N/A')}")
|
||||
content = result.get('content', '')
|
||||
print(f"Content preview: {content[:300]}...")
|
||||
|
||||
print("\n✅ Jina tool working correctly!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
"""PostgreSQL Tool Example.
|
||||
|
||||
This example demonstrates how to use the PostgreSQL tool with PraisonAI agents.
|
||||
|
||||
Requirements:
|
||||
pip install "praisonai[tools]"
|
||||
Docker: docker run -d --name postgres -e POSTGRES_PASSWORD=praison123 -e POSTGRES_DB=praisonai -p 5432:5432 postgres:16
|
||||
|
||||
Usage:
|
||||
python tool.py
|
||||
"""
|
||||
|
||||
from praisonai_tools import PostgresTool
|
||||
|
||||
|
||||
def main():
|
||||
# Initialize PostgreSQL tool
|
||||
pg = PostgresTool(
|
||||
host="localhost",
|
||||
port=5432,
|
||||
database="praisonai",
|
||||
user="postgres",
|
||||
password="praison123"
|
||||
)
|
||||
|
||||
# Example 1: List tables
|
||||
print("=" * 60)
|
||||
print("Example 1: List Tables")
|
||||
print("=" * 60)
|
||||
|
||||
tables = pg.list_tables()
|
||||
|
||||
if isinstance(tables, dict) and "error" in tables:
|
||||
print(f"Error: {tables['error']}")
|
||||
else:
|
||||
print(f"Found {len(tables)} tables:")
|
||||
for t in tables[:10]:
|
||||
print(f" - {t.get('table_name', t)}")
|
||||
|
||||
# Example 2: Query
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 2: Execute Query")
|
||||
print("=" * 60)
|
||||
|
||||
result = pg.query("SELECT version()")
|
||||
|
||||
if isinstance(result, dict) and "error" in result:
|
||||
print(f"Error: {result['error']}")
|
||||
else:
|
||||
print(f"PostgreSQL version: {result}")
|
||||
|
||||
print("\n✅ PostgreSQL tool working correctly!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
"""Serper Tool Example.
|
||||
|
||||
This example demonstrates how to use the Serper (Google Search) tool with PraisonAI agents.
|
||||
|
||||
Requirements:
|
||||
pip install "praisonai[tools]"
|
||||
export SERPER_API_KEY=your_api_key
|
||||
|
||||
Usage:
|
||||
python tool.py
|
||||
"""
|
||||
|
||||
import os
|
||||
from praisonai_tools import SerperTool
|
||||
|
||||
|
||||
def main():
|
||||
# Check for API key
|
||||
if not os.getenv("SERPER_API_KEY"):
|
||||
print("Error: SERPER_API_KEY environment variable not set")
|
||||
print("Get your API key from https://serper.dev/")
|
||||
return
|
||||
|
||||
# Initialize Serper tool
|
||||
serper = SerperTool()
|
||||
|
||||
# Example 1: Web search
|
||||
print("=" * 60)
|
||||
print("Example 1: Google Web Search")
|
||||
print("=" * 60)
|
||||
|
||||
results = serper.search("Python programming tutorials", max_results=3)
|
||||
|
||||
if results and "error" in results[0]:
|
||||
print(f"Error: {results[0]['error']}")
|
||||
else:
|
||||
print(f"Found {len(results)} results:")
|
||||
for r in results:
|
||||
print(f" - {r.get('title', 'N/A')[:50]}...")
|
||||
print(f" URL: {r.get('link', 'N/A')[:60]}...")
|
||||
print()
|
||||
|
||||
print("✅ Serper tool working correctly!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
"""Tavily Search Tool Example.
|
||||
|
||||
This example demonstrates how to use the Tavily search tool with PraisonAI agents.
|
||||
|
||||
Requirements:
|
||||
pip install "praisonai[tools]"
|
||||
export TAVILY_API_KEY=your_api_key
|
||||
|
||||
Usage:
|
||||
python tool.py
|
||||
"""
|
||||
|
||||
import os
|
||||
from praisonai_tools import TavilyTool
|
||||
|
||||
def main():
|
||||
# Check for API key
|
||||
if not os.getenv("TAVILY_API_KEY"):
|
||||
print("Error: TAVILY_API_KEY environment variable not set")
|
||||
print("Get your API key from https://tavily.com/")
|
||||
return
|
||||
|
||||
# Initialize Tavily tool
|
||||
tavily = TavilyTool()
|
||||
|
||||
# Example 1: Basic search
|
||||
print("=" * 60)
|
||||
print("Example 1: Basic Search")
|
||||
print("=" * 60)
|
||||
|
||||
results = tavily.search("What is quantum computing?", max_results=3)
|
||||
|
||||
if "error" in results:
|
||||
print(f"Error: {results['error']}")
|
||||
else:
|
||||
print(f"Query: {results.get('query')}")
|
||||
if "answer" in results:
|
||||
print(f"\nAI Answer: {results['answer'][:200]}...")
|
||||
print(f"\nResults ({len(results.get('results', []))}):")
|
||||
for r in results.get("results", []):
|
||||
print(f" - {r['title']}")
|
||||
print(f" URL: {r['url']}")
|
||||
print(f" Score: {r.get('score', 'N/A')}")
|
||||
|
||||
# Example 2: Search context for RAG
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 2: Search Context (for RAG)")
|
||||
print("=" * 60)
|
||||
|
||||
context = tavily.search_context("Python best practices 2024")
|
||||
print(f"Context (first 300 chars): {context[:300]}...")
|
||||
|
||||
print("\n✅ Tavily tool working correctly!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
"""Wikipedia Tool Example.
|
||||
|
||||
This example demonstrates how to use the Wikipedia tool with PraisonAI agents.
|
||||
|
||||
Requirements:
|
||||
pip install "praisonai[tools]"
|
||||
|
||||
Usage:
|
||||
python tool.py
|
||||
|
||||
Note: No API key required!
|
||||
"""
|
||||
|
||||
from praisonai_tools import WikipediaTool
|
||||
|
||||
|
||||
def main():
|
||||
# Initialize Wikipedia tool (no API key needed)
|
||||
wiki = WikipediaTool()
|
||||
|
||||
# Example 1: Search Wikipedia
|
||||
print("=" * 60)
|
||||
print("Example 1: Search Wikipedia")
|
||||
print("=" * 60)
|
||||
|
||||
results = wiki.search("Python programming", max_results=5)
|
||||
|
||||
if results and "error" in results[0]:
|
||||
print(f"Error: {results[0]['error']}")
|
||||
else:
|
||||
print(f"Found {len(results)} results:")
|
||||
for r in results:
|
||||
print(f" - {r['title']}")
|
||||
|
||||
# Example 2: Get page summary
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 2: Get Page Summary")
|
||||
print("=" * 60)
|
||||
|
||||
summary = wiki.summary("Artificial intelligence", sentences=3)
|
||||
|
||||
if "error" in summary:
|
||||
print(f"Error: {summary['error']}")
|
||||
else:
|
||||
print(f"Title: {summary['title']}")
|
||||
print(f"Summary: {summary['summary'][:300]}...")
|
||||
|
||||
# Example 3: Get full page
|
||||
print("\n" + "=" * 60)
|
||||
print("Example 3: Get Full Page")
|
||||
print("=" * 60)
|
||||
|
||||
page = wiki.get_page("Machine learning")
|
||||
|
||||
if "error" in page:
|
||||
print(f"Error: {page['error']}")
|
||||
if "options" in page:
|
||||
print(f"Options: {page['options'][:5]}")
|
||||
else:
|
||||
print(f"Title: {page['title']}")
|
||||
print(f"URL: {page['url']}")
|
||||
print(f"Categories: {page['categories'][:3]}")
|
||||
print(f"Content preview: {page['content'][:200]}...")
|
||||
|
||||
print("\n✅ Wikipedia tool working correctly!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,21 @@
|
||||
"""ChromaDB Vector Store - Basic Test"""
|
||||
import sys
|
||||
sys.path.insert(0, 'src/praisonai')
|
||||
from praisonai.persistence import create_knowledge_store
|
||||
from praisonai.persistence.knowledge.base import KnowledgeDocument
|
||||
|
||||
store = create_knowledge_store("chroma", path="/tmp/chroma_demo")
|
||||
try:
|
||||
store.delete_collection("demo")
|
||||
except Exception:
|
||||
pass
|
||||
store.create_collection("demo", dimension=1536)
|
||||
docs = [
|
||||
KnowledgeDocument(id="1", content="Python is a programming language", embedding=[0.1]*1536),
|
||||
KnowledgeDocument(id="2", content="JavaScript runs in browsers", embedding=[0.2]*1536)
|
||||
]
|
||||
store.insert("demo", docs)
|
||||
results = store.search("demo", query_embedding=[0.1]*1536, limit=1)
|
||||
print(f"Found: {len(results)} results")
|
||||
assert len(results) >= 1
|
||||
print("PASSED: ChromaDB vector store")
|
||||
@@ -0,0 +1,31 @@
|
||||
"""LanceDB Vector Store - Local Test"""
|
||||
import sys
|
||||
import os
|
||||
import shutil
|
||||
|
||||
try:
|
||||
import lancedb
|
||||
except ImportError:
|
||||
print("SKIPPED: LanceDB - lancedb not installed")
|
||||
sys.exit(0)
|
||||
|
||||
# Clean up previous test
|
||||
db_path = "/tmp/lancedb_test"
|
||||
if os.path.exists(db_path):
|
||||
shutil.rmtree(db_path)
|
||||
|
||||
db = lancedb.connect(db_path)
|
||||
|
||||
# Create table with vectors
|
||||
data = [
|
||||
{"id": "1", "text": "Machine learning is AI", "vector": [0.1] * 128},
|
||||
{"id": "2", "text": "Deep learning uses neural nets", "vector": [0.2] * 128},
|
||||
]
|
||||
table = db.create_table("demo", data)
|
||||
|
||||
# Search
|
||||
results = table.search([0.1] * 128).limit(1).to_list()
|
||||
print(f"Found: {len(results)} results")
|
||||
print(f"Top result: {results[0]['text'][:30]}...")
|
||||
assert len(results) >= 1
|
||||
print("PASSED: LanceDB vector store")
|
||||
@@ -0,0 +1,20 @@
|
||||
"""LanceDB Vector Store - Basic Test"""
|
||||
import sys
|
||||
sys.path.insert(0, 'src/praisonai')
|
||||
try:
|
||||
from praisonai.persistence import create_knowledge_store
|
||||
from praisonai.persistence.knowledge.base import KnowledgeDocument
|
||||
|
||||
store = create_knowledge_store("lancedb", path="/tmp/lancedb_demo")
|
||||
store.create_collection("demo", dimension=1536)
|
||||
docs = [
|
||||
KnowledgeDocument(id="doc1", content="The Eiffel Tower is in Paris", embedding=[0.1]*1536),
|
||||
KnowledgeDocument(id="doc2", content="The Statue of Liberty is in New York", embedding=[0.2]*1536)
|
||||
]
|
||||
store.insert("demo", docs)
|
||||
results = store.search("demo", query_embedding=[0.1]*1536, limit=1)
|
||||
print(f"Found: {len(results)} results")
|
||||
assert len(results) >= 1
|
||||
print("PASSED: LanceDB vector store")
|
||||
except ImportError as e:
|
||||
print(f"SKIPPED: LanceDB - {e}")
|
||||
@@ -0,0 +1,59 @@
|
||||
"""PGVector Vector Store - Docker Test"""
|
||||
import sys
|
||||
import os
|
||||
|
||||
# Docker: docker run -d --name pgvector -e POSTGRES_PASSWORD=postgres -p 5433:5432 pgvector/pgvector:pg16
|
||||
url = os.getenv("PGVECTOR_URL", "postgresql://postgres:postgres@localhost:5433/postgres")
|
||||
|
||||
try:
|
||||
import psycopg2
|
||||
except ImportError:
|
||||
print("SKIPPED: PGVector - psycopg2 not installed")
|
||||
sys.exit(0)
|
||||
|
||||
try:
|
||||
conn = psycopg2.connect(url)
|
||||
cur = conn.cursor()
|
||||
|
||||
# Ensure extension
|
||||
cur.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
||||
conn.commit()
|
||||
|
||||
# Create test table
|
||||
cur.execute("DROP TABLE IF EXISTS test_vectors")
|
||||
cur.execute("""
|
||||
CREATE TABLE test_vectors (
|
||||
id SERIAL PRIMARY KEY,
|
||||
content TEXT,
|
||||
embedding vector(128)
|
||||
)
|
||||
""")
|
||||
conn.commit()
|
||||
|
||||
# Insert vectors
|
||||
cur.execute("""
|
||||
INSERT INTO test_vectors (content, embedding) VALUES
|
||||
('Machine learning is AI', %s),
|
||||
('Deep learning uses neural nets', %s)
|
||||
""", ([0.1] * 128, [0.2] * 128))
|
||||
conn.commit()
|
||||
|
||||
# Search (cosine similarity)
|
||||
cur.execute("""
|
||||
SELECT content, embedding <=> %s::vector AS distance
|
||||
FROM test_vectors
|
||||
ORDER BY distance
|
||||
LIMIT 1
|
||||
""", ([0.1] * 128,))
|
||||
|
||||
row = cur.fetchone()
|
||||
print(f"Top result: {row[0][:30]}... (distance: {row[1]:.4f})")
|
||||
|
||||
# Cleanup
|
||||
cur.execute("DROP TABLE test_vectors")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
print("PASSED: PGVector vector store")
|
||||
except Exception as e:
|
||||
print(f"SKIPPED: PGVector - {e}")
|
||||
@@ -0,0 +1,23 @@
|
||||
"""PGVector Knowledge Store - Basic Test (requires Docker)"""
|
||||
import sys
|
||||
import os
|
||||
sys.path.insert(0, 'src/praisonai')
|
||||
from praisonai.persistence import create_knowledge_store
|
||||
from praisonai.persistence.knowledge.base import KnowledgeDocument
|
||||
|
||||
# Docker: docker run -d --name pgvector -e POSTGRES_PASSWORD=postgres -p 5433:5432 pgvector/pgvector:pg16
|
||||
url = os.getenv("PGVECTOR_URL", "postgresql://postgres:postgres@localhost:5433/postgres")
|
||||
try:
|
||||
store = create_knowledge_store("pgvector", url=url)
|
||||
store.create_collection("demo", dimension=1536)
|
||||
docs = [
|
||||
KnowledgeDocument(id="1", content="Kubernetes orchestrates containers", embedding=[0.1]*1536),
|
||||
KnowledgeDocument(id="2", content="Docker packages applications", embedding=[0.2]*1536)
|
||||
]
|
||||
store.insert("demo", docs)
|
||||
results = store.search("demo", query_embedding=[0.1]*1536, limit=1)
|
||||
print(f"Found: {len(results)} results")
|
||||
assert len(results) >= 1
|
||||
print("PASSED: PGVector vector store")
|
||||
except Exception as e:
|
||||
print(f"SKIPPED: PGVector - {e}")
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Pinecone Vector Store - Real API Test"""
|
||||
import os
|
||||
import sys
|
||||
sys.path.insert(0, 'src/praisonai')
|
||||
|
||||
# Requires: export PINECONE_API_KEY=...
|
||||
api_key = os.getenv("PINECONE_API_KEY")
|
||||
if not api_key:
|
||||
print("SKIPPED: Pinecone - PINECONE_API_KEY not set")
|
||||
sys.exit(0)
|
||||
|
||||
from pinecone import Pinecone
|
||||
|
||||
pc = Pinecone(api_key=api_key)
|
||||
indexes = pc.list_indexes()
|
||||
print(f"Pinecone connected! Found {len(list(indexes))} indexes")
|
||||
|
||||
# Use existing 'test' index or create one
|
||||
index_name = "test"
|
||||
if index_name not in [idx.name for idx in pc.list_indexes()]:
|
||||
print(f"SKIPPED: Pinecone - index '{index_name}' not found")
|
||||
sys.exit(0)
|
||||
|
||||
index = pc.Index(index_name)
|
||||
stats = index.describe_index_stats()
|
||||
print(f"Index '{index_name}' has {stats.total_vector_count} vectors")
|
||||
print("PASSED: Pinecone vector store")
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Qdrant Vector Store - Basic Test (requires Docker)"""
|
||||
import sys, os
|
||||
sys.path.insert(0, 'src/praisonai')
|
||||
from praisonai.persistence import create_knowledge_store
|
||||
from praisonai.persistence.knowledge.base import KnowledgeDocument
|
||||
|
||||
# Docker: docker run -d --name qdrant -p 6333:6333 qdrant/qdrant
|
||||
url = os.getenv("QDRANT_URL", "http://localhost:6333")
|
||||
store = create_knowledge_store("qdrant", url=url)
|
||||
try:
|
||||
store.delete_collection("demo")
|
||||
except Exception:
|
||||
pass
|
||||
store.create_collection("demo", dimension=1536)
|
||||
docs = [
|
||||
KnowledgeDocument(id="1", content="Machine learning is AI", embedding=[0.1]*1536),
|
||||
KnowledgeDocument(id="2", content="Deep learning uses neural nets", embedding=[0.2]*1536)
|
||||
]
|
||||
store.insert("demo", docs)
|
||||
results = store.search("demo", query_embedding=[0.1]*1536, limit=1)
|
||||
print(f"Found: {len(results)} results")
|
||||
assert len(results) >= 1
|
||||
print("PASSED: Qdrant vector store")
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Weaviate Cloud Vector Store - Real API Test"""
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Requires: export WEAVIATE_URL=... and WEAVIATE_API_KEY=...
|
||||
weaviate_url = os.getenv("WEAVIATE_URL", "https://thhoyczshsawyt16ot3uw.c0.europe-west3.gcp.weaviate.cloud")
|
||||
weaviate_key = os.getenv("WEAVIATE_API_KEY")
|
||||
if not weaviate_key:
|
||||
print("SKIPPED: Weaviate - WEAVIATE_API_KEY not set")
|
||||
sys.exit(0)
|
||||
|
||||
import weaviate
|
||||
from weaviate.classes.init import Auth
|
||||
|
||||
client = weaviate.connect_to_weaviate_cloud(
|
||||
cluster_url=weaviate_url,
|
||||
auth_credentials=Auth.api_key(weaviate_key)
|
||||
)
|
||||
|
||||
print(f"Weaviate connected: {client.is_ready()}")
|
||||
collections = client.collections.list_all()
|
||||
print(f"Found {len(collections)} collections")
|
||||
client.close()
|
||||
print("PASSED: Weaviate cloud vector store")
|
||||
@@ -54,6 +54,8 @@ except ImportError:
|
||||
MCP = None
|
||||
from .session import Session
|
||||
from .memory.memory import Memory
|
||||
from .db import db
|
||||
from .obs import obs
|
||||
# Workflows - import from dedicated workflows module
|
||||
from .workflows import (
|
||||
Workflow, WorkflowStep, WorkflowContext, StepResult,
|
||||
@@ -257,6 +259,8 @@ __all__ = [
|
||||
'AutoAgents',
|
||||
'Session',
|
||||
'Memory',
|
||||
'db',
|
||||
'obs',
|
||||
'display_interaction',
|
||||
'display_self_reflection',
|
||||
'display_instruction',
|
||||
|
||||
@@ -252,7 +252,9 @@ class Agent:
|
||||
history_in_context: Optional[int] = None,
|
||||
auto_save: Optional[str] = None,
|
||||
skills: Optional[List[str]] = None,
|
||||
skills_dirs: Optional[List[str]] = None
|
||||
skills_dirs: Optional[List[str]] = None,
|
||||
db: Optional[Any] = None,
|
||||
session_id: Optional[str] = None
|
||||
):
|
||||
"""Initialize an Agent instance.
|
||||
|
||||
@@ -615,6 +617,11 @@ Your Goal: {self.goal}
|
||||
self._skill_manager = None # Lazy loaded
|
||||
self._skills_initialized = False
|
||||
|
||||
# Database persistence (lazy - no imports until used)
|
||||
self._db = db
|
||||
self._session_id = session_id
|
||||
self._db_initialized = False
|
||||
|
||||
@property
|
||||
def console(self):
|
||||
"""Lazily initialize Rich Console only when needed."""
|
||||
@@ -1957,7 +1964,108 @@ Your Goal: {self.goal}"""
|
||||
# expand=False
|
||||
# )
|
||||
|
||||
def _init_db_session(self):
|
||||
"""Initialize DB session if db adapter is provided (lazy, first chat only)."""
|
||||
if self._db is None or self._db_initialized:
|
||||
return
|
||||
|
||||
# Generate session_id if not provided: default to per-hour ID (YYYYMMDDHH-agentname)
|
||||
if self._session_id is None:
|
||||
import hashlib
|
||||
from datetime import datetime, timezone
|
||||
# Per-hour session ID: YYYYMMDDHH (UTC) + agent name hash for uniqueness
|
||||
hour_str = datetime.now(timezone.utc).strftime("%Y%m%d%H")
|
||||
agent_hash = hashlib.md5(self.name.encode()).hexdigest()[:6]
|
||||
self._session_id = f"{hour_str}-{agent_hash}"
|
||||
|
||||
# Call db adapter's on_agent_start to get previous messages
|
||||
try:
|
||||
history = self._db.on_agent_start(
|
||||
agent_name=self.name,
|
||||
session_id=self._session_id,
|
||||
user_id=self.user_id,
|
||||
metadata={"role": self.role, "goal": self.goal}
|
||||
)
|
||||
|
||||
# Restore chat history from previous session
|
||||
if history:
|
||||
for msg in history:
|
||||
self.chat_history.append({
|
||||
"role": msg.role,
|
||||
"content": msg.content
|
||||
})
|
||||
logging.info(f"Resumed session {self._session_id} with {len(history)} messages")
|
||||
except Exception as e:
|
||||
logging.warning(f"Failed to initialize DB session: {e}")
|
||||
|
||||
self._db_initialized = True
|
||||
self._current_run_id = None # Track current run
|
||||
|
||||
def _start_run(self, input_content: str):
|
||||
"""Start a new run (turn) for persistence tracking."""
|
||||
if self._db is None:
|
||||
return
|
||||
|
||||
import uuid
|
||||
self._current_run_id = f"run-{uuid.uuid4().hex[:12]}"
|
||||
|
||||
try:
|
||||
if hasattr(self._db, 'on_run_start'):
|
||||
self._db.on_run_start(
|
||||
session_id=self._session_id,
|
||||
run_id=self._current_run_id,
|
||||
input_content=input_content,
|
||||
metadata={"agent_name": self.name}
|
||||
)
|
||||
except Exception as e:
|
||||
logging.warning(f"Failed to start run: {e}")
|
||||
|
||||
def _end_run(self, output_content: str, status: str = "completed", metrics: dict = None):
|
||||
"""End the current run (turn)."""
|
||||
if self._db is None or self._current_run_id is None:
|
||||
return
|
||||
|
||||
try:
|
||||
if hasattr(self._db, 'on_run_end'):
|
||||
self._db.on_run_end(
|
||||
session_id=self._session_id,
|
||||
run_id=self._current_run_id,
|
||||
output_content=output_content,
|
||||
status=status,
|
||||
metrics=metrics or {},
|
||||
metadata={"agent_name": self.name}
|
||||
)
|
||||
except Exception as e:
|
||||
logging.warning(f"Failed to end run: {e}")
|
||||
|
||||
self._current_run_id = None
|
||||
|
||||
def _persist_message(self, role: str, content: str):
|
||||
"""Persist a message to the DB if adapter is provided."""
|
||||
if self._db is None:
|
||||
return
|
||||
|
||||
try:
|
||||
if role == "user":
|
||||
self._db.on_user_message(self._session_id, content)
|
||||
elif role == "assistant":
|
||||
self._db.on_agent_message(self._session_id, content)
|
||||
except Exception as e:
|
||||
logging.warning(f"Failed to persist message: {e}")
|
||||
|
||||
@property
|
||||
def session_id(self) -> Optional[str]:
|
||||
"""Get the current session ID."""
|
||||
return self._session_id
|
||||
|
||||
def chat(self, prompt, temperature=1.0, tools=None, output_json=None, output_pydantic=None, reasoning_steps=False, stream=None, task_name=None, task_description=None, task_id=None):
|
||||
# Initialize DB session on first chat (lazy)
|
||||
self._init_db_session()
|
||||
|
||||
# Start a new run for this chat turn
|
||||
prompt_str = prompt if isinstance(prompt, str) else str(prompt)
|
||||
self._start_run(prompt_str)
|
||||
|
||||
# Reset the final display flag for each new conversation
|
||||
self._final_display_shown = False
|
||||
|
||||
@@ -2054,6 +2162,8 @@ Your Goal: {self.goal}"""
|
||||
self.chat_history[-1].get("content") == normalized_content):
|
||||
# Add user message to chat history BEFORE LLM call so handoffs can access it
|
||||
self.chat_history.append({"role": "user", "content": normalized_content})
|
||||
# Persist user message to DB
|
||||
self._persist_message("user", normalized_content)
|
||||
|
||||
try:
|
||||
# Pass everything to LLM class
|
||||
@@ -2083,6 +2193,8 @@ Your Goal: {self.goal}"""
|
||||
)
|
||||
|
||||
self.chat_history.append({"role": "assistant", "content": response_text})
|
||||
# Persist assistant message to DB
|
||||
self._persist_message("assistant", response_text)
|
||||
|
||||
# Log completion time if in debug mode
|
||||
if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
|
||||
@@ -2127,6 +2239,8 @@ Your Goal: {self.goal}"""
|
||||
self.chat_history[-1].get("content") == normalized_content):
|
||||
# Add user message to chat history BEFORE LLM call so handoffs can access it
|
||||
self.chat_history.append({"role": "user", "content": normalized_content})
|
||||
# Persist user message to DB (OpenAI path)
|
||||
self._persist_message("user", normalized_content)
|
||||
|
||||
reflection_count = 0
|
||||
start_time = time.time()
|
||||
@@ -2166,6 +2280,8 @@ Your Goal: {self.goal}"""
|
||||
# Add to chat history and return raw response
|
||||
# User message already added before LLM call via _build_messages
|
||||
self.chat_history.append({"role": "assistant", "content": response_text})
|
||||
# Persist assistant message to DB
|
||||
self._persist_message("assistant", response_text)
|
||||
# Apply guardrail validation even for JSON output
|
||||
try:
|
||||
validated_response = self._apply_guardrail_with_retry(response_text, original_prompt, temperature, tools, task_name, task_description, task_id)
|
||||
@@ -2181,6 +2297,8 @@ Your Goal: {self.goal}"""
|
||||
if not self.self_reflect:
|
||||
# User message already added before LLM call via _build_messages
|
||||
self.chat_history.append({"role": "assistant", "content": response_text})
|
||||
# Persist assistant message to DB (non-reflect path)
|
||||
self._persist_message("assistant", response_text)
|
||||
if self.verbose:
|
||||
logging.debug(f"Agent {self.name} final response: {response_text}")
|
||||
# Return only reasoning content if reasoning_steps is True
|
||||
@@ -2269,11 +2387,13 @@ Output MUST be JSON with 'reflection' and 'satisfactory'.
|
||||
validated_response = self._apply_guardrail_with_retry(response_text, original_prompt, temperature, tools, task_name, task_description, task_id)
|
||||
# Execute callback after validation
|
||||
self._execute_callback_and_display(original_prompt, validated_response, time.time() - start_time, task_name, task_description, task_id)
|
||||
self._end_run(validated_response, "completed", {"duration_ms": (time.time() - start_time) * 1000})
|
||||
return validated_response
|
||||
except Exception as e:
|
||||
logging.error(f"Agent {self.name}: Guardrail validation failed after reflection: {e}")
|
||||
# Rollback chat history on guardrail failure
|
||||
self.chat_history = self.chat_history[:chat_history_length]
|
||||
self._end_run(None, "error", {"error": str(e)})
|
||||
return None
|
||||
|
||||
# Check if we've hit max reflections
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
"""
|
||||
Observability interface for PraisonAI Agents.
|
||||
|
||||
This module provides the protocol and types for observability/tracing.
|
||||
Implementations are provided by the wrapper layer (praisonai_tools.observability).
|
||||
|
||||
Usage (simplest - recommended):
|
||||
from praisonaiagents import Agent, obs
|
||||
|
||||
agent = Agent(
|
||||
name="Assistant",
|
||||
observability=obs.auto(), # Auto-detect from env vars
|
||||
)
|
||||
agent.chat("Hello!") # Auto-traces to configured provider
|
||||
|
||||
Alternative (explicit provider):
|
||||
observability=obs.langfuse() # Langfuse
|
||||
observability=obs.langsmith() # LangSmith
|
||||
observability=obs.agentops() # AgentOps
|
||||
observability=obs.arize() # Arize Phoenix
|
||||
observability=obs.datadog() # Datadog
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional, Protocol, runtime_checkable
|
||||
from dataclasses import dataclass, field
|
||||
import time
|
||||
|
||||
|
||||
@dataclass
|
||||
class Span:
|
||||
"""Represents a span in a trace."""
|
||||
span_id: str
|
||||
trace_id: str
|
||||
name: str
|
||||
parent_span_id: Optional[str] = None
|
||||
start_time: float = field(default_factory=time.time)
|
||||
end_time: Optional[float] = None
|
||||
attributes: Dict[str, Any] = field(default_factory=dict)
|
||||
status: str = "ok"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Trace:
|
||||
"""Represents a complete trace."""
|
||||
trace_id: str
|
||||
session_id: Optional[str] = None
|
||||
agent_name: Optional[str] = None
|
||||
user_id: Optional[str] = None
|
||||
start_time: float = field(default_factory=time.time)
|
||||
end_time: Optional[float] = None
|
||||
spans: List[Span] = field(default_factory=list)
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
status: str = "ok"
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class ObservabilityAdapter(Protocol):
|
||||
"""Protocol for observability adapters."""
|
||||
|
||||
def on_trace_start(
|
||||
self,
|
||||
trace_id: str,
|
||||
session_id: Optional[str] = None,
|
||||
agent_name: Optional[str] = None,
|
||||
user_id: Optional[str] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
) -> None:
|
||||
"""Called when a trace starts."""
|
||||
...
|
||||
|
||||
def on_trace_end(
|
||||
self,
|
||||
trace_id: str,
|
||||
status: str = "ok",
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
) -> None:
|
||||
"""Called when a trace ends."""
|
||||
...
|
||||
|
||||
def on_span_start(
|
||||
self,
|
||||
span_id: str,
|
||||
trace_id: str,
|
||||
name: str,
|
||||
parent_span_id: Optional[str] = None,
|
||||
attributes: Optional[Dict[str, Any]] = None,
|
||||
) -> None:
|
||||
"""Called when a span starts."""
|
||||
...
|
||||
|
||||
def on_span_end(
|
||||
self,
|
||||
span_id: str,
|
||||
status: str = "ok",
|
||||
attributes: Optional[Dict[str, Any]] = None,
|
||||
) -> None:
|
||||
"""Called when a span ends."""
|
||||
...
|
||||
|
||||
def on_llm_call(
|
||||
self,
|
||||
span_id: str,
|
||||
model: str,
|
||||
messages: List[Dict[str, Any]],
|
||||
response: Optional[str] = None,
|
||||
tokens: Optional[Dict[str, int]] = None,
|
||||
latency_ms: Optional[float] = None,
|
||||
) -> None:
|
||||
"""Called when an LLM call is made."""
|
||||
...
|
||||
|
||||
def on_tool_call(
|
||||
self,
|
||||
span_id: str,
|
||||
tool_name: str,
|
||||
args: Dict[str, Any],
|
||||
result: Any,
|
||||
latency_ms: Optional[float] = None,
|
||||
) -> None:
|
||||
"""Called when a tool is invoked."""
|
||||
...
|
||||
|
||||
def flush(self) -> None:
|
||||
"""Flush any pending data."""
|
||||
...
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the adapter and release resources."""
|
||||
...
|
||||
|
||||
|
||||
class _LazyObsModule:
|
||||
"""
|
||||
Lazy proxy for observability backends.
|
||||
|
||||
Allows `from praisonaiagents import obs` without importing heavy deps.
|
||||
Actual provider classes are loaded only when accessed.
|
||||
"""
|
||||
|
||||
_PROVIDERS = {
|
||||
"langfuse": "praisonai_tools.observability.providers.langfuse_provider",
|
||||
"langsmith": "praisonai_tools.observability.providers.langsmith_provider",
|
||||
"agentops": "praisonai_tools.observability.providers.agentops_provider",
|
||||
"arize": "praisonai_tools.observability.providers.arize_phoenix_provider",
|
||||
"phoenix": "praisonai_tools.observability.providers.arize_phoenix_provider",
|
||||
"datadog": "praisonai_tools.observability.providers.datadog_provider",
|
||||
"mlflow": "praisonai_tools.observability.providers.mlflow_provider",
|
||||
"openlit": "praisonai_tools.observability.providers.openlit_provider",
|
||||
"opik": "praisonai_tools.observability.providers.opik_provider",
|
||||
"portkey": "praisonai_tools.observability.providers.portkey_provider",
|
||||
"traceloop": "praisonai_tools.observability.providers.traceloop_provider",
|
||||
"weave": "praisonai_tools.observability.providers.weave_provider",
|
||||
"braintrust": "praisonai_tools.observability.providers.braintrust_provider",
|
||||
"langtrace": "praisonai_tools.observability.providers.langtrace_provider",
|
||||
"langwatch": "praisonai_tools.observability.providers.langwatch_provider",
|
||||
"maxim": "praisonai_tools.observability.providers.maxim_provider",
|
||||
}
|
||||
|
||||
def __getattr__(self, name: str):
|
||||
if name in self._PROVIDERS:
|
||||
return self._create_provider_factory(name)
|
||||
raise AttributeError(f"module 'obs' has no attribute {name!r}")
|
||||
|
||||
def _create_provider_factory(self, provider_name: str):
|
||||
"""Create a factory function for a provider."""
|
||||
def factory(**kwargs):
|
||||
try:
|
||||
import importlib
|
||||
module = importlib.import_module(self._PROVIDERS[provider_name])
|
||||
# Get the provider class (naming convention: XxxProvider)
|
||||
class_name = f"{provider_name.title().replace('_', '')}Provider"
|
||||
provider_cls = getattr(module, class_name, None)
|
||||
if provider_cls:
|
||||
return provider_cls(**kwargs)
|
||||
# Fallback: try to find any class ending with Provider
|
||||
for attr_name in dir(module):
|
||||
if attr_name.endswith("Provider"):
|
||||
return getattr(module, attr_name)(**kwargs)
|
||||
raise AttributeError(f"No provider class found in {module}")
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
f"Observability provider '{provider_name}' requires praisonai-tools. "
|
||||
f"Install with: pip install praisonai-tools\n"
|
||||
f"Original error: {e}"
|
||||
) from e
|
||||
return factory
|
||||
|
||||
def auto(self, **kwargs):
|
||||
"""
|
||||
Auto-detect observability provider from environment variables.
|
||||
|
||||
Checks for common env vars like LANGFUSE_PUBLIC_KEY, LANGSMITH_API_KEY, etc.
|
||||
Returns None if no provider is detected.
|
||||
"""
|
||||
import os
|
||||
|
||||
# Check for common provider env vars
|
||||
provider_env_map = {
|
||||
"LANGFUSE_PUBLIC_KEY": "langfuse",
|
||||
"LANGSMITH_API_KEY": "langsmith",
|
||||
"AGENTOPS_API_KEY": "agentops",
|
||||
"ARIZE_API_KEY": "arize",
|
||||
"DATADOG_API_KEY": "datadog",
|
||||
"MLFLOW_TRACKING_URI": "mlflow",
|
||||
"OPENLIT_API_KEY": "openlit",
|
||||
"OPIK_API_KEY": "opik",
|
||||
"PORTKEY_API_KEY": "portkey",
|
||||
"TRACELOOP_API_KEY": "traceloop",
|
||||
"WANDB_API_KEY": "weave",
|
||||
"BRAINTRUST_API_KEY": "braintrust",
|
||||
}
|
||||
|
||||
for env_var, provider in provider_env_map.items():
|
||||
if os.getenv(env_var):
|
||||
return self._create_provider_factory(provider)(**kwargs)
|
||||
|
||||
return None
|
||||
|
||||
def __repr__(self):
|
||||
return "<module 'praisonaiagents.obs' (lazy observability backends)>"
|
||||
|
||||
def __call__(self, provider: str = "auto", **kwargs):
|
||||
"""
|
||||
Shortcut: obs("langfuse") or obs() for auto-detection.
|
||||
"""
|
||||
if provider == "auto":
|
||||
return self.auto(**kwargs)
|
||||
return self._create_provider_factory(provider)(**kwargs)
|
||||
|
||||
|
||||
# Singleton lazy module instance
|
||||
obs = _LazyObsModule()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ObservabilityAdapter",
|
||||
"Span",
|
||||
"Trace",
|
||||
"obs",
|
||||
]
|
||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "praisonaiagents"
|
||||
version = "0.2.0"
|
||||
version = "0.2.2"
|
||||
description = "Praison AI agents for completing complex tasks with Self Reflection Agents"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
|
||||
Generated
+1
-1
@@ -2422,7 +2422,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "praisonaiagents"
|
||||
version = "0.2.0"
|
||||
version = "0.2.2"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "aiohttp" },
|
||||
|
||||
+287
-1
@@ -261,6 +261,12 @@ npm install praisonai
|
||||
| ↳ 100+ LLM Support | [Example](examples/python/providers/openai/openai_gpt4_example.py) | [📖](https://docs.praison.ai/models) |
|
||||
| ↳ Callback Agents | [Example](examples/python/general/advanced-callback-systems.py) | [📖](https://docs.praison.ai/features/callbacks) |
|
||||
| ↳ Hooks | [Example](#9-hooks) | [📖](https://docs.praison.ai/features/hooks) |
|
||||
| ↳ Shadow Git Checkpoints | [Example](#10-shadow-git-checkpoints) | [📖](https://docs.praison.ai/features/checkpoints) |
|
||||
| ↳ Background Tasks | [Example](examples/background/basic_background.py) | [📖](https://docs.praison.ai/features/background-tasks) |
|
||||
| ↳ Policy Engine | [Example](examples/policy/basic_policy.py) | [📖](https://docs.praison.ai/features/policy-engine) |
|
||||
| ↳ Thinking Budgets | [Example](examples/thinking/basic_thinking.py) | [📖](https://docs.praison.ai/features/thinking-budgets) |
|
||||
| ↳ Output Styles | [Example](examples/output/basic_output.py) | [📖](https://docs.praison.ai/features/output-styles) |
|
||||
| ↳ Context Compaction | [Example](examples/compaction/basic_compaction.py) | [📖](https://docs.praison.ai/features/context-compaction) |
|
||||
| **📊 Monitoring & Management** | | |
|
||||
| ↳ Sessions Management | [Example](examples/python/sessions/comprehensive-session-management.py) | [📖](https://docs.praison.ai/features/sessions) |
|
||||
| ↳ Auto-Save Sessions | [Example](#session-management-python) | [📖](https://docs.praison.ai/docs/cli/session) |
|
||||
@@ -930,8 +936,191 @@ praisonai checkpoint diff # Show changes
|
||||
praisonai checkpoint restore abc123 # Restore to checkpoint
|
||||
```
|
||||
|
||||
**Links:**
|
||||
- [📖 Coding Docs](https://docs.praison.ai/features/checkpoints)
|
||||
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/checkpoint)
|
||||
- [💻 Example](examples/checkpoints/basic_checkpoints.py)
|
||||
|
||||
### 11. Field Names Reference (A-I-G-S)
|
||||
---
|
||||
|
||||
### 11. Background Tasks
|
||||
|
||||
Run agent tasks asynchronously without blocking:
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
from praisonaiagents.background import BackgroundRunner, BackgroundConfig
|
||||
|
||||
async def main():
|
||||
config = BackgroundConfig(max_concurrent_tasks=3)
|
||||
runner = BackgroundRunner(config=config)
|
||||
|
||||
async def my_task(name: str) -> str:
|
||||
await asyncio.sleep(2)
|
||||
return f"Task {name} completed"
|
||||
|
||||
task = await runner.submit(my_task, args=("example",), name="my_task")
|
||||
await task.wait(timeout=10.0)
|
||||
print(task.result)
|
||||
|
||||
asyncio.run(main())
|
||||
```
|
||||
|
||||
**CLI Commands:**
|
||||
```bash
|
||||
praisonai background list # List running tasks
|
||||
praisonai background status <id> # Check task status
|
||||
praisonai background cancel <id> # Cancel a task
|
||||
praisonai background clear # Clear completed tasks
|
||||
```
|
||||
|
||||
**Links:**
|
||||
- [📖 Coding Docs](https://docs.praison.ai/features/background-tasks)
|
||||
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/background)
|
||||
- [💻 Example](examples/background/basic_background.py)
|
||||
|
||||
---
|
||||
|
||||
### 12. Policy Engine
|
||||
|
||||
Control what agents can and cannot do with policy-based execution:
|
||||
|
||||
```python
|
||||
from praisonaiagents.policy import (
|
||||
PolicyEngine, Policy, PolicyRule, PolicyAction
|
||||
)
|
||||
|
||||
engine = PolicyEngine()
|
||||
|
||||
policy = Policy(
|
||||
name="no_delete",
|
||||
rules=[
|
||||
PolicyRule(
|
||||
action=PolicyAction.DENY,
|
||||
resource="tool:delete_*",
|
||||
reason="Delete operations blocked"
|
||||
)
|
||||
]
|
||||
)
|
||||
engine.add_policy(policy)
|
||||
|
||||
result = engine.check("tool:delete_file", {})
|
||||
print(f"Allowed: {result.allowed}")
|
||||
```
|
||||
|
||||
**CLI Commands:**
|
||||
```bash
|
||||
praisonai policy list # List policies
|
||||
praisonai policy check "tool:name" # Check if allowed
|
||||
praisonai policy init # Create template
|
||||
```
|
||||
|
||||
**Links:**
|
||||
- [📖 Coding Docs](https://docs.praison.ai/features/policy-engine)
|
||||
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/policy)
|
||||
- [💻 Example](examples/policy/basic_policy.py)
|
||||
|
||||
---
|
||||
|
||||
### 13. Thinking Budgets
|
||||
|
||||
Configure token budgets for extended thinking:
|
||||
|
||||
```python
|
||||
from praisonaiagents.thinking import ThinkingBudget, ThinkingTracker
|
||||
|
||||
# Use predefined levels
|
||||
budget = ThinkingBudget.high() # 16,000 tokens
|
||||
|
||||
# Track usage
|
||||
tracker = ThinkingTracker()
|
||||
session = tracker.start_session(budget_tokens=16000)
|
||||
tracker.end_session(session, tokens_used=12000)
|
||||
|
||||
summary = tracker.get_summary()
|
||||
print(f"Utilization: {summary['average_utilization']:.1%}")
|
||||
```
|
||||
|
||||
**CLI Commands:**
|
||||
```bash
|
||||
praisonai thinking status # Show current budget
|
||||
praisonai thinking set high # Set budget level
|
||||
praisonai thinking stats # Show usage statistics
|
||||
```
|
||||
|
||||
**Links:**
|
||||
- [📖 Coding Docs](https://docs.praison.ai/features/thinking-budgets)
|
||||
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/thinking)
|
||||
- [💻 Example](examples/thinking/basic_thinking.py)
|
||||
|
||||
---
|
||||
|
||||
### 14. Output Styles
|
||||
|
||||
Configure how agents format their responses:
|
||||
|
||||
```python
|
||||
from praisonaiagents.output import OutputStyle, OutputFormatter
|
||||
|
||||
# Use preset styles
|
||||
style = OutputStyle.concise()
|
||||
formatter = OutputFormatter(style)
|
||||
|
||||
# Format output
|
||||
text = "# Hello\n\nThis is **bold** text."
|
||||
plain = formatter.format(text)
|
||||
print(plain)
|
||||
```
|
||||
|
||||
**CLI Commands:**
|
||||
```bash
|
||||
praisonai output status # Show current style
|
||||
praisonai output set concise # Set output style
|
||||
```
|
||||
|
||||
**Links:**
|
||||
- [📖 Coding Docs](https://docs.praison.ai/features/output-styles)
|
||||
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/output-style)
|
||||
- [💻 Example](examples/output/basic_output.py)
|
||||
|
||||
---
|
||||
|
||||
### 15. Context Compaction
|
||||
|
||||
Automatically manage context window size:
|
||||
|
||||
```python
|
||||
from praisonaiagents.compaction import (
|
||||
ContextCompactor, CompactionStrategy
|
||||
)
|
||||
|
||||
compactor = ContextCompactor(
|
||||
max_tokens=4000,
|
||||
strategy=CompactionStrategy.SLIDING,
|
||||
preserve_recent=3
|
||||
)
|
||||
|
||||
messages = [...] # Your conversation history
|
||||
compacted, result = compactor.compact(messages)
|
||||
|
||||
print(f"Compression: {result.compression_ratio:.1%}")
|
||||
```
|
||||
|
||||
**CLI Commands:**
|
||||
```bash
|
||||
praisonai compaction status # Show settings
|
||||
praisonai compaction set sliding # Set strategy
|
||||
praisonai compaction stats # Show statistics
|
||||
```
|
||||
|
||||
**Links:**
|
||||
- [📖 Coding Docs](https://docs.praison.ai/features/context-compaction)
|
||||
- [📖 CLI Docs](https://docs.praison.ai/docs/cli/compaction)
|
||||
- [💻 Example](examples/compaction/basic_compaction.py)
|
||||
|
||||
---
|
||||
|
||||
### 16. Field Names Reference (A-I-G-S)
|
||||
|
||||
PraisonAI accepts both old (agents.yaml) and new (workflow.yaml) field names. Use the **canonical names** for new projects:
|
||||
|
||||
@@ -2535,6 +2724,103 @@ PraisonAI provides zero-dependency persistent memory for agents. For detailed ex
|
||||
|
||||
---
|
||||
|
||||
## 💾 Persistence (Databases)
|
||||
|
||||
Enable automatic conversation persistence with 2 lines of code:
|
||||
|
||||
```python
|
||||
from praisonaiagents import Agent, db
|
||||
|
||||
agent = Agent(
|
||||
name="Assistant",
|
||||
db=db(database_url="postgresql://localhost/mydb"), # db(...) shortcut
|
||||
session_id="my-session" # Optional: defaults to per-hour ID (YYYYMMDDHH)
|
||||
)
|
||||
agent.chat("Hello!") # Auto-persists messages, runs, traces
|
||||
```
|
||||
|
||||
### Persistence CLI Commands
|
||||
|
||||
| Command | Description |
|
||||
|---------|-------------|
|
||||
| `praisonai persistence doctor` | Validate DB connectivity |
|
||||
| `praisonai persistence run` | Run agent with persistence |
|
||||
| `praisonai persistence resume` | Resume existing session |
|
||||
| `praisonai persistence export` | Export session to JSONL |
|
||||
| `praisonai persistence import` | Import session from JSONL |
|
||||
| `praisonai persistence migrate` | Apply schema migrations |
|
||||
| `praisonai persistence status` | Show schema status |
|
||||
|
||||
### Databases Table
|
||||
|
||||
| Database | Store Type | Install | Example | Docs |
|
||||
|----------|------------|---------|---------|------|
|
||||
| PostgreSQL | Conversation | `pip install "praisonai[tools]"` | [simple_db_agent.py](examples/persistence/simple_db_agent.py) | [docs](https://docs.praison.ai/docs/databases/postgres) |
|
||||
| MySQL | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| SQLite | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| SingleStore | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| Supabase | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| SurrealDB | Conversation | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| Qdrant | Knowledge | `pip install "praisonai[tools]"` | [knowledge_qdrant.py](examples/persistence/knowledge_qdrant.py) | [docs](https://docs.praison.ai/docs/databases/qdrant) |
|
||||
| ChromaDB | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| Pinecone | Knowledge | `pip install pinecone` | [pinecone_wow.py](examples/vector/pinecone_wow.py) | [docs](https://docs.praison.ai/docs/databases/pinecone) |
|
||||
| Weaviate | Knowledge | `pip install weaviate-client` | [weaviate_wow.py](examples/vector/weaviate_wow.py) | [docs](https://docs.praison.ai/docs/databases/weaviate) |
|
||||
| LanceDB | Knowledge | `pip install lancedb` | [lancedb_real_wow.py](examples/vector/lancedb_real_wow.py) | [docs](https://docs.praison.ai/docs/databases/lancedb) |
|
||||
| Milvus | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| PGVector | Knowledge | `pip install psycopg2-binary` | [pgvector_real_wow.py](examples/vector/pgvector_real_wow.py) | [docs](https://docs.praison.ai/docs/databases/pgvector) |
|
||||
| Redis Vector | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| Cassandra | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| ClickHouse | Knowledge | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| Redis | State | `pip install "praisonai[tools]"` | [state_redis.py](examples/persistence/state_redis.py) | [docs](https://docs.praison.ai/docs/databases/redis) |
|
||||
| MongoDB | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| DynamoDB | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| Firestore | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| Upstash | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
| Memory | State | `pip install "praisonai[tools]"` | - | [docs](https://docs.praison.ai/docs/databases/overview) |
|
||||
|
||||
---
|
||||
|
||||
## 🔧 Tools Table
|
||||
|
||||
Install all tools with: `pip install "praisonai[tools]"`
|
||||
|
||||
| Tool | Category | Import | Docs |
|
||||
|------|----------|--------|------|
|
||||
| Tavily | Web Search | `from praisonai_tools import TavilyTool` | [docs](https://docs.praison.ai/docs/tools/external/tavily) |
|
||||
| DuckDuckGo | Web Search | `from praisonai_tools import DuckDuckGoTool` | [docs](https://docs.praison.ai/docs/tools/external/duckduckgo) |
|
||||
| Exa | Web Search | `from praisonai_tools import ExaTool` | [docs](https://docs.praison.ai/docs/tools/external/exa) |
|
||||
| Serper | Web Search | `from praisonai_tools import SerperTool` | [docs](https://docs.praison.ai/docs/tools/external/serper) |
|
||||
| Jina | Web Reader | `from praisonai_tools import JinaTool` | [docs](https://docs.praison.ai/docs/tools/external/jina) |
|
||||
| Firecrawl | Web Scraping | `from praisonai_tools import FirecrawlTool` | [docs](https://docs.praison.ai/docs/tools/external/firecrawl) |
|
||||
| Crawl4AI | Web Scraping | `from praisonai_tools import Crawl4AITool` | [docs](https://docs.praison.ai/docs/tools/external/crawl4ai) |
|
||||
| Wikipedia | Knowledge | `from praisonai_tools import WikipediaTool` | [docs](https://docs.praison.ai/docs/tools/external/wikipedia) |
|
||||
| ArXiv | Research | `from praisonai_tools import ArxivTool` | [docs](https://docs.praison.ai/docs/tools/external/arxiv) |
|
||||
| HackerNews | News | `from praisonai_tools import HackerNewsTool` | [docs](https://docs.praison.ai/docs/tools/external/hackernews) |
|
||||
| YouTube | Media | `from praisonai_tools import YouTubeTool` | [docs](https://docs.praison.ai/docs/tools/external/youtube) |
|
||||
| Weather | Data | `from praisonai_tools import WeatherTool` | [docs](https://docs.praison.ai/docs/tools/external/weather) |
|
||||
| PostgreSQL | Database | `from praisonai_tools import PostgresTool` | [docs](https://docs.praison.ai/docs/tools/external/postgres) |
|
||||
| MySQL | Database | `from praisonai_tools import MySQLTool` | [docs](https://docs.praison.ai/docs/tools/external/mysql) |
|
||||
| SQLite | Database | `from praisonai_tools import SQLiteTool` | [docs](https://docs.praison.ai/docs/tools/external/sqlite) |
|
||||
| MongoDB | Database | `from praisonai_tools import MongoDBTool` | [docs](https://docs.praison.ai/docs/tools/external/mongodb) |
|
||||
| Redis | Database | `from praisonai_tools import RedisTool` | [docs](https://docs.praison.ai/docs/tools/external/redis) |
|
||||
| Qdrant | Vector DB | `from praisonai_tools import QdrantTool` | [docs](https://docs.praison.ai/docs/tools/external/qdrant) |
|
||||
| GitHub | DevOps | `from praisonai_tools import GitHubTool` | [docs](https://docs.praison.ai/docs/tools/external/github) |
|
||||
| Slack | Communication | `from praisonai_tools import SlackTool` | [docs](https://docs.praison.ai/docs/tools/external/slack) |
|
||||
| Discord | Communication | `from praisonai_tools import DiscordTool` | [docs](https://docs.praison.ai/docs/tools/external/discord) |
|
||||
| Telegram | Communication | `from praisonai_tools import TelegramTool` | [docs](https://docs.praison.ai/docs/tools/external/telegram) |
|
||||
| Email | Communication | `from praisonai_tools import EmailTool` | [docs](https://docs.praison.ai/docs/tools/external/email) |
|
||||
| Notion | Productivity | `from praisonai_tools import NotionTool` | [docs](https://docs.praison.ai/docs/tools/external/notion) |
|
||||
| File | File System | `from praisonai_tools import FileTool` | [docs](https://docs.praison.ai/docs/tools/external/file) |
|
||||
| Shell | System | `from praisonai_tools import ShellTool` | [docs](https://docs.praison.ai/docs/tools/external/shell) |
|
||||
| Python | Code | `from praisonai_tools import PythonTool` | [docs](https://docs.praison.ai/docs/tools/external/python) |
|
||||
| JSON | Data | `from praisonai_tools import JSONTool` | [docs](https://docs.praison.ai/docs/tools/external/json) |
|
||||
| CSV | Data | `from praisonai_tools import CSVTool` | [docs](https://docs.praison.ai/docs/tools/external/csv) |
|
||||
| Calculator | Math | `from praisonai_tools import CalculatorTool` | [docs](https://docs.praison.ai/docs/tools/external/calculator) |
|
||||
|
||||
> See [full tools documentation](https://docs.praison.ai/docs/tools/tools) for all 100+ available tools.
|
||||
|
||||
---
|
||||
|
||||
## 🎓 Video Tutorials
|
||||
|
||||
Learn PraisonAI through our comprehensive video series:
|
||||
|
||||
@@ -3,8 +3,8 @@ class Praisonai < Formula
|
||||
|
||||
desc "AI tools for various AI applications"
|
||||
homepage "https://github.com/MervinPraison/PraisonAI"
|
||||
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.4.0.tar.gz"
|
||||
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.4.0.tar.gz | shasum -a 256`.split.first
|
||||
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.4.4.tar.gz"
|
||||
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.4.4.tar.gz | shasum -a 256`.split.first
|
||||
license "MIT"
|
||||
|
||||
depends_on "[email protected]"
|
||||
|
||||
@@ -0,0 +1,293 @@
|
||||
"""
|
||||
Observability CLI Handler
|
||||
|
||||
CLI commands for managing observability providers.
|
||||
Usage: praisonai obs <command>
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from .base import FlagHandler
|
||||
|
||||
|
||||
class ObservabilityHandler(FlagHandler):
|
||||
"""
|
||||
Handler for observability CLI commands.
|
||||
|
||||
Commands:
|
||||
praisonai obs list - List available providers
|
||||
praisonai obs doctor - Check provider connectivity
|
||||
praisonai obs init <name> - Initialize a provider
|
||||
"""
|
||||
|
||||
@property
|
||||
def feature_name(self) -> str:
|
||||
return "observability"
|
||||
|
||||
@property
|
||||
def flag_name(self) -> str:
|
||||
return "obs"
|
||||
|
||||
@property
|
||||
def flag_help(self) -> str:
|
||||
return "Observability commands (list, doctor, init)"
|
||||
|
||||
def check_dependencies(self) -> Tuple[bool, str]:
|
||||
"""Check if observability module is available."""
|
||||
try:
|
||||
from praisonai_tools.observability import obs
|
||||
return True, ""
|
||||
except ImportError:
|
||||
return False, "Observability requires praisonai-tools. Install with: pip install praisonai-tools"
|
||||
|
||||
def list_providers(self) -> Dict[str, Any]:
|
||||
"""List all available observability providers."""
|
||||
result = {
|
||||
"registered": [],
|
||||
"available": [],
|
||||
"configured": [],
|
||||
}
|
||||
|
||||
try:
|
||||
from praisonai_tools.observability.manager import ObservabilityManager
|
||||
from praisonai_tools.observability.config import PROVIDER_ENV_KEYS
|
||||
|
||||
mgr = ObservabilityManager()
|
||||
|
||||
# Get all registered providers
|
||||
result["registered"] = mgr.list_providers()
|
||||
|
||||
# Check which have env vars configured
|
||||
for provider, keys in PROVIDER_ENV_KEYS.items():
|
||||
if all(os.getenv(key) for key in keys):
|
||||
result["configured"].append(provider)
|
||||
|
||||
# Check which SDKs are available
|
||||
for provider in result["registered"]:
|
||||
try:
|
||||
mgr._load_provider(provider)
|
||||
if provider in mgr._providers:
|
||||
provider_class = mgr._providers[provider]
|
||||
instance = provider_class()
|
||||
if instance.is_available():
|
||||
result["available"].append(provider)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
def doctor(self, provider: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""Run diagnostics on observability setup."""
|
||||
try:
|
||||
from praisonai_tools.observability import obs
|
||||
|
||||
if provider:
|
||||
obs.init(provider=provider)
|
||||
|
||||
return obs.doctor()
|
||||
|
||||
except ImportError:
|
||||
return {
|
||||
"error": "praisonai-tools not installed",
|
||||
"enabled": False,
|
||||
}
|
||||
|
||||
def init_provider(self, provider: str, **kwargs) -> bool:
|
||||
"""Initialize a specific provider."""
|
||||
try:
|
||||
from praisonai_tools.observability import obs
|
||||
return obs.init(provider=provider, **kwargs)
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
def execute(self, action: str = "list", **kwargs) -> Any:
|
||||
"""
|
||||
Execute observability command.
|
||||
|
||||
Args:
|
||||
action: Command to execute (list, doctor, init)
|
||||
**kwargs: Additional arguments
|
||||
|
||||
Returns:
|
||||
Command result
|
||||
"""
|
||||
if action == "list":
|
||||
return self._execute_list()
|
||||
elif action == "doctor":
|
||||
return self._execute_doctor(kwargs.get("provider"))
|
||||
elif action == "init":
|
||||
return self._execute_init(kwargs.get("provider"), **kwargs)
|
||||
else:
|
||||
self.print_status(f"Unknown action: {action}", "error")
|
||||
return None
|
||||
|
||||
def _execute_list(self) -> Dict[str, Any]:
|
||||
"""Execute list command."""
|
||||
result = self.list_providers()
|
||||
|
||||
self.print_status("\n📊 Observability Providers", "info")
|
||||
self.print_status("=" * 40, "info")
|
||||
|
||||
# Provider status table
|
||||
all_providers = [
|
||||
"agentops", "langfuse", "langsmith", "traceloop",
|
||||
"arize_phoenix", "openlit", "langtrace", "langwatch",
|
||||
"datadog", "mlflow", "opik", "portkey",
|
||||
"braintrust", "maxim", "weave", "neatlogs",
|
||||
"langdb", "atla", "patronus", "truefoundry",
|
||||
]
|
||||
|
||||
for provider in all_providers:
|
||||
status_parts = []
|
||||
if provider in result.get("available", []):
|
||||
status_parts.append("✅ SDK")
|
||||
else:
|
||||
status_parts.append("❌ SDK")
|
||||
|
||||
if provider in result.get("configured", []):
|
||||
status_parts.append("🔑 Keys")
|
||||
else:
|
||||
status_parts.append("⚪ Keys")
|
||||
|
||||
status = " | ".join(status_parts)
|
||||
self.print_status(f" {provider:20} {status}", "info")
|
||||
|
||||
self.print_status("\n✅ = Available ❌ = Not installed 🔑 = API key configured", "info")
|
||||
|
||||
return result
|
||||
|
||||
def _execute_doctor(self, provider: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""Execute doctor command."""
|
||||
result = self.doctor(provider)
|
||||
|
||||
self.print_status("\n🩺 Observability Diagnostics", "info")
|
||||
self.print_status("=" * 40, "info")
|
||||
|
||||
self.print_status(f" Enabled: {result.get('enabled', False)}", "info")
|
||||
self.print_status(f" Provider: {result.get('provider', 'None')}", "info")
|
||||
|
||||
if result.get("connection_status") is not None:
|
||||
status = "✅" if result["connection_status"] else "❌"
|
||||
self.print_status(f" Connection: {status} {result.get('connection_message', '')}", "info")
|
||||
|
||||
if result.get("available_providers"):
|
||||
self.print_status(f" Available: {', '.join(result['available_providers'])}", "info")
|
||||
|
||||
return result
|
||||
|
||||
def _execute_init(self, provider: str, **kwargs) -> bool:
|
||||
"""Execute init command."""
|
||||
if not provider:
|
||||
self.print_status("Provider name required. Usage: praisonai obs init <provider>", "error")
|
||||
return False
|
||||
|
||||
success = self.init_provider(provider, **kwargs)
|
||||
|
||||
if success:
|
||||
self.print_status(f"✅ {provider} initialized successfully", "success")
|
||||
else:
|
||||
self.print_status(f"❌ Failed to initialize {provider}", "error")
|
||||
self.print_status(f" Check that {provider} SDK is installed and API keys are set", "info")
|
||||
|
||||
return success
|
||||
|
||||
def apply_to_agent_config(self, config: Dict[str, Any], flag_value: Any) -> Dict[str, Any]:
|
||||
"""Apply observability to agent config."""
|
||||
if flag_value:
|
||||
try:
|
||||
from praisonai_tools.observability import obs
|
||||
|
||||
if isinstance(flag_value, str):
|
||||
obs.init(provider=flag_value)
|
||||
else:
|
||||
obs.init()
|
||||
|
||||
self.print_status("📊 Observability enabled", "success")
|
||||
except ImportError:
|
||||
self.print_status("Observability requires praisonai-tools", "warning")
|
||||
|
||||
return config
|
||||
|
||||
|
||||
# Provider environment variable reference
|
||||
PROVIDER_SETUP_GUIDE = {
|
||||
"agentops": {
|
||||
"env_vars": ["AGENTOPS_API_KEY"],
|
||||
"install": "pip install agentops",
|
||||
"docs": "https://agentops.ai/",
|
||||
},
|
||||
"langfuse": {
|
||||
"env_vars": ["LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY"],
|
||||
"install": "pip install opentelemetry-sdk opentelemetry-exporter-otlp",
|
||||
"docs": "https://langfuse.com/",
|
||||
},
|
||||
"langsmith": {
|
||||
"env_vars": ["LANGSMITH_API_KEY"],
|
||||
"install": "pip install opentelemetry-sdk opentelemetry-exporter-otlp",
|
||||
"docs": "https://smith.langchain.com/",
|
||||
},
|
||||
"traceloop": {
|
||||
"env_vars": ["TRACELOOP_API_KEY"],
|
||||
"install": "pip install traceloop-sdk",
|
||||
"docs": "https://traceloop.com/",
|
||||
},
|
||||
"arize_phoenix": {
|
||||
"env_vars": ["PHOENIX_API_KEY"],
|
||||
"install": "pip install arize-phoenix openinference-instrumentation",
|
||||
"docs": "https://phoenix.arize.com/",
|
||||
},
|
||||
"openlit": {
|
||||
"env_vars": [],
|
||||
"install": "pip install openlit",
|
||||
"docs": "https://github.com/openlit/openlit",
|
||||
},
|
||||
"langtrace": {
|
||||
"env_vars": ["LANGTRACE_API_KEY"],
|
||||
"install": "pip install langtrace-python-sdk",
|
||||
"docs": "https://langtrace.ai/",
|
||||
},
|
||||
"langwatch": {
|
||||
"env_vars": ["LANGWATCH_API_KEY"],
|
||||
"install": "pip install langwatch",
|
||||
"docs": "https://langwatch.ai/",
|
||||
},
|
||||
"datadog": {
|
||||
"env_vars": ["DD_API_KEY"],
|
||||
"install": "pip install ddtrace",
|
||||
"docs": "https://www.datadoghq.com/product/llm-observability/",
|
||||
},
|
||||
"mlflow": {
|
||||
"env_vars": ["MLFLOW_TRACKING_URI"],
|
||||
"install": "pip install mlflow",
|
||||
"docs": "https://mlflow.org/",
|
||||
},
|
||||
"opik": {
|
||||
"env_vars": ["OPIK_API_KEY"],
|
||||
"install": "pip install opik",
|
||||
"docs": "https://www.comet.com/docs/opik/",
|
||||
},
|
||||
"portkey": {
|
||||
"env_vars": ["PORTKEY_API_KEY"],
|
||||
"install": "pip install portkey-ai",
|
||||
"docs": "https://portkey.ai/",
|
||||
},
|
||||
"braintrust": {
|
||||
"env_vars": ["BRAINTRUST_API_KEY"],
|
||||
"install": "pip install braintrust",
|
||||
"docs": "https://www.braintrust.dev/",
|
||||
},
|
||||
"maxim": {
|
||||
"env_vars": ["MAXIM_API_KEY"],
|
||||
"install": "pip install maxim-py",
|
||||
"docs": "https://getmaxim.ai/",
|
||||
},
|
||||
"weave": {
|
||||
"env_vars": ["WANDB_API_KEY"],
|
||||
"install": "pip install weave",
|
||||
"docs": "https://weave-docs.wandb.ai/",
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,637 @@
|
||||
"""
|
||||
Persistence CLI feature for PraisonAI.
|
||||
|
||||
Provides commands for database persistence configuration and management:
|
||||
- praisonai persistence doctor - Validate DB connectivity
|
||||
- praisonai persistence run - Run agent with persistence
|
||||
- praisonai persistence resume - Resume a session
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def handle_persistence_command(args: list) -> bool:
|
||||
"""
|
||||
Handle persistence subcommands.
|
||||
|
||||
Args:
|
||||
args: Command line arguments after 'persistence'
|
||||
|
||||
Returns:
|
||||
True if command was handled, False otherwise
|
||||
"""
|
||||
if not args:
|
||||
print_persistence_help()
|
||||
return True
|
||||
|
||||
subcommand = args[0].lower()
|
||||
sub_args = args[1:]
|
||||
|
||||
if subcommand == "doctor":
|
||||
return handle_doctor(sub_args)
|
||||
elif subcommand == "run":
|
||||
return handle_run(sub_args)
|
||||
elif subcommand == "resume":
|
||||
return handle_resume(sub_args)
|
||||
elif subcommand == "export":
|
||||
return handle_export(sub_args)
|
||||
elif subcommand == "import":
|
||||
return handle_import(sub_args)
|
||||
elif subcommand == "migrate":
|
||||
return handle_migrate(sub_args)
|
||||
elif subcommand == "status":
|
||||
return handle_status(sub_args)
|
||||
elif subcommand in ("--help", "-h", "help"):
|
||||
print_persistence_help()
|
||||
return True
|
||||
else:
|
||||
print(f"Unknown persistence subcommand: {subcommand}")
|
||||
print_persistence_help()
|
||||
return True
|
||||
|
||||
|
||||
def print_persistence_help():
|
||||
"""Print persistence command help."""
|
||||
help_text = """
|
||||
PraisonAI Persistence Commands
|
||||
==============================
|
||||
|
||||
Usage: praisonai persistence <command> [options]
|
||||
|
||||
Commands:
|
||||
doctor Validate database connectivity and configuration
|
||||
run Run an agent with persistence enabled
|
||||
resume Resume an existing session
|
||||
|
||||
Doctor Command:
|
||||
praisonai persistence doctor [options]
|
||||
|
||||
Options:
|
||||
--conversation-url URL Conversation store URL (postgres/mysql/sqlite)
|
||||
--knowledge-url URL Knowledge store URL (qdrant/chroma/pinecone)
|
||||
--state-url URL State store URL (redis/memory)
|
||||
--all Test all configured stores
|
||||
|
||||
Environment Variables:
|
||||
PRAISON_CONVERSATION_URL Default conversation store URL
|
||||
PRAISON_KNOWLEDGE_URL Default knowledge store URL
|
||||
PRAISON_STATE_URL Default state store URL
|
||||
|
||||
Run Command:
|
||||
praisonai persistence run [options] "prompt"
|
||||
|
||||
Options:
|
||||
--session-id ID Session identifier (auto-generated if not provided)
|
||||
--user-id ID User identifier (default: "default")
|
||||
--conversation-url URL Conversation store URL
|
||||
--knowledge-url URL Knowledge store URL
|
||||
--state-url URL State store URL
|
||||
--agent-name NAME Agent name (default: "Assistant")
|
||||
--agent-instructions TEXT Agent instructions
|
||||
--dry-run Show configuration without running
|
||||
|
||||
Resume Command:
|
||||
praisonai persistence resume --session-id ID [options]
|
||||
|
||||
Options:
|
||||
--session-id ID Session to resume (required)
|
||||
--conversation-url URL Conversation store URL
|
||||
--show-history Display conversation history
|
||||
--continue "prompt" Continue with a new prompt
|
||||
|
||||
Export/Import Commands:
|
||||
praisonai persistence export --session-id ID [options]
|
||||
praisonai persistence import --file FILE [options]
|
||||
|
||||
Options:
|
||||
--session-id ID Session to export (required for export)
|
||||
--conversation-url URL Conversation store URL
|
||||
--file FILE File path for export/import (JSONL format)
|
||||
--output FILE Output file path (default: session-{id}.jsonl)
|
||||
|
||||
Examples:
|
||||
# Check PostgreSQL connectivity
|
||||
praisonai persistence doctor --conversation-url "postgresql://localhost/mydb"
|
||||
|
||||
# Run agent with persistence
|
||||
praisonai persistence run --session-id "my-session" "Hello, remember my name is Alice"
|
||||
|
||||
# Resume a session
|
||||
praisonai persistence resume --session-id "my-session" --show-history
|
||||
|
||||
# Continue a session
|
||||
praisonai persistence resume --session-id "my-session" --continue "What's my name?"
|
||||
|
||||
# Export a session
|
||||
praisonai persistence export --session-id "my-session" --output session.jsonl
|
||||
|
||||
# Import a session
|
||||
praisonai persistence import --file session.jsonl
|
||||
"""
|
||||
print(help_text)
|
||||
|
||||
|
||||
def parse_persistence_args(args: list) -> dict:
|
||||
"""Parse persistence command arguments."""
|
||||
config = {
|
||||
"session_id": None,
|
||||
"user_id": "default",
|
||||
"conversation_url": os.getenv("PRAISON_CONVERSATION_URL"),
|
||||
"knowledge_url": os.getenv("PRAISON_KNOWLEDGE_URL"),
|
||||
"state_url": os.getenv("PRAISON_STATE_URL"),
|
||||
"agent_name": "Assistant",
|
||||
"agent_instructions": "You are a helpful assistant.",
|
||||
"dry_run": False,
|
||||
"show_history": False,
|
||||
"all_stores": False,
|
||||
"prompt": None,
|
||||
"continue_prompt": None,
|
||||
"output_file": None,
|
||||
"input_file": None,
|
||||
}
|
||||
|
||||
i = 0
|
||||
while i < len(args):
|
||||
arg = args[i]
|
||||
|
||||
if arg == "--session-id" and i + 1 < len(args):
|
||||
config["session_id"] = args[i + 1]
|
||||
i += 2
|
||||
elif arg == "--user-id" and i + 1 < len(args):
|
||||
config["user_id"] = args[i + 1]
|
||||
i += 2
|
||||
elif arg == "--conversation-url" and i + 1 < len(args):
|
||||
config["conversation_url"] = args[i + 1]
|
||||
i += 2
|
||||
elif arg == "--knowledge-url" and i + 1 < len(args):
|
||||
config["knowledge_url"] = args[i + 1]
|
||||
i += 2
|
||||
elif arg == "--state-url" and i + 1 < len(args):
|
||||
config["state_url"] = args[i + 1]
|
||||
i += 2
|
||||
elif arg == "--agent-name" and i + 1 < len(args):
|
||||
config["agent_name"] = args[i + 1]
|
||||
i += 2
|
||||
elif arg == "--agent-instructions" and i + 1 < len(args):
|
||||
config["agent_instructions"] = args[i + 1]
|
||||
i += 2
|
||||
elif arg == "--dry-run":
|
||||
config["dry_run"] = True
|
||||
i += 1
|
||||
elif arg == "--show-history":
|
||||
config["show_history"] = True
|
||||
i += 1
|
||||
elif arg == "--all":
|
||||
config["all_stores"] = True
|
||||
i += 1
|
||||
elif arg == "--continue" and i + 1 < len(args):
|
||||
config["continue_prompt"] = args[i + 1]
|
||||
i += 2
|
||||
elif arg == "--output" and i + 1 < len(args):
|
||||
config["output_file"] = args[i + 1]
|
||||
i += 2
|
||||
elif arg == "--file" and i + 1 < len(args):
|
||||
config["input_file"] = args[i + 1]
|
||||
i += 2
|
||||
elif not arg.startswith("-"):
|
||||
config["prompt"] = arg
|
||||
i += 1
|
||||
else:
|
||||
i += 1
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def handle_doctor(args: list) -> bool:
|
||||
"""Handle persistence doctor command."""
|
||||
config = parse_persistence_args(args)
|
||||
|
||||
print("=" * 50)
|
||||
print("PraisonAI Persistence Doctor")
|
||||
print("=" * 50)
|
||||
|
||||
results = []
|
||||
|
||||
# Test conversation store
|
||||
if config["conversation_url"] or config["all_stores"]:
|
||||
url = config["conversation_url"]
|
||||
if url:
|
||||
print(f"\n[Conversation Store] Testing: {_mask_url(url)}")
|
||||
success, msg = _test_conversation_store(url)
|
||||
results.append(("Conversation", success, msg))
|
||||
print(f" {'✅' if success else '❌'} {msg}")
|
||||
else:
|
||||
print("\n[Conversation Store] Not configured (set --conversation-url or PRAISON_CONVERSATION_URL)")
|
||||
|
||||
# Test knowledge store
|
||||
if config["knowledge_url"] or config["all_stores"]:
|
||||
url = config["knowledge_url"]
|
||||
if url:
|
||||
print(f"\n[Knowledge Store] Testing: {_mask_url(url)}")
|
||||
success, msg = _test_knowledge_store(url)
|
||||
results.append(("Knowledge", success, msg))
|
||||
print(f" {'✅' if success else '❌'} {msg}")
|
||||
else:
|
||||
print("\n[Knowledge Store] Not configured (set --knowledge-url or PRAISON_KNOWLEDGE_URL)")
|
||||
|
||||
# Test state store
|
||||
if config["state_url"] or config["all_stores"]:
|
||||
url = config["state_url"]
|
||||
if url:
|
||||
print(f"\n[State Store] Testing: {_mask_url(url)}")
|
||||
success, msg = _test_state_store(url)
|
||||
results.append(("State", success, msg))
|
||||
print(f" {'✅' if success else '❌'} {msg}")
|
||||
else:
|
||||
print("\n[State Store] Not configured (set --state-url or PRAISON_STATE_URL)")
|
||||
|
||||
# Summary
|
||||
print("\n" + "=" * 50)
|
||||
if results:
|
||||
passed = sum(1 for _, s, _ in results if s)
|
||||
total = len(results)
|
||||
print(f"Results: {passed}/{total} stores connected successfully")
|
||||
else:
|
||||
print("No stores configured. Use --conversation-url, --knowledge-url, or --state-url")
|
||||
print("=" * 50)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def handle_run(args: list) -> bool:
|
||||
"""Handle persistence run command."""
|
||||
config = parse_persistence_args(args)
|
||||
|
||||
if not config["prompt"]:
|
||||
print("Error: No prompt provided")
|
||||
print("Usage: praisonai persistence run [options] \"your prompt\"")
|
||||
return True
|
||||
|
||||
if not config["conversation_url"]:
|
||||
print("Error: No conversation URL configured")
|
||||
print("Set --conversation-url or PRAISON_CONVERSATION_URL")
|
||||
return True
|
||||
|
||||
if config["dry_run"]:
|
||||
print("=" * 50)
|
||||
print("Dry Run - Configuration")
|
||||
print("=" * 50)
|
||||
print(f"Session ID: {config['session_id'] or '(auto-generated)'}")
|
||||
print(f"User ID: {config['user_id']}")
|
||||
print(f"Conversation URL: {_mask_url(config['conversation_url'])}")
|
||||
print(f"Agent Name: {config['agent_name']}")
|
||||
print(f"Prompt: {config['prompt'][:50]}...")
|
||||
return True
|
||||
|
||||
# Run agent with persistence
|
||||
try:
|
||||
from praisonai.db import PraisonDB
|
||||
from praisonaiagents import Agent
|
||||
|
||||
db = PraisonDB(
|
||||
database_url=config["conversation_url"],
|
||||
state_url=config["state_url"],
|
||||
knowledge_url=config["knowledge_url"],
|
||||
)
|
||||
|
||||
agent = Agent(
|
||||
name=config["agent_name"],
|
||||
instructions=config["agent_instructions"],
|
||||
db=db,
|
||||
session_id=config["session_id"],
|
||||
verbose=True
|
||||
)
|
||||
|
||||
print(f"\n[Session: {agent.session_id}]")
|
||||
response = agent.chat(config["prompt"])
|
||||
print(f"\nAgent: {response}")
|
||||
|
||||
db.close()
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
return True
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def handle_resume(args: list) -> bool:
|
||||
"""Handle persistence resume command."""
|
||||
config = parse_persistence_args(args)
|
||||
|
||||
if not config["session_id"]:
|
||||
print("Error: --session-id is required for resume")
|
||||
return True
|
||||
|
||||
if not config["conversation_url"]:
|
||||
print("Error: No conversation URL configured")
|
||||
print("Set --conversation-url or PRAISON_CONVERSATION_URL")
|
||||
return True
|
||||
|
||||
try:
|
||||
from praisonai.db import PraisonDB
|
||||
|
||||
db = PraisonDB(database_url=config["conversation_url"])
|
||||
|
||||
# Get session history
|
||||
history = db.on_agent_start(
|
||||
agent_name="Resume",
|
||||
session_id=config["session_id"],
|
||||
user_id=config["user_id"]
|
||||
)
|
||||
|
||||
print(f"\n[Session: {config['session_id']}]")
|
||||
print(f"Messages in history: {len(history)}")
|
||||
|
||||
if config["show_history"]:
|
||||
print("\n--- Conversation History ---")
|
||||
for msg in history:
|
||||
role = msg.role.upper()
|
||||
content = msg.content[:100] + "..." if len(msg.content) > 100 else msg.content
|
||||
print(f"[{role}] {content}")
|
||||
print("--- End History ---\n")
|
||||
|
||||
if config["continue_prompt"]:
|
||||
from praisonaiagents import Agent
|
||||
|
||||
agent = Agent(
|
||||
name=config["agent_name"],
|
||||
instructions=config["agent_instructions"],
|
||||
db=db,
|
||||
session_id=config["session_id"],
|
||||
verbose=True
|
||||
)
|
||||
|
||||
response = agent.chat(config["continue_prompt"])
|
||||
print(f"\nAgent: {response}")
|
||||
|
||||
db.close()
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
return True
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def handle_export(args: list) -> bool:
|
||||
"""Handle persistence export command."""
|
||||
config = parse_persistence_args(args)
|
||||
|
||||
if not config["session_id"]:
|
||||
print("Error: --session-id is required for export")
|
||||
return True
|
||||
|
||||
if not config["conversation_url"]:
|
||||
print("Error: No conversation URL configured")
|
||||
return True
|
||||
|
||||
try:
|
||||
from praisonai.db import PraisonDB
|
||||
import json
|
||||
|
||||
db = PraisonDB(database_url=config["conversation_url"])
|
||||
|
||||
# Export session
|
||||
data = db.export_session(config["session_id"])
|
||||
|
||||
if not data:
|
||||
print(f"Error: Session {config['session_id']} not found")
|
||||
db.close()
|
||||
return True
|
||||
|
||||
# Determine output file
|
||||
output_file = config.get("output_file") or f"session-{config['session_id']}.jsonl"
|
||||
|
||||
with open(output_file, 'w') as f:
|
||||
f.write(json.dumps(data, default=str) + '\n')
|
||||
|
||||
print(f"Exported session to: {output_file}")
|
||||
print(f"Messages: {len(data.get('messages', []))}")
|
||||
|
||||
db.close()
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
return True
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def handle_import(args: list) -> bool:
|
||||
"""Handle persistence import command."""
|
||||
config = parse_persistence_args(args)
|
||||
|
||||
input_file = config.get("input_file")
|
||||
if not input_file:
|
||||
print("Error: --file is required for import")
|
||||
return True
|
||||
|
||||
if not config["conversation_url"]:
|
||||
print("Error: No conversation URL configured")
|
||||
return True
|
||||
|
||||
try:
|
||||
from praisonai.db import PraisonDB
|
||||
import json
|
||||
|
||||
db = PraisonDB(database_url=config["conversation_url"])
|
||||
|
||||
with open(input_file, 'r') as f:
|
||||
data = json.loads(f.readline())
|
||||
|
||||
session_id = db.import_session(data)
|
||||
|
||||
print(f"Imported session: {session_id}")
|
||||
print(f"Messages: {len(data.get('messages', []))}")
|
||||
|
||||
db.close()
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
return True
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def handle_migrate(args: list) -> bool:
|
||||
"""Handle persistence migrate command."""
|
||||
config = parse_persistence_args(args)
|
||||
|
||||
if not config["conversation_url"] and not config["state_url"]:
|
||||
print("Error: No database URL configured")
|
||||
print("Set --conversation-url or --state-url")
|
||||
return True
|
||||
|
||||
try:
|
||||
from praisonai.db import PraisonDB
|
||||
from praisonai.persistence.migrations import MigrationManager
|
||||
|
||||
db = PraisonDB(
|
||||
database_url=config["conversation_url"],
|
||||
state_url=config["state_url"]
|
||||
)
|
||||
|
||||
manager = MigrationManager(db)
|
||||
|
||||
# Check for --up or --down flags
|
||||
direction = "up"
|
||||
target = None
|
||||
for i, arg in enumerate(args):
|
||||
if arg == "--down":
|
||||
direction = "down"
|
||||
elif arg == "--target" and i + 1 < len(args):
|
||||
target = args[i + 1]
|
||||
|
||||
if direction == "up":
|
||||
print("Applying pending migrations...")
|
||||
applied = manager.migrate_up(target)
|
||||
if applied:
|
||||
print(f"Applied migrations: {', '.join(applied)}")
|
||||
else:
|
||||
print("No pending migrations")
|
||||
else:
|
||||
if not target:
|
||||
print("Error: --target required for --down migration")
|
||||
return True
|
||||
print(f"Rolling back to version {target}...")
|
||||
rolled_back = manager.migrate_down(target)
|
||||
if rolled_back:
|
||||
print(f"Rolled back: {', '.join(rolled_back)}")
|
||||
else:
|
||||
print("No migrations to rollback")
|
||||
|
||||
db.close()
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
return True
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def handle_status(args: list) -> bool:
|
||||
"""Handle persistence status command."""
|
||||
config = parse_persistence_args(args)
|
||||
|
||||
if not config["conversation_url"] and not config["state_url"]:
|
||||
print("Error: No database URL configured")
|
||||
return True
|
||||
|
||||
try:
|
||||
from praisonai.db import PraisonDB
|
||||
from praisonai.persistence.migrations import MigrationManager
|
||||
|
||||
db = PraisonDB(
|
||||
database_url=config["conversation_url"],
|
||||
state_url=config["state_url"]
|
||||
)
|
||||
|
||||
manager = MigrationManager(db)
|
||||
status = manager.get_migration_status()
|
||||
|
||||
print("\n=== Schema Status ===")
|
||||
print(f"Current version: {status['current_version'] or '(not set)'}")
|
||||
print(f"Latest version: {status['latest_version']}")
|
||||
print(f"Pending: {status['pending_count']} migration(s)")
|
||||
|
||||
if status['pending_versions']:
|
||||
print(f"Pending versions: {', '.join(status['pending_versions'])}")
|
||||
|
||||
db.close()
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
return True
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _mask_url(url: str) -> str:
|
||||
"""Mask sensitive parts of URL."""
|
||||
if not url:
|
||||
return "(not set)"
|
||||
|
||||
# Mask password in URLs
|
||||
import re
|
||||
masked = re.sub(r'://([^:]+):([^@]+)@', r'://\1:****@', url)
|
||||
return masked
|
||||
|
||||
|
||||
def _test_conversation_store(url: str) -> tuple:
|
||||
"""Test conversation store connectivity."""
|
||||
try:
|
||||
from praisonai.persistence.factory import create_conversation_store
|
||||
|
||||
# Detect backend
|
||||
if "postgresql" in url or "postgres" in url:
|
||||
backend = "postgres"
|
||||
elif "mysql" in url:
|
||||
backend = "mysql"
|
||||
elif url.endswith(".db") or "sqlite" in url:
|
||||
backend = "sqlite"
|
||||
else:
|
||||
backend = "sqlite"
|
||||
|
||||
store = create_conversation_store(backend, url=url)
|
||||
|
||||
# Try a simple operation
|
||||
sessions = store.list_sessions(limit=1)
|
||||
store.close()
|
||||
|
||||
return True, f"Connected ({backend})"
|
||||
except Exception as e:
|
||||
return False, str(e)
|
||||
|
||||
|
||||
def _test_knowledge_store(url: str) -> tuple:
|
||||
"""Test knowledge store connectivity."""
|
||||
try:
|
||||
from praisonai.persistence.factory import create_knowledge_store
|
||||
|
||||
# Detect backend
|
||||
if "qdrant" in url or ":6333" in url:
|
||||
backend = "qdrant"
|
||||
elif "chroma" in url:
|
||||
backend = "chroma"
|
||||
else:
|
||||
backend = "qdrant"
|
||||
|
||||
store = create_knowledge_store(backend, url=url)
|
||||
|
||||
# Try a simple operation
|
||||
exists = store.collection_exists("test_doctor")
|
||||
store.close()
|
||||
|
||||
return True, f"Connected ({backend})"
|
||||
except Exception as e:
|
||||
return False, str(e)
|
||||
|
||||
|
||||
def _test_state_store(url: str) -> tuple:
|
||||
"""Test state store connectivity."""
|
||||
try:
|
||||
from praisonai.persistence.factory import create_state_store
|
||||
|
||||
# Detect backend
|
||||
if "redis" in url:
|
||||
backend = "redis"
|
||||
else:
|
||||
backend = "memory"
|
||||
|
||||
store = create_state_store(backend, url=url)
|
||||
|
||||
# Try a simple operation
|
||||
store.set("_doctor_test", "ok", ttl=5)
|
||||
value = store.get("_doctor_test")
|
||||
store.delete("_doctor_test")
|
||||
store.close()
|
||||
|
||||
return True, f"Connected ({backend})"
|
||||
except Exception as e:
|
||||
return False, str(e)
|
||||
@@ -266,6 +266,12 @@ class PraisonAI:
|
||||
file_input = self.read_file_if_provided(getattr(args, 'file', None))
|
||||
|
||||
if args.command:
|
||||
# Handle persistence command
|
||||
if args.command == "persistence":
|
||||
from praisonai.cli.features.persistence import handle_persistence_command
|
||||
handle_persistence_command(unknown_args)
|
||||
return None
|
||||
|
||||
# Handle schedule command
|
||||
if args.command == "schedule":
|
||||
from praisonai.cli.features.agent_scheduler import AgentSchedulerHandler
|
||||
|
||||
@@ -57,7 +57,7 @@ class CloudDeployer:
|
||||
file.write("FROM python:3.11-slim\n")
|
||||
file.write("WORKDIR /app\n")
|
||||
file.write("COPY . .\n")
|
||||
file.write("RUN pip install flask praisonai==2.4.0 gunicorn markdown\n")
|
||||
file.write("RUN pip install flask praisonai==2.4.4 gunicorn markdown\n")
|
||||
file.write("EXPOSE 8080\n")
|
||||
file.write('CMD ["gunicorn", "-b", "0.0.0.0:8080", "api:app"]\n')
|
||||
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
"""
|
||||
PraisonAI Persistence Layer
|
||||
|
||||
Provides database integrations for:
|
||||
- Conversation history persistence (ConversationStore)
|
||||
- Knowledge storage and retrieval (KnowledgeStore)
|
||||
- Session/state management (StateStore)
|
||||
|
||||
All integrations use lazy loading to avoid importing unused dependencies.
|
||||
|
||||
Supported Backends (22 total):
|
||||
- ConversationStore (6): postgres, mysql, sqlite, singlestore, supabase, surrealdb
|
||||
- KnowledgeStore (10): qdrant, pinecone, chroma, weaviate, lancedb, milvus, pgvector, redis, cassandra, clickhouse
|
||||
- StateStore (6): redis, dynamodb, firestore, mongodb, upstash, memory
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
__all__ = [
|
||||
# Base interfaces
|
||||
"ConversationStore",
|
||||
"ConversationSession",
|
||||
"ConversationMessage",
|
||||
"KnowledgeStore",
|
||||
"KnowledgeDocument",
|
||||
"StateStore",
|
||||
# Orchestrator
|
||||
"PersistenceOrchestrator",
|
||||
# Factory functions
|
||||
"create_conversation_store",
|
||||
"create_knowledge_store",
|
||||
"create_state_store",
|
||||
"create_stores_from_config",
|
||||
# Config
|
||||
"PersistenceConfig",
|
||||
# Agent hooks
|
||||
"wrap_agent_with_persistence",
|
||||
"PersistentAgent",
|
||||
"create_persistent_session",
|
||||
]
|
||||
|
||||
# Lazy imports to avoid loading dependencies until needed
|
||||
def __getattr__(name: str):
|
||||
if name in ("ConversationStore", "ConversationSession", "ConversationMessage"):
|
||||
from .conversation.base import ConversationStore, ConversationSession, ConversationMessage
|
||||
return locals()[name]
|
||||
|
||||
if name in ("KnowledgeStore", "KnowledgeDocument"):
|
||||
from .knowledge.base import KnowledgeStore, KnowledgeDocument
|
||||
return locals()[name]
|
||||
|
||||
if name == "StateStore":
|
||||
from .state.base import StateStore
|
||||
return StateStore
|
||||
|
||||
if name == "PersistenceOrchestrator":
|
||||
from .orchestrator import PersistenceOrchestrator
|
||||
return PersistenceOrchestrator
|
||||
|
||||
if name == "PersistenceConfig":
|
||||
from .config import PersistenceConfig
|
||||
return PersistenceConfig
|
||||
|
||||
if name == "create_conversation_store":
|
||||
from .factory import create_conversation_store
|
||||
return create_conversation_store
|
||||
|
||||
if name == "create_knowledge_store":
|
||||
from .factory import create_knowledge_store
|
||||
return create_knowledge_store
|
||||
|
||||
if name == "create_state_store":
|
||||
from .factory import create_state_store
|
||||
return create_state_store
|
||||
|
||||
if name == "create_stores_from_config":
|
||||
from .factory import create_stores_from_config
|
||||
return create_stores_from_config
|
||||
|
||||
if name in ("wrap_agent_with_persistence", "PersistentAgent", "create_persistent_session"):
|
||||
from .hooks.agent_hooks import wrap_agent_with_persistence, PersistentAgent, create_persistent_session
|
||||
return locals()[name]
|
||||
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -0,0 +1,238 @@
|
||||
"""
|
||||
Configuration handling for PraisonAI persistence layer.
|
||||
|
||||
Supports configuration via:
|
||||
- Environment variables
|
||||
- Config file (YAML/JSON)
|
||||
- CLI arguments
|
||||
- Direct Python API
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Environment variable mappings
|
||||
ENV_VARS = {
|
||||
# Conversation stores
|
||||
"conversation_store": "PRAISONAI_CONVERSATION_STORE",
|
||||
"conversation_url": "PRAISONAI_CONVERSATION_URL",
|
||||
# Knowledge stores
|
||||
"knowledge_store": "PRAISONAI_KNOWLEDGE_STORE",
|
||||
"knowledge_url": "PRAISONAI_KNOWLEDGE_URL",
|
||||
# State stores
|
||||
"state_store": "PRAISONAI_STATE_STORE",
|
||||
"state_url": "PRAISONAI_STATE_URL",
|
||||
# Common
|
||||
"session_id": "PRAISONAI_SESSION_ID",
|
||||
"user_id": "PRAISONAI_USER_ID",
|
||||
# PostgreSQL
|
||||
"postgres_url": "POSTGRES_URL",
|
||||
"postgres_host": "POSTGRES_HOST",
|
||||
"postgres_port": "POSTGRES_PORT",
|
||||
"postgres_database": "POSTGRES_DATABASE",
|
||||
"postgres_user": "POSTGRES_USER",
|
||||
"postgres_password": "POSTGRES_PASSWORD",
|
||||
# Qdrant
|
||||
"qdrant_url": "QDRANT_URL",
|
||||
"qdrant_host": "QDRANT_HOST",
|
||||
"qdrant_port": "QDRANT_PORT",
|
||||
"qdrant_api_key": "QDRANT_API_KEY",
|
||||
# Redis
|
||||
"redis_url": "REDIS_URL",
|
||||
"redis_host": "REDIS_HOST",
|
||||
"redis_port": "REDIS_PORT",
|
||||
"redis_password": "REDIS_PASSWORD",
|
||||
# Pinecone
|
||||
"pinecone_api_key": "PINECONE_API_KEY",
|
||||
"pinecone_environment": "PINECONE_ENVIRONMENT",
|
||||
"pinecone_index": "PINECONE_INDEX",
|
||||
# ChromaDB
|
||||
"chroma_path": "CHROMA_DB_PATH",
|
||||
"chroma_host": "CHROMA_HOST",
|
||||
"chroma_port": "CHROMA_PORT",
|
||||
# Weaviate
|
||||
"weaviate_url": "WEAVIATE_URL",
|
||||
"weaviate_api_key": "WEAVIATE_API_KEY",
|
||||
# Milvus
|
||||
"milvus_host": "MILVUS_HOST",
|
||||
"milvus_port": "MILVUS_PORT",
|
||||
"milvus_token": "MILVUS_TOKEN",
|
||||
# MongoDB
|
||||
"mongodb_url": "MONGODB_URL",
|
||||
"mongodb_database": "MONGODB_DATABASE",
|
||||
# DynamoDB
|
||||
"aws_region": "AWS_REGION",
|
||||
"dynamodb_table": "DYNAMODB_TABLE",
|
||||
# Firestore
|
||||
"google_credentials": "GOOGLE_APPLICATION_CREDENTIALS",
|
||||
"firestore_project": "FIRESTORE_PROJECT",
|
||||
# Upstash
|
||||
"upstash_redis_url": "UPSTASH_REDIS_URL",
|
||||
"upstash_redis_token": "UPSTASH_REDIS_TOKEN",
|
||||
"upstash_vector_url": "UPSTASH_VECTOR_URL",
|
||||
"upstash_vector_token": "UPSTASH_VECTOR_TOKEN",
|
||||
# Supabase
|
||||
"supabase_url": "SUPABASE_URL",
|
||||
"supabase_key": "SUPABASE_KEY",
|
||||
# SurrealDB
|
||||
"surrealdb_url": "SURREALDB_URL",
|
||||
"surrealdb_namespace": "SURREALDB_NS",
|
||||
"surrealdb_database": "SURREALDB_DB",
|
||||
# MySQL
|
||||
"mysql_url": "MYSQL_URL",
|
||||
"mysql_host": "MYSQL_HOST",
|
||||
"mysql_port": "MYSQL_PORT",
|
||||
"mysql_database": "MYSQL_DATABASE",
|
||||
"mysql_user": "MYSQL_USER",
|
||||
"mysql_password": "MYSQL_PASSWORD",
|
||||
# SingleStore
|
||||
"singlestore_url": "SINGLESTORE_URL",
|
||||
# ClickHouse
|
||||
"clickhouse_host": "CLICKHOUSE_HOST",
|
||||
"clickhouse_port": "CLICKHOUSE_PORT",
|
||||
"clickhouse_user": "CLICKHOUSE_USER",
|
||||
"clickhouse_password": "CLICKHOUSE_PASSWORD",
|
||||
# Cassandra
|
||||
"cassandra_hosts": "CASSANDRA_HOSTS",
|
||||
"cassandra_keyspace": "CASSANDRA_KEYSPACE",
|
||||
# LanceDB
|
||||
"lancedb_path": "LANCEDB_PATH",
|
||||
"lancedb_uri": "LANCEDB_URI",
|
||||
}
|
||||
|
||||
# Supported backends
|
||||
CONVERSATION_BACKENDS = ["postgres", "mysql", "sqlite", "singlestore", "supabase", "surrealdb"]
|
||||
KNOWLEDGE_BACKENDS = ["qdrant", "pinecone", "chroma", "weaviate", "lancedb", "milvus", "pgvector", "redis", "cassandra", "clickhouse"]
|
||||
STATE_BACKENDS = ["redis", "dynamodb", "firestore", "mongodb", "upstash", "memory"]
|
||||
|
||||
|
||||
@dataclass
|
||||
class PersistenceConfig:
|
||||
"""
|
||||
Configuration for PraisonAI persistence layer.
|
||||
|
||||
Example:
|
||||
config = PersistenceConfig(
|
||||
conversation_store="postgres",
|
||||
conversation_url="postgresql://localhost:5432/praisonai",
|
||||
knowledge_store="qdrant",
|
||||
knowledge_url="http://localhost:6333",
|
||||
state_store="redis",
|
||||
state_url="redis://localhost:6379",
|
||||
)
|
||||
"""
|
||||
# Store backends
|
||||
conversation_store: Optional[str] = None
|
||||
conversation_url: Optional[str] = None
|
||||
conversation_options: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
knowledge_store: Optional[str] = None
|
||||
knowledge_url: Optional[str] = None
|
||||
knowledge_options: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
state_store: Optional[str] = None
|
||||
state_url: Optional[str] = None
|
||||
state_options: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
# Session/user context
|
||||
session_id: Optional[str] = None
|
||||
user_id: Optional[str] = None
|
||||
|
||||
# General options
|
||||
auto_create_tables: bool = True
|
||||
lazy_init: bool = True
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "PersistenceConfig":
|
||||
"""Create config from environment variables."""
|
||||
return cls(
|
||||
conversation_store=os.getenv(ENV_VARS["conversation_store"]),
|
||||
conversation_url=os.getenv(ENV_VARS["conversation_url"]) or os.getenv(ENV_VARS["postgres_url"]),
|
||||
knowledge_store=os.getenv(ENV_VARS["knowledge_store"]),
|
||||
knowledge_url=os.getenv(ENV_VARS["knowledge_url"]) or os.getenv(ENV_VARS["qdrant_url"]),
|
||||
state_store=os.getenv(ENV_VARS["state_store"]),
|
||||
state_url=os.getenv(ENV_VARS["state_url"]) or os.getenv(ENV_VARS["redis_url"]),
|
||||
session_id=os.getenv(ENV_VARS["session_id"]),
|
||||
user_id=os.getenv(ENV_VARS["user_id"]),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "PersistenceConfig":
|
||||
"""Create config from dictionary."""
|
||||
return cls(**{k: v for k, v in data.items() if hasattr(cls, k)})
|
||||
|
||||
@classmethod
|
||||
def from_yaml(cls, path: str) -> "PersistenceConfig":
|
||||
"""Load config from YAML file."""
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
raise ImportError("PyYAML required for YAML config. Install with: pip install pyyaml")
|
||||
|
||||
with open(path) as f:
|
||||
data = yaml.safe_load(f)
|
||||
|
||||
# Handle nested persistence config
|
||||
if "persistence" in data:
|
||||
data = data["persistence"]
|
||||
|
||||
return cls.from_dict(data)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert config to dictionary."""
|
||||
return {
|
||||
"conversation_store": self.conversation_store,
|
||||
"conversation_url": self.conversation_url,
|
||||
"conversation_options": self.conversation_options,
|
||||
"knowledge_store": self.knowledge_store,
|
||||
"knowledge_url": self.knowledge_url,
|
||||
"knowledge_options": self.knowledge_options,
|
||||
"state_store": self.state_store,
|
||||
"state_url": self.state_url,
|
||||
"state_options": self.state_options,
|
||||
"session_id": self.session_id,
|
||||
"user_id": self.user_id,
|
||||
"auto_create_tables": self.auto_create_tables,
|
||||
"lazy_init": self.lazy_init,
|
||||
}
|
||||
|
||||
def validate(self) -> bool:
|
||||
"""Validate configuration."""
|
||||
valid = True
|
||||
|
||||
if self.conversation_store and self.conversation_store not in CONVERSATION_BACKENDS:
|
||||
logger.error(f"Invalid conversation_store: {self.conversation_store}. "
|
||||
f"Supported: {CONVERSATION_BACKENDS}")
|
||||
valid = False
|
||||
|
||||
if self.knowledge_store and self.knowledge_store not in KNOWLEDGE_BACKENDS:
|
||||
logger.error(f"Invalid knowledge_store: {self.knowledge_store}. "
|
||||
f"Supported: {KNOWLEDGE_BACKENDS}")
|
||||
valid = False
|
||||
|
||||
if self.state_store and self.state_store not in STATE_BACKENDS:
|
||||
logger.error(f"Invalid state_store: {self.state_store}. "
|
||||
f"Supported: {STATE_BACKENDS}")
|
||||
valid = False
|
||||
|
||||
return valid
|
||||
|
||||
|
||||
def get_env_var(key: str, default: Optional[str] = None) -> Optional[str]:
|
||||
"""Get environment variable by config key name."""
|
||||
env_name = ENV_VARS.get(key, key.upper())
|
||||
return os.getenv(env_name, default)
|
||||
|
||||
|
||||
def list_available_backends() -> Dict[str, list]:
|
||||
"""List all available backends by store type."""
|
||||
return {
|
||||
"conversation": CONVERSATION_BACKENDS,
|
||||
"knowledge": KNOWLEDGE_BACKENDS,
|
||||
"state": STATE_BACKENDS,
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
"""
|
||||
ConversationStore implementations for session and message persistence.
|
||||
|
||||
Supported backends:
|
||||
- PostgreSQL (postgres)
|
||||
- MySQL (mysql)
|
||||
- SQLite (sqlite)
|
||||
- SingleStore (singlestore)
|
||||
- Supabase (supabase)
|
||||
- SurrealDB (surrealdb)
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
__all__ = [
|
||||
"ConversationStore",
|
||||
"ConversationSession",
|
||||
"ConversationMessage",
|
||||
]
|
||||
|
||||
def __getattr__(name: str):
|
||||
if name in ("ConversationStore", "ConversationSession", "ConversationMessage"):
|
||||
from .base import ConversationStore, ConversationSession, ConversationMessage
|
||||
return locals()[name]
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -0,0 +1,427 @@
|
||||
"""
|
||||
Async MySQL implementation of ConversationStore.
|
||||
|
||||
Requires: aiomysql
|
||||
Install: pip install aiomysql
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from typing import List, Optional
|
||||
|
||||
from .base import ConversationStore, ConversationSession, ConversationMessage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AsyncMySQLConversationStore(ConversationStore):
|
||||
"""
|
||||
Async MySQL conversation store using aiomysql.
|
||||
|
||||
Provides high-performance async database operations.
|
||||
|
||||
Example:
|
||||
store = AsyncMySQLConversationStore(
|
||||
host="localhost",
|
||||
user="root",
|
||||
password="password",
|
||||
database="praisonai"
|
||||
)
|
||||
await store.init()
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: Optional[str] = None,
|
||||
host: str = "localhost",
|
||||
port: int = 3306,
|
||||
database: str = "praisonai",
|
||||
user: str = "root",
|
||||
password: str = "",
|
||||
table_prefix: str = "praisonai_",
|
||||
pool_size: int = 10,
|
||||
):
|
||||
"""
|
||||
Initialize async MySQL store.
|
||||
|
||||
Args:
|
||||
url: MySQL connection URL (takes precedence)
|
||||
host: Database host
|
||||
port: Database port
|
||||
database: Database name
|
||||
user: Database user
|
||||
password: Database password
|
||||
table_prefix: Prefix for table names
|
||||
pool_size: Connection pool size
|
||||
"""
|
||||
self.url = url
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.database = database
|
||||
self.user = user
|
||||
self.password = password
|
||||
self.table_prefix = table_prefix
|
||||
self.pool_size = pool_size
|
||||
self._pool = None
|
||||
self._initialized = False
|
||||
|
||||
if url:
|
||||
self._parse_url(url)
|
||||
|
||||
def _parse_url(self, url: str):
|
||||
"""Parse MySQL URL into components."""
|
||||
from urllib.parse import urlparse
|
||||
parsed = urlparse(url)
|
||||
self.host = parsed.hostname or "localhost"
|
||||
self.port = parsed.port or 3306
|
||||
self.database = parsed.path.lstrip('/') or "praisonai"
|
||||
self.user = parsed.username or "root"
|
||||
self.password = parsed.password or ""
|
||||
|
||||
async def init(self):
|
||||
"""Initialize connection pool and create tables."""
|
||||
if self._initialized:
|
||||
return
|
||||
|
||||
try:
|
||||
import aiomysql
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"aiomysql is required for async MySQL support. "
|
||||
"Install with: pip install aiomysql"
|
||||
)
|
||||
|
||||
self._pool = await aiomysql.create_pool(
|
||||
host=self.host,
|
||||
port=self.port,
|
||||
user=self.user,
|
||||
password=self.password,
|
||||
db=self.database,
|
||||
minsize=1,
|
||||
maxsize=self.pool_size,
|
||||
autocommit=True
|
||||
)
|
||||
|
||||
await self._create_tables()
|
||||
self._initialized = True
|
||||
|
||||
async def _create_tables(self):
|
||||
"""Create required tables if they don't exist."""
|
||||
sessions_table = f"{self.table_prefix}sessions"
|
||||
messages_table = f"{self.table_prefix}messages"
|
||||
|
||||
async with self._pool.acquire() as conn:
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(f"""
|
||||
CREATE TABLE IF NOT EXISTS {sessions_table} (
|
||||
session_id VARCHAR(255) PRIMARY KEY,
|
||||
user_id VARCHAR(255),
|
||||
agent_id VARCHAR(255),
|
||||
name VARCHAR(255),
|
||||
metadata JSON,
|
||||
created_at DOUBLE,
|
||||
updated_at DOUBLE,
|
||||
INDEX idx_user_id (user_id),
|
||||
INDEX idx_agent_id (agent_id)
|
||||
)
|
||||
""")
|
||||
|
||||
await cur.execute(f"""
|
||||
CREATE TABLE IF NOT EXISTS {messages_table} (
|
||||
id VARCHAR(255) PRIMARY KEY,
|
||||
session_id VARCHAR(255),
|
||||
role VARCHAR(50),
|
||||
content TEXT,
|
||||
metadata JSON,
|
||||
created_at DOUBLE,
|
||||
INDEX idx_session_id (session_id),
|
||||
FOREIGN KEY (session_id) REFERENCES {sessions_table}(session_id) ON DELETE CASCADE
|
||||
)
|
||||
""")
|
||||
|
||||
async def async_create_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Create a new session asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
table = f"{self.table_prefix}sessions"
|
||||
async with self._pool.acquire() as conn:
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(f"""
|
||||
INSERT INTO {table} (session_id, user_id, agent_id, name, metadata, created_at, updated_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||||
""", (session.session_id, session.user_id, session.agent_id, session.name,
|
||||
json.dumps(session.metadata) if session.metadata else None,
|
||||
session.created_at, session.updated_at))
|
||||
|
||||
return session
|
||||
|
||||
def create_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Sync wrapper for create_session."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_create_session(session)
|
||||
)
|
||||
|
||||
async def async_get_session(self, session_id: str) -> Optional[ConversationSession]:
|
||||
"""Get a session by ID asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
table = f"{self.table_prefix}sessions"
|
||||
async with self._pool.acquire() as conn:
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(f"""
|
||||
SELECT session_id, user_id, agent_id, name, metadata, created_at, updated_at
|
||||
FROM {table} WHERE session_id = %s
|
||||
""", (session_id,))
|
||||
row = await cur.fetchone()
|
||||
|
||||
if row:
|
||||
return ConversationSession(
|
||||
session_id=row[0],
|
||||
user_id=row[1],
|
||||
agent_id=row[2],
|
||||
name=row[3],
|
||||
metadata=json.loads(row[4]) if row[4] else None,
|
||||
created_at=row[5],
|
||||
updated_at=row[6]
|
||||
)
|
||||
return None
|
||||
|
||||
def get_session(self, session_id: str) -> Optional[ConversationSession]:
|
||||
"""Sync wrapper for get_session."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_get_session(session_id)
|
||||
)
|
||||
|
||||
async def async_update_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Update an existing session asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
session.updated_at = time.time()
|
||||
table = f"{self.table_prefix}sessions"
|
||||
|
||||
async with self._pool.acquire() as conn:
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(f"""
|
||||
UPDATE {table}
|
||||
SET name = %s, metadata = %s, updated_at = %s
|
||||
WHERE session_id = %s
|
||||
""", (session.name, json.dumps(session.metadata) if session.metadata else None,
|
||||
session.updated_at, session.session_id))
|
||||
|
||||
return session
|
||||
|
||||
def update_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Sync wrapper for update_session."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_update_session(session)
|
||||
)
|
||||
|
||||
async def async_delete_session(self, session_id: str) -> bool:
|
||||
"""Delete a session asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
table = f"{self.table_prefix}sessions"
|
||||
async with self._pool.acquire() as conn:
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(f"""
|
||||
DELETE FROM {table} WHERE session_id = %s
|
||||
""", (session_id,))
|
||||
return cur.rowcount > 0
|
||||
|
||||
def delete_session(self, session_id: str) -> bool:
|
||||
"""Sync wrapper for delete_session."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_delete_session(session_id)
|
||||
)
|
||||
|
||||
async def async_list_sessions(
|
||||
self,
|
||||
user_id: Optional[str] = None,
|
||||
agent_id: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0
|
||||
) -> List[ConversationSession]:
|
||||
"""List sessions asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
table = f"{self.table_prefix}sessions"
|
||||
conditions = []
|
||||
params = []
|
||||
|
||||
if user_id:
|
||||
conditions.append("user_id = %s")
|
||||
params.append(user_id)
|
||||
|
||||
if agent_id:
|
||||
conditions.append("agent_id = %s")
|
||||
params.append(agent_id)
|
||||
|
||||
where_clause = f"WHERE {' AND '.join(conditions)}" if conditions else ""
|
||||
params.extend([limit, offset])
|
||||
|
||||
async with self._pool.acquire() as conn:
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(f"""
|
||||
SELECT session_id, user_id, agent_id, name, metadata, created_at, updated_at
|
||||
FROM {table} {where_clause}
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT %s OFFSET %s
|
||||
""", params)
|
||||
rows = await cur.fetchall()
|
||||
|
||||
return [
|
||||
ConversationSession(
|
||||
session_id=row[0],
|
||||
user_id=row[1],
|
||||
agent_id=row[2],
|
||||
name=row[3],
|
||||
metadata=json.loads(row[4]) if row[4] else None,
|
||||
created_at=row[5],
|
||||
updated_at=row[6]
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def list_sessions(
|
||||
self,
|
||||
user_id: Optional[str] = None,
|
||||
agent_id: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0
|
||||
) -> List[ConversationSession]:
|
||||
"""Sync wrapper for list_sessions."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_list_sessions(user_id, agent_id, limit, offset)
|
||||
)
|
||||
|
||||
async def async_add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
|
||||
"""Add a message asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
message.session_id = session_id
|
||||
table = f"{self.table_prefix}messages"
|
||||
|
||||
async with self._pool.acquire() as conn:
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(f"""
|
||||
INSERT INTO {table} (id, session_id, role, content, metadata, created_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s)
|
||||
""", (message.id, session_id, message.role, message.content,
|
||||
json.dumps(message.metadata) if message.metadata else None,
|
||||
message.created_at))
|
||||
|
||||
return message
|
||||
|
||||
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
|
||||
"""Sync wrapper for add_message."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_add_message(session_id, message)
|
||||
)
|
||||
|
||||
async def async_get_messages(
|
||||
self,
|
||||
session_id: str,
|
||||
limit: Optional[int] = None,
|
||||
before: Optional[float] = None,
|
||||
after: Optional[float] = None
|
||||
) -> List[ConversationMessage]:
|
||||
"""Get messages asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
table = f"{self.table_prefix}messages"
|
||||
conditions = ["session_id = %s"]
|
||||
params = [session_id]
|
||||
|
||||
if before:
|
||||
conditions.append("created_at < %s")
|
||||
params.append(before)
|
||||
|
||||
if after:
|
||||
conditions.append("created_at > %s")
|
||||
params.append(after)
|
||||
|
||||
where_clause = f"WHERE {' AND '.join(conditions)}"
|
||||
limit_clause = "LIMIT %s" if limit else ""
|
||||
if limit:
|
||||
params.append(limit)
|
||||
|
||||
async with self._pool.acquire() as conn:
|
||||
async with conn.cursor() as cur:
|
||||
await cur.execute(f"""
|
||||
SELECT id, session_id, role, content, metadata, created_at
|
||||
FROM {table} {where_clause}
|
||||
ORDER BY created_at ASC {limit_clause}
|
||||
""", params)
|
||||
rows = await cur.fetchall()
|
||||
|
||||
return [
|
||||
ConversationMessage(
|
||||
id=row[0],
|
||||
session_id=row[1],
|
||||
role=row[2],
|
||||
content=row[3],
|
||||
metadata=json.loads(row[4]) if row[4] else None,
|
||||
created_at=row[5]
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def get_messages(
|
||||
self,
|
||||
session_id: str,
|
||||
limit: Optional[int] = None,
|
||||
before: Optional[float] = None,
|
||||
after: Optional[float] = None
|
||||
) -> List[ConversationMessage]:
|
||||
"""Sync wrapper for get_messages."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_get_messages(session_id, limit, before, after)
|
||||
)
|
||||
|
||||
async def async_delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
|
||||
"""Delete messages asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
table = f"{self.table_prefix}messages"
|
||||
|
||||
async with self._pool.acquire() as conn:
|
||||
async with conn.cursor() as cur:
|
||||
if message_ids is None:
|
||||
await cur.execute(f"""
|
||||
DELETE FROM {table} WHERE session_id = %s
|
||||
""", (session_id,))
|
||||
else:
|
||||
placeholders = ','.join(['%s'] * len(message_ids))
|
||||
await cur.execute(f"""
|
||||
DELETE FROM {table} WHERE session_id = %s AND id IN ({placeholders})
|
||||
""", [session_id] + message_ids)
|
||||
return cur.rowcount
|
||||
|
||||
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
|
||||
"""Sync wrapper for delete_messages."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_delete_messages(session_id, message_ids)
|
||||
)
|
||||
|
||||
async def async_close(self) -> None:
|
||||
"""Close the connection pool."""
|
||||
if self._pool:
|
||||
self._pool.close()
|
||||
await self._pool.wait_closed()
|
||||
self._pool = None
|
||||
self._initialized = False
|
||||
|
||||
def close(self) -> None:
|
||||
"""Sync wrapper for close."""
|
||||
if self._pool:
|
||||
asyncio.get_event_loop().run_until_complete(self.async_close())
|
||||
@@ -0,0 +1,410 @@
|
||||
"""
|
||||
Async PostgreSQL implementation of ConversationStore.
|
||||
|
||||
Requires: asyncpg
|
||||
Install: pip install asyncpg
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from typing import List, Optional
|
||||
|
||||
from .base import ConversationStore, ConversationSession, ConversationMessage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AsyncPostgresConversationStore(ConversationStore):
|
||||
"""
|
||||
Async PostgreSQL conversation store using asyncpg.
|
||||
|
||||
Provides high-performance async database operations.
|
||||
|
||||
Example:
|
||||
store = AsyncPostgresConversationStore(
|
||||
url="postgresql://user:pass@localhost:5432/praisonai"
|
||||
)
|
||||
await store.init()
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: Optional[str] = None,
|
||||
host: str = "localhost",
|
||||
port: int = 5432,
|
||||
database: str = "praisonai",
|
||||
user: str = "postgres",
|
||||
password: str = "",
|
||||
table_prefix: str = "praisonai_",
|
||||
pool_size: int = 10,
|
||||
):
|
||||
"""
|
||||
Initialize async PostgreSQL store.
|
||||
|
||||
Args:
|
||||
url: PostgreSQL connection URL (takes precedence)
|
||||
host: Database host
|
||||
port: Database port
|
||||
database: Database name
|
||||
user: Database user
|
||||
password: Database password
|
||||
table_prefix: Prefix for table names
|
||||
pool_size: Connection pool size
|
||||
"""
|
||||
self.url = url
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.database = database
|
||||
self.user = user
|
||||
self.password = password
|
||||
self.table_prefix = table_prefix
|
||||
self.pool_size = pool_size
|
||||
self._pool = None
|
||||
self._initialized = False
|
||||
|
||||
async def init(self):
|
||||
"""Initialize connection pool and create tables."""
|
||||
if self._initialized:
|
||||
return
|
||||
|
||||
try:
|
||||
import asyncpg
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"asyncpg is required for async PostgreSQL support. "
|
||||
"Install with: pip install asyncpg"
|
||||
)
|
||||
|
||||
if self.url:
|
||||
self._pool = await asyncpg.create_pool(self.url, min_size=1, max_size=self.pool_size)
|
||||
else:
|
||||
self._pool = await asyncpg.create_pool(
|
||||
host=self.host,
|
||||
port=self.port,
|
||||
database=self.database,
|
||||
user=self.user,
|
||||
password=self.password,
|
||||
min_size=1,
|
||||
max_size=self.pool_size
|
||||
)
|
||||
|
||||
await self._create_tables()
|
||||
self._initialized = True
|
||||
|
||||
async def _create_tables(self):
|
||||
"""Create required tables if they don't exist."""
|
||||
sessions_table = f"{self.table_prefix}sessions"
|
||||
messages_table = f"{self.table_prefix}messages"
|
||||
|
||||
async with self._pool.acquire() as conn:
|
||||
await conn.execute(f"""
|
||||
CREATE TABLE IF NOT EXISTS {sessions_table} (
|
||||
session_id VARCHAR(255) PRIMARY KEY,
|
||||
user_id VARCHAR(255),
|
||||
agent_id VARCHAR(255),
|
||||
name VARCHAR(255),
|
||||
metadata JSONB,
|
||||
created_at DOUBLE PRECISION,
|
||||
updated_at DOUBLE PRECISION
|
||||
)
|
||||
""")
|
||||
|
||||
await conn.execute(f"""
|
||||
CREATE TABLE IF NOT EXISTS {messages_table} (
|
||||
id VARCHAR(255) PRIMARY KEY,
|
||||
session_id VARCHAR(255) REFERENCES {sessions_table}(session_id) ON DELETE CASCADE,
|
||||
role VARCHAR(50),
|
||||
content TEXT,
|
||||
metadata JSONB,
|
||||
created_at DOUBLE PRECISION
|
||||
)
|
||||
""")
|
||||
|
||||
await conn.execute(f"""
|
||||
CREATE INDEX IF NOT EXISTS idx_{messages_table}_session
|
||||
ON {messages_table}(session_id)
|
||||
""")
|
||||
|
||||
async def async_create_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Create a new session asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
table = f"{self.table_prefix}sessions"
|
||||
async with self._pool.acquire() as conn:
|
||||
await conn.execute(f"""
|
||||
INSERT INTO {table} (session_id, user_id, agent_id, name, metadata, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
""", session.session_id, session.user_id, session.agent_id, session.name,
|
||||
json.dumps(session.metadata) if session.metadata else None,
|
||||
session.created_at, session.updated_at)
|
||||
|
||||
return session
|
||||
|
||||
def create_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Sync wrapper for create_session."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_create_session(session)
|
||||
)
|
||||
|
||||
async def async_get_session(self, session_id: str) -> Optional[ConversationSession]:
|
||||
"""Get a session by ID asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
table = f"{self.table_prefix}sessions"
|
||||
async with self._pool.acquire() as conn:
|
||||
row = await conn.fetchrow(f"""
|
||||
SELECT * FROM {table} WHERE session_id = $1
|
||||
""", session_id)
|
||||
|
||||
if row:
|
||||
return ConversationSession(
|
||||
session_id=row['session_id'],
|
||||
user_id=row['user_id'],
|
||||
agent_id=row['agent_id'],
|
||||
name=row['name'],
|
||||
metadata=json.loads(row['metadata']) if row['metadata'] else None,
|
||||
created_at=row['created_at'],
|
||||
updated_at=row['updated_at']
|
||||
)
|
||||
return None
|
||||
|
||||
def get_session(self, session_id: str) -> Optional[ConversationSession]:
|
||||
"""Sync wrapper for get_session."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_get_session(session_id)
|
||||
)
|
||||
|
||||
async def async_update_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Update an existing session asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
session.updated_at = time.time()
|
||||
table = f"{self.table_prefix}sessions"
|
||||
|
||||
async with self._pool.acquire() as conn:
|
||||
await conn.execute(f"""
|
||||
UPDATE {table}
|
||||
SET name = $2, metadata = $3, updated_at = $4
|
||||
WHERE session_id = $1
|
||||
""", session.session_id, session.name,
|
||||
json.dumps(session.metadata) if session.metadata else None,
|
||||
session.updated_at)
|
||||
|
||||
return session
|
||||
|
||||
def update_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Sync wrapper for update_session."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_update_session(session)
|
||||
)
|
||||
|
||||
async def async_delete_session(self, session_id: str) -> bool:
|
||||
"""Delete a session asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
table = f"{self.table_prefix}sessions"
|
||||
async with self._pool.acquire() as conn:
|
||||
result = await conn.execute(f"""
|
||||
DELETE FROM {table} WHERE session_id = $1
|
||||
""", session_id)
|
||||
|
||||
return "DELETE 1" in result
|
||||
|
||||
def delete_session(self, session_id: str) -> bool:
|
||||
"""Sync wrapper for delete_session."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_delete_session(session_id)
|
||||
)
|
||||
|
||||
async def async_list_sessions(
|
||||
self,
|
||||
user_id: Optional[str] = None,
|
||||
agent_id: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0
|
||||
) -> List[ConversationSession]:
|
||||
"""List sessions asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
table = f"{self.table_prefix}sessions"
|
||||
conditions = []
|
||||
params = []
|
||||
param_idx = 1
|
||||
|
||||
if user_id:
|
||||
conditions.append(f"user_id = ${param_idx}")
|
||||
params.append(user_id)
|
||||
param_idx += 1
|
||||
|
||||
if agent_id:
|
||||
conditions.append(f"agent_id = ${param_idx}")
|
||||
params.append(agent_id)
|
||||
param_idx += 1
|
||||
|
||||
where_clause = f"WHERE {' AND '.join(conditions)}" if conditions else ""
|
||||
params.extend([limit, offset])
|
||||
|
||||
async with self._pool.acquire() as conn:
|
||||
rows = await conn.fetch(f"""
|
||||
SELECT * FROM {table} {where_clause}
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT ${param_idx} OFFSET ${param_idx + 1}
|
||||
""", *params)
|
||||
|
||||
return [
|
||||
ConversationSession(
|
||||
session_id=row['session_id'],
|
||||
user_id=row['user_id'],
|
||||
agent_id=row['agent_id'],
|
||||
name=row['name'],
|
||||
metadata=json.loads(row['metadata']) if row['metadata'] else None,
|
||||
created_at=row['created_at'],
|
||||
updated_at=row['updated_at']
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def list_sessions(
|
||||
self,
|
||||
user_id: Optional[str] = None,
|
||||
agent_id: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0
|
||||
) -> List[ConversationSession]:
|
||||
"""Sync wrapper for list_sessions."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_list_sessions(user_id, agent_id, limit, offset)
|
||||
)
|
||||
|
||||
async def async_add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
|
||||
"""Add a message asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
message.session_id = session_id
|
||||
table = f"{self.table_prefix}messages"
|
||||
|
||||
async with self._pool.acquire() as conn:
|
||||
await conn.execute(f"""
|
||||
INSERT INTO {table} (id, session_id, role, content, metadata, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
""", message.id, session_id, message.role, message.content,
|
||||
json.dumps(message.metadata) if message.metadata else None,
|
||||
message.created_at)
|
||||
|
||||
return message
|
||||
|
||||
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
|
||||
"""Sync wrapper for add_message."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_add_message(session_id, message)
|
||||
)
|
||||
|
||||
async def async_get_messages(
|
||||
self,
|
||||
session_id: str,
|
||||
limit: Optional[int] = None,
|
||||
before: Optional[float] = None,
|
||||
after: Optional[float] = None
|
||||
) -> List[ConversationMessage]:
|
||||
"""Get messages asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
table = f"{self.table_prefix}messages"
|
||||
conditions = ["session_id = $1"]
|
||||
params = [session_id]
|
||||
param_idx = 2
|
||||
|
||||
if before:
|
||||
conditions.append(f"created_at < ${param_idx}")
|
||||
params.append(before)
|
||||
param_idx += 1
|
||||
|
||||
if after:
|
||||
conditions.append(f"created_at > ${param_idx}")
|
||||
params.append(after)
|
||||
param_idx += 1
|
||||
|
||||
where_clause = f"WHERE {' AND '.join(conditions)}"
|
||||
limit_clause = f"LIMIT ${param_idx}" if limit else ""
|
||||
if limit:
|
||||
params.append(limit)
|
||||
|
||||
async with self._pool.acquire() as conn:
|
||||
rows = await conn.fetch(f"""
|
||||
SELECT * FROM {table} {where_clause}
|
||||
ORDER BY created_at ASC {limit_clause}
|
||||
""", *params)
|
||||
|
||||
return [
|
||||
ConversationMessage(
|
||||
id=row['id'],
|
||||
session_id=row['session_id'],
|
||||
role=row['role'],
|
||||
content=row['content'],
|
||||
metadata=json.loads(row['metadata']) if row['metadata'] else None,
|
||||
created_at=row['created_at']
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def get_messages(
|
||||
self,
|
||||
session_id: str,
|
||||
limit: Optional[int] = None,
|
||||
before: Optional[float] = None,
|
||||
after: Optional[float] = None
|
||||
) -> List[ConversationMessage]:
|
||||
"""Sync wrapper for get_messages."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_get_messages(session_id, limit, before, after)
|
||||
)
|
||||
|
||||
async def async_delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
|
||||
"""Delete messages asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
table = f"{self.table_prefix}messages"
|
||||
|
||||
async with self._pool.acquire() as conn:
|
||||
if message_ids is None:
|
||||
result = await conn.execute(f"""
|
||||
DELETE FROM {table} WHERE session_id = $1
|
||||
""", session_id)
|
||||
else:
|
||||
result = await conn.execute(f"""
|
||||
DELETE FROM {table} WHERE session_id = $1 AND id = ANY($2)
|
||||
""", session_id, message_ids)
|
||||
|
||||
# Parse count from result like "DELETE 5"
|
||||
try:
|
||||
return int(result.split()[-1])
|
||||
except (ValueError, IndexError):
|
||||
return 0
|
||||
|
||||
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
|
||||
"""Sync wrapper for delete_messages."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_delete_messages(session_id, message_ids)
|
||||
)
|
||||
|
||||
async def async_close(self) -> None:
|
||||
"""Close the connection pool."""
|
||||
if self._pool:
|
||||
await self._pool.close()
|
||||
self._pool = None
|
||||
self._initialized = False
|
||||
|
||||
def close(self) -> None:
|
||||
"""Sync wrapper for close."""
|
||||
if self._pool:
|
||||
asyncio.get_event_loop().run_until_complete(self.async_close())
|
||||
@@ -0,0 +1,371 @@
|
||||
"""
|
||||
Async SQLite implementation of ConversationStore.
|
||||
|
||||
Requires: aiosqlite
|
||||
Install: pip install aiosqlite
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from typing import List, Optional
|
||||
|
||||
from .base import ConversationStore, ConversationSession, ConversationMessage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AsyncSQLiteConversationStore(ConversationStore):
|
||||
"""
|
||||
Async SQLite conversation store using aiosqlite.
|
||||
|
||||
Provides async database operations for SQLite.
|
||||
|
||||
Example:
|
||||
store = AsyncSQLiteConversationStore(path="./conversations.db")
|
||||
await store.init()
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
path: str = "praisonai_conversations.db",
|
||||
table_prefix: str = "praisonai_",
|
||||
):
|
||||
"""
|
||||
Initialize async SQLite store.
|
||||
|
||||
Args:
|
||||
path: Path to SQLite database file
|
||||
table_prefix: Prefix for table names
|
||||
"""
|
||||
self.path = path
|
||||
self.table_prefix = table_prefix
|
||||
self._conn = None
|
||||
self._initialized = False
|
||||
|
||||
async def init(self):
|
||||
"""Initialize connection and create tables."""
|
||||
if self._initialized:
|
||||
return
|
||||
|
||||
try:
|
||||
import aiosqlite
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"aiosqlite is required for async SQLite support. "
|
||||
"Install with: pip install aiosqlite"
|
||||
)
|
||||
|
||||
self._aiosqlite = aiosqlite
|
||||
self._conn = await aiosqlite.connect(self.path)
|
||||
self._conn.row_factory = aiosqlite.Row
|
||||
await self._create_tables()
|
||||
self._initialized = True
|
||||
|
||||
async def _create_tables(self):
|
||||
"""Create required tables if they don't exist."""
|
||||
sessions_table = f"{self.table_prefix}sessions"
|
||||
messages_table = f"{self.table_prefix}messages"
|
||||
|
||||
await self._conn.execute(f"""
|
||||
CREATE TABLE IF NOT EXISTS {sessions_table} (
|
||||
session_id TEXT PRIMARY KEY,
|
||||
user_id TEXT,
|
||||
agent_id TEXT,
|
||||
name TEXT,
|
||||
metadata TEXT,
|
||||
created_at REAL,
|
||||
updated_at REAL
|
||||
)
|
||||
""")
|
||||
|
||||
await self._conn.execute(f"""
|
||||
CREATE TABLE IF NOT EXISTS {messages_table} (
|
||||
id TEXT PRIMARY KEY,
|
||||
session_id TEXT REFERENCES {sessions_table}(session_id) ON DELETE CASCADE,
|
||||
role TEXT,
|
||||
content TEXT,
|
||||
metadata TEXT,
|
||||
created_at REAL
|
||||
)
|
||||
""")
|
||||
|
||||
await self._conn.execute(f"""
|
||||
CREATE INDEX IF NOT EXISTS idx_{messages_table}_session
|
||||
ON {messages_table}(session_id)
|
||||
""")
|
||||
|
||||
await self._conn.commit()
|
||||
|
||||
async def async_create_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Create a new session asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
table = f"{self.table_prefix}sessions"
|
||||
await self._conn.execute(f"""
|
||||
INSERT INTO {table} (session_id, user_id, agent_id, name, metadata, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""", (session.session_id, session.user_id, session.agent_id, session.name,
|
||||
json.dumps(session.metadata) if session.metadata else None,
|
||||
session.created_at, session.updated_at))
|
||||
await self._conn.commit()
|
||||
|
||||
return session
|
||||
|
||||
def create_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Sync wrapper for create_session."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_create_session(session)
|
||||
)
|
||||
|
||||
async def async_get_session(self, session_id: str) -> Optional[ConversationSession]:
|
||||
"""Get a session by ID asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
table = f"{self.table_prefix}sessions"
|
||||
async with self._conn.execute(f"""
|
||||
SELECT * FROM {table} WHERE session_id = ?
|
||||
""", (session_id,)) as cursor:
|
||||
row = await cursor.fetchone()
|
||||
|
||||
if row:
|
||||
return ConversationSession(
|
||||
session_id=row['session_id'],
|
||||
user_id=row['user_id'],
|
||||
agent_id=row['agent_id'],
|
||||
name=row['name'],
|
||||
metadata=json.loads(row['metadata']) if row['metadata'] else None,
|
||||
created_at=row['created_at'],
|
||||
updated_at=row['updated_at']
|
||||
)
|
||||
return None
|
||||
|
||||
def get_session(self, session_id: str) -> Optional[ConversationSession]:
|
||||
"""Sync wrapper for get_session."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_get_session(session_id)
|
||||
)
|
||||
|
||||
async def async_update_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Update an existing session asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
session.updated_at = time.time()
|
||||
table = f"{self.table_prefix}sessions"
|
||||
|
||||
await self._conn.execute(f"""
|
||||
UPDATE {table}
|
||||
SET name = ?, metadata = ?, updated_at = ?
|
||||
WHERE session_id = ?
|
||||
""", (session.name, json.dumps(session.metadata) if session.metadata else None,
|
||||
session.updated_at, session.session_id))
|
||||
await self._conn.commit()
|
||||
|
||||
return session
|
||||
|
||||
def update_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Sync wrapper for update_session."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_update_session(session)
|
||||
)
|
||||
|
||||
async def async_delete_session(self, session_id: str) -> bool:
|
||||
"""Delete a session asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
table = f"{self.table_prefix}sessions"
|
||||
cursor = await self._conn.execute(f"""
|
||||
DELETE FROM {table} WHERE session_id = ?
|
||||
""", (session_id,))
|
||||
await self._conn.commit()
|
||||
|
||||
return cursor.rowcount > 0
|
||||
|
||||
def delete_session(self, session_id: str) -> bool:
|
||||
"""Sync wrapper for delete_session."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_delete_session(session_id)
|
||||
)
|
||||
|
||||
async def async_list_sessions(
|
||||
self,
|
||||
user_id: Optional[str] = None,
|
||||
agent_id: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0
|
||||
) -> List[ConversationSession]:
|
||||
"""List sessions asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
table = f"{self.table_prefix}sessions"
|
||||
conditions = []
|
||||
params = []
|
||||
|
||||
if user_id:
|
||||
conditions.append("user_id = ?")
|
||||
params.append(user_id)
|
||||
|
||||
if agent_id:
|
||||
conditions.append("agent_id = ?")
|
||||
params.append(agent_id)
|
||||
|
||||
where_clause = f"WHERE {' AND '.join(conditions)}" if conditions else ""
|
||||
params.extend([limit, offset])
|
||||
|
||||
async with self._conn.execute(f"""
|
||||
SELECT * FROM {table} {where_clause}
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT ? OFFSET ?
|
||||
""", params) as cursor:
|
||||
rows = await cursor.fetchall()
|
||||
|
||||
return [
|
||||
ConversationSession(
|
||||
session_id=row['session_id'],
|
||||
user_id=row['user_id'],
|
||||
agent_id=row['agent_id'],
|
||||
name=row['name'],
|
||||
metadata=json.loads(row['metadata']) if row['metadata'] else None,
|
||||
created_at=row['created_at'],
|
||||
updated_at=row['updated_at']
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def list_sessions(
|
||||
self,
|
||||
user_id: Optional[str] = None,
|
||||
agent_id: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0
|
||||
) -> List[ConversationSession]:
|
||||
"""Sync wrapper for list_sessions."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_list_sessions(user_id, agent_id, limit, offset)
|
||||
)
|
||||
|
||||
async def async_add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
|
||||
"""Add a message asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
message.session_id = session_id
|
||||
table = f"{self.table_prefix}messages"
|
||||
|
||||
await self._conn.execute(f"""
|
||||
INSERT INTO {table} (id, session_id, role, content, metadata, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
""", (message.id, session_id, message.role, message.content,
|
||||
json.dumps(message.metadata) if message.metadata else None,
|
||||
message.created_at))
|
||||
await self._conn.commit()
|
||||
|
||||
return message
|
||||
|
||||
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
|
||||
"""Sync wrapper for add_message."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_add_message(session_id, message)
|
||||
)
|
||||
|
||||
async def async_get_messages(
|
||||
self,
|
||||
session_id: str,
|
||||
limit: Optional[int] = None,
|
||||
before: Optional[float] = None,
|
||||
after: Optional[float] = None
|
||||
) -> List[ConversationMessage]:
|
||||
"""Get messages asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
table = f"{self.table_prefix}messages"
|
||||
conditions = ["session_id = ?"]
|
||||
params = [session_id]
|
||||
|
||||
if before:
|
||||
conditions.append("created_at < ?")
|
||||
params.append(before)
|
||||
|
||||
if after:
|
||||
conditions.append("created_at > ?")
|
||||
params.append(after)
|
||||
|
||||
where_clause = f"WHERE {' AND '.join(conditions)}"
|
||||
limit_clause = f"LIMIT ?" if limit else ""
|
||||
if limit:
|
||||
params.append(limit)
|
||||
|
||||
async with self._conn.execute(f"""
|
||||
SELECT * FROM {table} {where_clause}
|
||||
ORDER BY created_at ASC {limit_clause}
|
||||
""", params) as cursor:
|
||||
rows = await cursor.fetchall()
|
||||
|
||||
return [
|
||||
ConversationMessage(
|
||||
id=row['id'],
|
||||
session_id=row['session_id'],
|
||||
role=row['role'],
|
||||
content=row['content'],
|
||||
metadata=json.loads(row['metadata']) if row['metadata'] else None,
|
||||
created_at=row['created_at']
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
def get_messages(
|
||||
self,
|
||||
session_id: str,
|
||||
limit: Optional[int] = None,
|
||||
before: Optional[float] = None,
|
||||
after: Optional[float] = None
|
||||
) -> List[ConversationMessage]:
|
||||
"""Sync wrapper for get_messages."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_get_messages(session_id, limit, before, after)
|
||||
)
|
||||
|
||||
async def async_delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
|
||||
"""Delete messages asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
table = f"{self.table_prefix}messages"
|
||||
|
||||
if message_ids is None:
|
||||
cursor = await self._conn.execute(f"""
|
||||
DELETE FROM {table} WHERE session_id = ?
|
||||
""", (session_id,))
|
||||
else:
|
||||
placeholders = ','.join(['?' for _ in message_ids])
|
||||
cursor = await self._conn.execute(f"""
|
||||
DELETE FROM {table} WHERE session_id = ? AND id IN ({placeholders})
|
||||
""", [session_id] + message_ids)
|
||||
|
||||
await self._conn.commit()
|
||||
return cursor.rowcount
|
||||
|
||||
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
|
||||
"""Sync wrapper for delete_messages."""
|
||||
return asyncio.get_event_loop().run_until_complete(
|
||||
self.async_delete_messages(session_id, message_ids)
|
||||
)
|
||||
|
||||
async def async_close(self) -> None:
|
||||
"""Close the connection."""
|
||||
if self._conn:
|
||||
await self._conn.close()
|
||||
self._conn = None
|
||||
self._initialized = False
|
||||
|
||||
def close(self) -> None:
|
||||
"""Sync wrapper for close."""
|
||||
if self._conn:
|
||||
asyncio.get_event_loop().run_until_complete(self.async_close())
|
||||
@@ -0,0 +1,151 @@
|
||||
"""
|
||||
Base interfaces for ConversationStore.
|
||||
|
||||
ConversationStore handles session and message persistence for conversation history.
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
import time
|
||||
import uuid
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConversationMessage:
|
||||
"""A single message in a conversation."""
|
||||
id: str = field(default_factory=lambda: str(uuid.uuid4()))
|
||||
session_id: str = ""
|
||||
role: str = "" # user, assistant, system, tool
|
||||
content: str = ""
|
||||
tool_calls: Optional[List[Dict[str, Any]]] = None
|
||||
tool_call_id: Optional[str] = None
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
created_at: float = field(default_factory=time.time)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": self.id,
|
||||
"session_id": self.session_id,
|
||||
"role": self.role,
|
||||
"content": self.content,
|
||||
"tool_calls": self.tool_calls,
|
||||
"tool_call_id": self.tool_call_id,
|
||||
"metadata": self.metadata,
|
||||
"created_at": self.created_at,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "ConversationMessage":
|
||||
return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__})
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConversationSession:
|
||||
"""A conversation session containing messages."""
|
||||
session_id: str = field(default_factory=lambda: str(uuid.uuid4()))
|
||||
user_id: Optional[str] = None
|
||||
agent_id: Optional[str] = None
|
||||
name: Optional[str] = None
|
||||
state: Optional[Dict[str, Any]] = None
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
created_at: float = field(default_factory=time.time)
|
||||
updated_at: float = field(default_factory=time.time)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"session_id": self.session_id,
|
||||
"user_id": self.user_id,
|
||||
"agent_id": self.agent_id,
|
||||
"name": self.name,
|
||||
"state": self.state,
|
||||
"metadata": self.metadata,
|
||||
"created_at": self.created_at,
|
||||
"updated_at": self.updated_at,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "ConversationSession":
|
||||
return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__})
|
||||
|
||||
|
||||
class ConversationStore(ABC):
|
||||
"""
|
||||
Abstract base class for conversation persistence.
|
||||
|
||||
Implementations handle session and message storage for different backends:
|
||||
- PostgreSQL, MySQL, SQLite (relational)
|
||||
- SingleStore, Supabase, SurrealDB (hybrid)
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def create_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Create a new session."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def get_session(self, session_id: str) -> Optional[ConversationSession]:
|
||||
"""Get a session by ID."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def update_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Update an existing session."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def delete_session(self, session_id: str) -> bool:
|
||||
"""Delete a session and all its messages."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def list_sessions(
|
||||
self,
|
||||
user_id: Optional[str] = None,
|
||||
agent_id: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0
|
||||
) -> List[ConversationSession]:
|
||||
"""List sessions, optionally filtered by user or agent."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
|
||||
"""Add a message to a session."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def get_messages(
|
||||
self,
|
||||
session_id: str,
|
||||
limit: Optional[int] = None,
|
||||
before: Optional[float] = None,
|
||||
after: Optional[float] = None
|
||||
) -> List[ConversationMessage]:
|
||||
"""Get messages from a session."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
|
||||
"""Delete messages. If message_ids is None, delete all messages in session."""
|
||||
raise NotImplementedError
|
||||
|
||||
def upsert_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Create or update a session."""
|
||||
existing = self.get_session(session.session_id)
|
||||
if existing:
|
||||
session.updated_at = time.time()
|
||||
return self.update_session(session)
|
||||
return self.create_session(session)
|
||||
|
||||
@abstractmethod
|
||||
def close(self) -> None:
|
||||
"""Close the store and release resources."""
|
||||
raise NotImplementedError
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.close()
|
||||
return False
|
||||
@@ -0,0 +1,250 @@
|
||||
"""
|
||||
JSON file-based implementation of ConversationStore.
|
||||
|
||||
Simple file-based storage for development and testing.
|
||||
No external dependencies required.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
import threading
|
||||
|
||||
from .base import ConversationStore, ConversationSession, ConversationMessage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class JSONConversationStore(ConversationStore):
|
||||
"""
|
||||
JSON file-based conversation store.
|
||||
|
||||
Stores sessions and messages in JSON files.
|
||||
Suitable for development, testing, and small-scale deployments.
|
||||
|
||||
Example:
|
||||
store = JSONConversationStore(path="./data/conversations")
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
path: str = "./praisonai_conversations",
|
||||
pretty: bool = True,
|
||||
):
|
||||
"""
|
||||
Initialize JSON conversation store.
|
||||
|
||||
Args:
|
||||
path: Directory path for JSON files
|
||||
pretty: Use pretty-printed JSON (default: True)
|
||||
"""
|
||||
self.path = Path(path)
|
||||
self.pretty = pretty
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# Create directory if it doesn't exist
|
||||
self.path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Sessions index file
|
||||
self._index_file = self.path / "_sessions_index.json"
|
||||
self._load_index()
|
||||
|
||||
def _load_index(self):
|
||||
"""Load or create sessions index."""
|
||||
if self._index_file.exists():
|
||||
with open(self._index_file, 'r') as f:
|
||||
self._index = json.load(f)
|
||||
else:
|
||||
self._index = {"sessions": {}}
|
||||
self._save_index()
|
||||
|
||||
def _save_index(self):
|
||||
"""Save sessions index."""
|
||||
with open(self._index_file, 'w') as f:
|
||||
if self.pretty:
|
||||
json.dump(self._index, f, indent=2, default=str)
|
||||
else:
|
||||
json.dump(self._index, f, default=str)
|
||||
|
||||
def _session_file(self, session_id: str) -> Path:
|
||||
"""Get path to session file."""
|
||||
return self.path / f"{session_id}.json"
|
||||
|
||||
def _load_session_data(self, session_id: str) -> Optional[Dict]:
|
||||
"""Load session data from file."""
|
||||
file_path = self._session_file(session_id)
|
||||
if file_path.exists():
|
||||
with open(file_path, 'r') as f:
|
||||
return json.load(f)
|
||||
return None
|
||||
|
||||
def _save_session_data(self, session_id: str, data: Dict):
|
||||
"""Save session data to file."""
|
||||
file_path = self._session_file(session_id)
|
||||
with open(file_path, 'w') as f:
|
||||
if self.pretty:
|
||||
json.dump(data, f, indent=2, default=str)
|
||||
else:
|
||||
json.dump(data, f, default=str)
|
||||
|
||||
def create_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Create a new session."""
|
||||
with self._lock:
|
||||
data = {
|
||||
"session": session.to_dict(),
|
||||
"messages": []
|
||||
}
|
||||
self._save_session_data(session.session_id, data)
|
||||
|
||||
# Update index
|
||||
self._index["sessions"][session.session_id] = {
|
||||
"user_id": session.user_id,
|
||||
"agent_id": session.agent_id,
|
||||
"name": session.name,
|
||||
"created_at": session.created_at,
|
||||
"updated_at": session.updated_at,
|
||||
}
|
||||
self._save_index()
|
||||
|
||||
return session
|
||||
|
||||
def get_session(self, session_id: str) -> Optional[ConversationSession]:
|
||||
"""Get a session by ID."""
|
||||
data = self._load_session_data(session_id)
|
||||
if data and "session" in data:
|
||||
return ConversationSession.from_dict(data["session"])
|
||||
return None
|
||||
|
||||
def update_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Update an existing session."""
|
||||
with self._lock:
|
||||
data = self._load_session_data(session.session_id)
|
||||
if not data:
|
||||
raise ValueError(f"Session {session.session_id} not found")
|
||||
|
||||
session.updated_at = time.time()
|
||||
data["session"] = session.to_dict()
|
||||
self._save_session_data(session.session_id, data)
|
||||
|
||||
# Update index
|
||||
self._index["sessions"][session.session_id].update({
|
||||
"name": session.name,
|
||||
"updated_at": session.updated_at,
|
||||
})
|
||||
self._save_index()
|
||||
|
||||
return session
|
||||
|
||||
def delete_session(self, session_id: str) -> bool:
|
||||
"""Delete a session and all its messages."""
|
||||
with self._lock:
|
||||
file_path = self._session_file(session_id)
|
||||
if file_path.exists():
|
||||
file_path.unlink()
|
||||
|
||||
# Update index
|
||||
if session_id in self._index["sessions"]:
|
||||
del self._index["sessions"][session_id]
|
||||
self._save_index()
|
||||
|
||||
return True
|
||||
return False
|
||||
|
||||
def list_sessions(
|
||||
self,
|
||||
user_id: Optional[str] = None,
|
||||
agent_id: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0
|
||||
) -> List[ConversationSession]:
|
||||
"""List sessions, optionally filtered by user or agent."""
|
||||
sessions = []
|
||||
|
||||
for sid, info in self._index["sessions"].items():
|
||||
if user_id and info.get("user_id") != user_id:
|
||||
continue
|
||||
if agent_id and info.get("agent_id") != agent_id:
|
||||
continue
|
||||
|
||||
session = self.get_session(sid)
|
||||
if session:
|
||||
sessions.append(session)
|
||||
|
||||
# Sort by updated_at descending
|
||||
sessions.sort(key=lambda s: s.updated_at, reverse=True)
|
||||
|
||||
return sessions[offset:offset + limit]
|
||||
|
||||
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
|
||||
"""Add a message to a session."""
|
||||
with self._lock:
|
||||
data = self._load_session_data(session_id)
|
||||
if not data:
|
||||
raise ValueError(f"Session {session_id} not found")
|
||||
|
||||
message.session_id = session_id
|
||||
data["messages"].append(message.to_dict())
|
||||
|
||||
# Update session timestamp
|
||||
data["session"]["updated_at"] = time.time()
|
||||
|
||||
self._save_session_data(session_id, data)
|
||||
|
||||
# Update index
|
||||
if session_id in self._index["sessions"]:
|
||||
self._index["sessions"][session_id]["updated_at"] = time.time()
|
||||
self._save_index()
|
||||
|
||||
return message
|
||||
|
||||
def get_messages(
|
||||
self,
|
||||
session_id: str,
|
||||
limit: Optional[int] = None,
|
||||
before: Optional[float] = None,
|
||||
after: Optional[float] = None
|
||||
) -> List[ConversationMessage]:
|
||||
"""Get messages from a session."""
|
||||
data = self._load_session_data(session_id)
|
||||
if not data:
|
||||
return []
|
||||
|
||||
messages = [ConversationMessage.from_dict(m) for m in data.get("messages", [])]
|
||||
|
||||
# Apply filters
|
||||
if before:
|
||||
messages = [m for m in messages if m.created_at < before]
|
||||
if after:
|
||||
messages = [m for m in messages if m.created_at > after]
|
||||
|
||||
# Sort by created_at
|
||||
messages.sort(key=lambda m: m.created_at)
|
||||
|
||||
if limit:
|
||||
messages = messages[-limit:]
|
||||
|
||||
return messages
|
||||
|
||||
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
|
||||
"""Delete messages. If message_ids is None, delete all messages in session."""
|
||||
with self._lock:
|
||||
data = self._load_session_data(session_id)
|
||||
if not data:
|
||||
return 0
|
||||
|
||||
if message_ids is None:
|
||||
count = len(data["messages"])
|
||||
data["messages"] = []
|
||||
else:
|
||||
original_count = len(data["messages"])
|
||||
data["messages"] = [m for m in data["messages"] if m.get("id") not in message_ids]
|
||||
count = original_count - len(data["messages"])
|
||||
|
||||
self._save_session_data(session_id, data)
|
||||
return count
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the store."""
|
||||
pass # No resources to release
|
||||
@@ -0,0 +1,387 @@
|
||||
"""
|
||||
MySQL implementation of ConversationStore.
|
||||
|
||||
Requires: mysql-connector-python or pymysql
|
||||
Install: pip install mysql-connector-python
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, List, Optional
|
||||
|
||||
from .base import ConversationStore, ConversationSession, ConversationMessage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MySQLConversationStore(ConversationStore):
|
||||
"""
|
||||
MySQL-based conversation store.
|
||||
|
||||
Example:
|
||||
store = MySQLConversationStore(
|
||||
url="mysql://user:password@localhost:3306/praisonai"
|
||||
)
|
||||
"""
|
||||
|
||||
SCHEMA_VERSION = "1.0.0"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: Optional[str] = None,
|
||||
host: str = "localhost",
|
||||
port: int = 3306,
|
||||
database: str = "praisonai",
|
||||
user: str = "root",
|
||||
password: str = "",
|
||||
table_prefix: str = "praison_",
|
||||
auto_create_tables: bool = True,
|
||||
pool_size: int = 5,
|
||||
):
|
||||
"""
|
||||
Initialize MySQL conversation store.
|
||||
|
||||
Args:
|
||||
url: Full connection URL (overrides individual params)
|
||||
host: Database host
|
||||
port: Database port
|
||||
database: Database name
|
||||
user: Database user
|
||||
password: Database password
|
||||
table_prefix: Prefix for table names
|
||||
auto_create_tables: Create tables if they don't exist
|
||||
pool_size: Connection pool size
|
||||
"""
|
||||
try:
|
||||
import mysql.connector
|
||||
from mysql.connector import pooling
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"mysql-connector-python is required for MySQL support. "
|
||||
"Install with: pip install mysql-connector-python"
|
||||
)
|
||||
|
||||
self._mysql = mysql.connector
|
||||
self.table_prefix = table_prefix
|
||||
self.sessions_table = f"{table_prefix}sessions"
|
||||
self.messages_table = f"{table_prefix}messages"
|
||||
|
||||
# Parse URL if provided
|
||||
if url:
|
||||
# Parse mysql://user:pass@host:port/database
|
||||
from urllib.parse import urlparse
|
||||
parsed = urlparse(url)
|
||||
host = parsed.hostname or host
|
||||
port = parsed.port or port
|
||||
database = parsed.path.lstrip("/") or database
|
||||
user = parsed.username or user
|
||||
password = parsed.password or password
|
||||
|
||||
# Create connection pool
|
||||
self._pool = pooling.MySQLConnectionPool(
|
||||
pool_name="praison_mysql_pool",
|
||||
pool_size=pool_size,
|
||||
host=host,
|
||||
port=port,
|
||||
database=database,
|
||||
user=user,
|
||||
password=password,
|
||||
)
|
||||
|
||||
if auto_create_tables:
|
||||
self._create_tables()
|
||||
|
||||
def _get_conn(self):
|
||||
"""Get a connection from the pool."""
|
||||
return self._pool.get_connection()
|
||||
|
||||
def _create_tables(self) -> None:
|
||||
"""Create tables if they don't exist."""
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
|
||||
# Sessions table
|
||||
cur.execute(f"""
|
||||
CREATE TABLE IF NOT EXISTS {self.sessions_table} (
|
||||
session_id VARCHAR(255) PRIMARY KEY,
|
||||
user_id VARCHAR(255),
|
||||
agent_id VARCHAR(255),
|
||||
name VARCHAR(255),
|
||||
state JSON,
|
||||
metadata JSON,
|
||||
created_at DOUBLE,
|
||||
updated_at DOUBLE,
|
||||
INDEX idx_user (user_id),
|
||||
INDEX idx_agent (agent_id),
|
||||
INDEX idx_updated (updated_at DESC)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
""")
|
||||
|
||||
# Messages table
|
||||
cur.execute(f"""
|
||||
CREATE TABLE IF NOT EXISTS {self.messages_table} (
|
||||
id VARCHAR(255) PRIMARY KEY,
|
||||
session_id VARCHAR(255) NOT NULL,
|
||||
role VARCHAR(50) NOT NULL,
|
||||
content TEXT,
|
||||
tool_calls JSON,
|
||||
tool_call_id VARCHAR(255),
|
||||
metadata JSON,
|
||||
created_at DOUBLE,
|
||||
INDEX idx_session (session_id),
|
||||
INDEX idx_created (session_id, created_at),
|
||||
FOREIGN KEY (session_id) REFERENCES {self.sessions_table}(session_id) ON DELETE CASCADE
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
|
||||
""")
|
||||
|
||||
conn.commit()
|
||||
logger.info(f"MySQL tables created: {self.sessions_table}, {self.messages_table}")
|
||||
finally:
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
def create_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Create a new session."""
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"""
|
||||
INSERT INTO {self.sessions_table}
|
||||
(session_id, user_id, agent_id, name, state, metadata, created_at, updated_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
|
||||
""", (
|
||||
session.session_id,
|
||||
session.user_id,
|
||||
session.agent_id,
|
||||
session.name,
|
||||
json.dumps(session.state) if session.state else None,
|
||||
json.dumps(session.metadata) if session.metadata else None,
|
||||
session.created_at,
|
||||
session.updated_at,
|
||||
))
|
||||
conn.commit()
|
||||
return session
|
||||
finally:
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
def get_session(self, session_id: str) -> Optional[ConversationSession]:
|
||||
"""Get a session by ID."""
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
cur = conn.cursor(dictionary=True)
|
||||
cur.execute(f"""
|
||||
SELECT * FROM {self.sessions_table} WHERE session_id = %s
|
||||
""", (session_id,))
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
return ConversationSession(
|
||||
session_id=row["session_id"],
|
||||
user_id=row["user_id"],
|
||||
agent_id=row["agent_id"],
|
||||
name=row["name"],
|
||||
state=row["state"] if isinstance(row["state"], dict) else (json.loads(row["state"]) if row["state"] else None),
|
||||
metadata=row["metadata"] if isinstance(row["metadata"], dict) else (json.loads(row["metadata"]) if row["metadata"] else None),
|
||||
created_at=row["created_at"],
|
||||
updated_at=row["updated_at"],
|
||||
)
|
||||
finally:
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
def update_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Update an existing session."""
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"""
|
||||
UPDATE {self.sessions_table}
|
||||
SET user_id = %s, agent_id = %s, name = %s,
|
||||
state = %s, metadata = %s, updated_at = %s
|
||||
WHERE session_id = %s
|
||||
""", (
|
||||
session.user_id,
|
||||
session.agent_id,
|
||||
session.name,
|
||||
json.dumps(session.state) if session.state else None,
|
||||
json.dumps(session.metadata) if session.metadata else None,
|
||||
session.updated_at,
|
||||
session.session_id,
|
||||
))
|
||||
conn.commit()
|
||||
return session
|
||||
finally:
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
def delete_session(self, session_id: str) -> bool:
|
||||
"""Delete a session and all its messages."""
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"""
|
||||
DELETE FROM {self.sessions_table} WHERE session_id = %s
|
||||
""", (session_id,))
|
||||
deleted = cur.rowcount > 0
|
||||
conn.commit()
|
||||
return deleted
|
||||
finally:
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
def list_sessions(
|
||||
self,
|
||||
user_id: Optional[str] = None,
|
||||
agent_id: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0
|
||||
) -> List[ConversationSession]:
|
||||
"""List sessions."""
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
cur = conn.cursor(dictionary=True)
|
||||
|
||||
conditions = []
|
||||
params: List[Any] = []
|
||||
|
||||
if user_id:
|
||||
conditions.append("user_id = %s")
|
||||
params.append(user_id)
|
||||
if agent_id:
|
||||
conditions.append("agent_id = %s")
|
||||
params.append(agent_id)
|
||||
|
||||
where_clause = ""
|
||||
if conditions:
|
||||
where_clause = "WHERE " + " AND ".join(conditions)
|
||||
|
||||
params.extend([limit, offset])
|
||||
|
||||
cur.execute(f"""
|
||||
SELECT * FROM {self.sessions_table}
|
||||
{where_clause}
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT %s OFFSET %s
|
||||
""", params)
|
||||
|
||||
return [
|
||||
ConversationSession(
|
||||
session_id=row["session_id"],
|
||||
user_id=row["user_id"],
|
||||
agent_id=row["agent_id"],
|
||||
name=row["name"],
|
||||
state=row["state"] if isinstance(row["state"], dict) else (json.loads(row["state"]) if row["state"] else None),
|
||||
metadata=row["metadata"] if isinstance(row["metadata"], dict) else (json.loads(row["metadata"]) if row["metadata"] else None),
|
||||
created_at=row["created_at"],
|
||||
updated_at=row["updated_at"],
|
||||
)
|
||||
for row in cur.fetchall()
|
||||
]
|
||||
finally:
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
|
||||
"""Add a message to a session."""
|
||||
message.session_id = session_id
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"""
|
||||
INSERT INTO {self.messages_table}
|
||||
(id, session_id, role, content, tool_calls, tool_call_id, metadata, created_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
|
||||
""", (
|
||||
message.id,
|
||||
session_id,
|
||||
message.role,
|
||||
message.content,
|
||||
json.dumps(message.tool_calls) if message.tool_calls else None,
|
||||
message.tool_call_id,
|
||||
json.dumps(message.metadata) if message.metadata else None,
|
||||
message.created_at,
|
||||
))
|
||||
conn.commit()
|
||||
return message
|
||||
finally:
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
def get_messages(
|
||||
self,
|
||||
session_id: str,
|
||||
limit: Optional[int] = None,
|
||||
before: Optional[float] = None,
|
||||
after: Optional[float] = None
|
||||
) -> List[ConversationMessage]:
|
||||
"""Get messages from a session."""
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
cur = conn.cursor(dictionary=True)
|
||||
|
||||
conditions = ["session_id = %s"]
|
||||
params: List[Any] = [session_id]
|
||||
|
||||
if before:
|
||||
conditions.append("created_at < %s")
|
||||
params.append(before)
|
||||
if after:
|
||||
conditions.append("created_at > %s")
|
||||
params.append(after)
|
||||
|
||||
where_clause = "WHERE " + " AND ".join(conditions)
|
||||
limit_clause = f"LIMIT {limit}" if limit else ""
|
||||
|
||||
cur.execute(f"""
|
||||
SELECT * FROM {self.messages_table}
|
||||
{where_clause}
|
||||
ORDER BY created_at ASC
|
||||
{limit_clause}
|
||||
""", params)
|
||||
|
||||
return [
|
||||
ConversationMessage(
|
||||
id=row["id"],
|
||||
session_id=row["session_id"],
|
||||
role=row["role"],
|
||||
content=row["content"],
|
||||
tool_calls=row["tool_calls"] if isinstance(row["tool_calls"], list) else (json.loads(row["tool_calls"]) if row["tool_calls"] else None),
|
||||
tool_call_id=row["tool_call_id"],
|
||||
metadata=row["metadata"] if isinstance(row["metadata"], dict) else (json.loads(row["metadata"]) if row["metadata"] else None),
|
||||
created_at=row["created_at"],
|
||||
)
|
||||
for row in cur.fetchall()
|
||||
]
|
||||
finally:
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
|
||||
"""Delete messages."""
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
|
||||
if message_ids:
|
||||
placeholders = ",".join(["%s"] * len(message_ids))
|
||||
cur.execute(f"""
|
||||
DELETE FROM {self.messages_table}
|
||||
WHERE session_id = %s AND id IN ({placeholders})
|
||||
""", [session_id] + message_ids)
|
||||
else:
|
||||
cur.execute(f"""
|
||||
DELETE FROM {self.messages_table} WHERE session_id = %s
|
||||
""", (session_id,))
|
||||
|
||||
deleted = cur.rowcount
|
||||
conn.commit()
|
||||
return deleted
|
||||
finally:
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the store."""
|
||||
pass # Pool handles cleanup
|
||||
@@ -0,0 +1,401 @@
|
||||
"""
|
||||
PostgreSQL implementation of ConversationStore.
|
||||
|
||||
Requires: psycopg2-binary or psycopg2
|
||||
Install: pip install psycopg2-binary
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import ConversationStore, ConversationSession, ConversationMessage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PostgresConversationStore(ConversationStore):
|
||||
"""
|
||||
PostgreSQL-based conversation store.
|
||||
|
||||
Connection URL format: postgresql://user:password@host:port/database
|
||||
|
||||
Example:
|
||||
store = PostgresConversationStore(
|
||||
url="postgresql://postgres:password@localhost:5432/praisonai"
|
||||
)
|
||||
"""
|
||||
|
||||
SCHEMA_VERSION = "1.0.0"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: Optional[str] = None,
|
||||
host: str = "localhost",
|
||||
port: int = 5432,
|
||||
database: str = "praisonai",
|
||||
user: str = "postgres",
|
||||
password: str = "",
|
||||
schema: str = "public",
|
||||
table_prefix: str = "praison_",
|
||||
auto_create_tables: bool = True,
|
||||
pool_size: int = 5,
|
||||
):
|
||||
"""
|
||||
Initialize PostgreSQL conversation store.
|
||||
|
||||
Args:
|
||||
url: Full connection URL (overrides individual params)
|
||||
host: Database host
|
||||
port: Database port
|
||||
database: Database name
|
||||
user: Database user
|
||||
password: Database password
|
||||
schema: PostgreSQL schema
|
||||
table_prefix: Prefix for table names
|
||||
auto_create_tables: Create tables if they don't exist
|
||||
pool_size: Connection pool size
|
||||
"""
|
||||
try:
|
||||
import psycopg2
|
||||
from psycopg2 import pool as pg_pool
|
||||
from psycopg2.extras import RealDictCursor
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"psycopg2 is required for PostgreSQL support. "
|
||||
"Install with: pip install psycopg2-binary"
|
||||
)
|
||||
|
||||
self._psycopg2 = psycopg2
|
||||
self._RealDictCursor = RealDictCursor
|
||||
|
||||
self.schema = schema
|
||||
self.table_prefix = table_prefix
|
||||
self.sessions_table = f"{schema}.{table_prefix}sessions"
|
||||
self.messages_table = f"{schema}.{table_prefix}messages"
|
||||
|
||||
# Build connection params
|
||||
if url:
|
||||
self._pool = pg_pool.ThreadedConnectionPool(1, pool_size, url)
|
||||
else:
|
||||
self._pool = pg_pool.ThreadedConnectionPool(
|
||||
1, pool_size,
|
||||
host=host,
|
||||
port=port,
|
||||
database=database,
|
||||
user=user,
|
||||
password=password,
|
||||
)
|
||||
|
||||
if auto_create_tables:
|
||||
self._create_tables()
|
||||
|
||||
def _get_conn(self):
|
||||
"""Get a connection from the pool."""
|
||||
return self._pool.getconn()
|
||||
|
||||
def _put_conn(self, conn):
|
||||
"""Return a connection to the pool."""
|
||||
self._pool.putconn(conn)
|
||||
|
||||
def _create_tables(self) -> None:
|
||||
"""Create tables if they don't exist."""
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
# Create schema if not exists
|
||||
cur.execute(f"CREATE SCHEMA IF NOT EXISTS {self.schema}")
|
||||
|
||||
# Sessions table
|
||||
cur.execute(f"""
|
||||
CREATE TABLE IF NOT EXISTS {self.sessions_table} (
|
||||
session_id VARCHAR(255) PRIMARY KEY,
|
||||
user_id VARCHAR(255),
|
||||
agent_id VARCHAR(255),
|
||||
name VARCHAR(255),
|
||||
state JSONB,
|
||||
metadata JSONB,
|
||||
created_at DOUBLE PRECISION,
|
||||
updated_at DOUBLE PRECISION
|
||||
)
|
||||
""")
|
||||
|
||||
# Sessions indexes
|
||||
cur.execute(f"""
|
||||
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}sessions_user
|
||||
ON {self.sessions_table}(user_id)
|
||||
""")
|
||||
cur.execute(f"""
|
||||
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}sessions_agent
|
||||
ON {self.sessions_table}(agent_id)
|
||||
""")
|
||||
cur.execute(f"""
|
||||
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}sessions_updated
|
||||
ON {self.sessions_table}(updated_at DESC)
|
||||
""")
|
||||
|
||||
# Messages table
|
||||
cur.execute(f"""
|
||||
CREATE TABLE IF NOT EXISTS {self.messages_table} (
|
||||
id VARCHAR(255) PRIMARY KEY,
|
||||
session_id VARCHAR(255) NOT NULL
|
||||
REFERENCES {self.sessions_table}(session_id) ON DELETE CASCADE,
|
||||
role VARCHAR(50) NOT NULL,
|
||||
content TEXT,
|
||||
tool_calls JSONB,
|
||||
tool_call_id VARCHAR(255),
|
||||
metadata JSONB,
|
||||
created_at DOUBLE PRECISION DEFAULT EXTRACT(EPOCH FROM NOW())
|
||||
)
|
||||
""")
|
||||
|
||||
# Messages indexes
|
||||
cur.execute(f"""
|
||||
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}messages_session
|
||||
ON {self.messages_table}(session_id)
|
||||
""")
|
||||
cur.execute(f"""
|
||||
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}messages_created
|
||||
ON {self.messages_table}(session_id, created_at)
|
||||
""")
|
||||
|
||||
conn.commit()
|
||||
logger.info(f"PostgreSQL tables created: {self.sessions_table}, {self.messages_table}")
|
||||
finally:
|
||||
self._put_conn(conn)
|
||||
|
||||
def create_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Create a new session."""
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(f"""
|
||||
INSERT INTO {self.sessions_table}
|
||||
(session_id, user_id, agent_id, name, state, metadata, created_at, updated_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
|
||||
RETURNING session_id
|
||||
""", (
|
||||
session.session_id,
|
||||
session.user_id,
|
||||
session.agent_id,
|
||||
session.name,
|
||||
json.dumps(session.state) if session.state else None,
|
||||
json.dumps(session.metadata) if session.metadata else None,
|
||||
session.created_at,
|
||||
session.updated_at,
|
||||
))
|
||||
conn.commit()
|
||||
return session
|
||||
finally:
|
||||
self._put_conn(conn)
|
||||
|
||||
def get_session(self, session_id: str) -> Optional[ConversationSession]:
|
||||
"""Get a session by ID."""
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
with conn.cursor(cursor_factory=self._RealDictCursor) as cur:
|
||||
cur.execute(f"""
|
||||
SELECT * FROM {self.sessions_table} WHERE session_id = %s
|
||||
""", (session_id,))
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
return ConversationSession(
|
||||
session_id=row["session_id"],
|
||||
user_id=row["user_id"],
|
||||
agent_id=row["agent_id"],
|
||||
name=row["name"],
|
||||
state=row["state"],
|
||||
metadata=row["metadata"],
|
||||
created_at=row["created_at"],
|
||||
updated_at=row["updated_at"],
|
||||
)
|
||||
finally:
|
||||
self._put_conn(conn)
|
||||
|
||||
def update_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Update an existing session."""
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(f"""
|
||||
UPDATE {self.sessions_table}
|
||||
SET user_id = %s, agent_id = %s, name = %s,
|
||||
state = %s, metadata = %s, updated_at = %s
|
||||
WHERE session_id = %s
|
||||
""", (
|
||||
session.user_id,
|
||||
session.agent_id,
|
||||
session.name,
|
||||
json.dumps(session.state) if session.state else None,
|
||||
json.dumps(session.metadata) if session.metadata else None,
|
||||
session.updated_at,
|
||||
session.session_id,
|
||||
))
|
||||
conn.commit()
|
||||
return session
|
||||
finally:
|
||||
self._put_conn(conn)
|
||||
|
||||
def delete_session(self, session_id: str) -> bool:
|
||||
"""Delete a session and all its messages."""
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(f"""
|
||||
DELETE FROM {self.sessions_table} WHERE session_id = %s
|
||||
""", (session_id,))
|
||||
deleted = cur.rowcount > 0
|
||||
conn.commit()
|
||||
return deleted
|
||||
finally:
|
||||
self._put_conn(conn)
|
||||
|
||||
def list_sessions(
|
||||
self,
|
||||
user_id: Optional[str] = None,
|
||||
agent_id: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0
|
||||
) -> List[ConversationSession]:
|
||||
"""List sessions, optionally filtered by user or agent."""
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
with conn.cursor(cursor_factory=self._RealDictCursor) as cur:
|
||||
conditions = []
|
||||
params = []
|
||||
|
||||
if user_id:
|
||||
conditions.append("user_id = %s")
|
||||
params.append(user_id)
|
||||
if agent_id:
|
||||
conditions.append("agent_id = %s")
|
||||
params.append(agent_id)
|
||||
|
||||
where_clause = ""
|
||||
if conditions:
|
||||
where_clause = "WHERE " + " AND ".join(conditions)
|
||||
|
||||
params.extend([limit, offset])
|
||||
|
||||
cur.execute(f"""
|
||||
SELECT * FROM {self.sessions_table}
|
||||
{where_clause}
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT %s OFFSET %s
|
||||
""", params)
|
||||
|
||||
return [
|
||||
ConversationSession(
|
||||
session_id=row["session_id"],
|
||||
user_id=row["user_id"],
|
||||
agent_id=row["agent_id"],
|
||||
name=row["name"],
|
||||
state=row["state"],
|
||||
metadata=row["metadata"],
|
||||
created_at=row["created_at"],
|
||||
updated_at=row["updated_at"],
|
||||
)
|
||||
for row in cur.fetchall()
|
||||
]
|
||||
finally:
|
||||
self._put_conn(conn)
|
||||
|
||||
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
|
||||
"""Add a message to a session."""
|
||||
message.session_id = session_id
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(f"""
|
||||
INSERT INTO {self.messages_table}
|
||||
(id, session_id, role, content, tool_calls, tool_call_id, metadata, created_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
|
||||
""", (
|
||||
message.id,
|
||||
session_id,
|
||||
message.role,
|
||||
message.content,
|
||||
json.dumps(message.tool_calls) if message.tool_calls else None,
|
||||
message.tool_call_id,
|
||||
json.dumps(message.metadata) if message.metadata else None,
|
||||
message.created_at,
|
||||
))
|
||||
conn.commit()
|
||||
return message
|
||||
finally:
|
||||
self._put_conn(conn)
|
||||
|
||||
def get_messages(
|
||||
self,
|
||||
session_id: str,
|
||||
limit: Optional[int] = None,
|
||||
before: Optional[float] = None,
|
||||
after: Optional[float] = None
|
||||
) -> List[ConversationMessage]:
|
||||
"""Get messages from a session."""
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
with conn.cursor(cursor_factory=self._RealDictCursor) as cur:
|
||||
conditions = ["session_id = %s"]
|
||||
params: List[Any] = [session_id]
|
||||
|
||||
if before:
|
||||
conditions.append("created_at < %s")
|
||||
params.append(before)
|
||||
if after:
|
||||
conditions.append("created_at > %s")
|
||||
params.append(after)
|
||||
|
||||
where_clause = "WHERE " + " AND ".join(conditions)
|
||||
limit_clause = f"LIMIT {limit}" if limit else ""
|
||||
|
||||
cur.execute(f"""
|
||||
SELECT * FROM {self.messages_table}
|
||||
{where_clause}
|
||||
ORDER BY created_at ASC
|
||||
{limit_clause}
|
||||
""", params)
|
||||
|
||||
return [
|
||||
ConversationMessage(
|
||||
id=row["id"],
|
||||
session_id=row["session_id"],
|
||||
role=row["role"],
|
||||
content=row["content"],
|
||||
tool_calls=row["tool_calls"],
|
||||
tool_call_id=row["tool_call_id"],
|
||||
metadata=row["metadata"],
|
||||
created_at=row["created_at"],
|
||||
)
|
||||
for row in cur.fetchall()
|
||||
]
|
||||
finally:
|
||||
self._put_conn(conn)
|
||||
|
||||
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
|
||||
"""Delete messages. If message_ids is None, delete all messages in session."""
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
if message_ids:
|
||||
placeholders = ",".join(["%s"] * len(message_ids))
|
||||
cur.execute(f"""
|
||||
DELETE FROM {self.messages_table}
|
||||
WHERE session_id = %s AND id IN ({placeholders})
|
||||
""", [session_id] + message_ids)
|
||||
else:
|
||||
cur.execute(f"""
|
||||
DELETE FROM {self.messages_table} WHERE session_id = %s
|
||||
""", (session_id,))
|
||||
deleted = cur.rowcount
|
||||
conn.commit()
|
||||
return deleted
|
||||
finally:
|
||||
self._put_conn(conn)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the store and release resources."""
|
||||
if self._pool:
|
||||
self._pool.closeall()
|
||||
self._pool = None
|
||||
@@ -0,0 +1,240 @@
|
||||
"""
|
||||
SingleStore implementation of ConversationStore.
|
||||
|
||||
Requires: singlestoredb
|
||||
Install: pip install singlestoredb
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, List, Optional
|
||||
|
||||
from .base import ConversationStore, ConversationSession, ConversationMessage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SingleStoreConversationStore(ConversationStore):
|
||||
"""
|
||||
SingleStore-based conversation store.
|
||||
|
||||
SingleStore is MySQL-compatible with additional vector capabilities.
|
||||
|
||||
Example:
|
||||
store = SingleStoreConversationStore(
|
||||
url="singlestoredb://user:password@host:3306/database"
|
||||
)
|
||||
"""
|
||||
|
||||
SCHEMA_VERSION = "1.0.0"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: Optional[str] = None,
|
||||
host: str = "localhost",
|
||||
port: int = 3306,
|
||||
database: str = "praisonai",
|
||||
user: str = "root",
|
||||
password: str = "",
|
||||
table_prefix: str = "praison_",
|
||||
auto_create_tables: bool = True,
|
||||
):
|
||||
try:
|
||||
import singlestoredb as s2
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"singlestoredb is required for SingleStore support. "
|
||||
"Install with: pip install singlestoredb"
|
||||
)
|
||||
|
||||
self._s2 = s2
|
||||
self.table_prefix = table_prefix
|
||||
self.sessions_table = f"{table_prefix}sessions"
|
||||
self.messages_table = f"{table_prefix}messages"
|
||||
|
||||
if url:
|
||||
self._conn = s2.connect(url)
|
||||
else:
|
||||
self._conn = s2.connect(
|
||||
host=host,
|
||||
port=port,
|
||||
database=database,
|
||||
user=user,
|
||||
password=password,
|
||||
)
|
||||
|
||||
if auto_create_tables:
|
||||
self._create_tables()
|
||||
|
||||
def _create_tables(self) -> None:
|
||||
"""Create tables if they don't exist."""
|
||||
cur = self._conn.cursor()
|
||||
|
||||
cur.execute(f"""
|
||||
CREATE TABLE IF NOT EXISTS {self.sessions_table} (
|
||||
session_id VARCHAR(255) PRIMARY KEY,
|
||||
user_id VARCHAR(255),
|
||||
agent_id VARCHAR(255),
|
||||
name VARCHAR(255),
|
||||
state JSON,
|
||||
metadata JSON,
|
||||
created_at DOUBLE,
|
||||
updated_at DOUBLE,
|
||||
KEY idx_user (user_id),
|
||||
KEY idx_agent (agent_id)
|
||||
)
|
||||
""")
|
||||
|
||||
cur.execute(f"""
|
||||
CREATE TABLE IF NOT EXISTS {self.messages_table} (
|
||||
id VARCHAR(255) PRIMARY KEY,
|
||||
session_id VARCHAR(255) NOT NULL,
|
||||
role VARCHAR(50) NOT NULL,
|
||||
content TEXT,
|
||||
tool_calls JSON,
|
||||
tool_call_id VARCHAR(255),
|
||||
metadata JSON,
|
||||
created_at DOUBLE,
|
||||
KEY idx_session (session_id),
|
||||
KEY idx_created (session_id, created_at)
|
||||
)
|
||||
""")
|
||||
|
||||
self._conn.commit()
|
||||
logger.info(f"SingleStore tables created")
|
||||
|
||||
def create_session(self, session: ConversationSession) -> ConversationSession:
|
||||
cur = self._conn.cursor()
|
||||
cur.execute(f"""
|
||||
INSERT INTO {self.sessions_table}
|
||||
(session_id, user_id, agent_id, name, state, metadata, created_at, updated_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
|
||||
""", (
|
||||
session.session_id, session.user_id, session.agent_id, session.name,
|
||||
json.dumps(session.state) if session.state else None,
|
||||
json.dumps(session.metadata) if session.metadata else None,
|
||||
session.created_at, session.updated_at,
|
||||
))
|
||||
self._conn.commit()
|
||||
return session
|
||||
|
||||
def get_session(self, session_id: str) -> Optional[ConversationSession]:
|
||||
cur = self._conn.cursor()
|
||||
cur.execute(f"SELECT * FROM {self.sessions_table} WHERE session_id = %s", (session_id,))
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
cols = [d[0] for d in cur.description]
|
||||
data = dict(zip(cols, row))
|
||||
return ConversationSession(
|
||||
session_id=data["session_id"], user_id=data["user_id"],
|
||||
agent_id=data["agent_id"], name=data["name"],
|
||||
state=json.loads(data["state"]) if data["state"] else None,
|
||||
metadata=json.loads(data["metadata"]) if data["metadata"] else None,
|
||||
created_at=data["created_at"], updated_at=data["updated_at"],
|
||||
)
|
||||
|
||||
def update_session(self, session: ConversationSession) -> ConversationSession:
|
||||
cur = self._conn.cursor()
|
||||
cur.execute(f"""
|
||||
UPDATE {self.sessions_table}
|
||||
SET user_id=%s, agent_id=%s, name=%s, state=%s, metadata=%s, updated_at=%s
|
||||
WHERE session_id=%s
|
||||
""", (
|
||||
session.user_id, session.agent_id, session.name,
|
||||
json.dumps(session.state) if session.state else None,
|
||||
json.dumps(session.metadata) if session.metadata else None,
|
||||
session.updated_at, session.session_id,
|
||||
))
|
||||
self._conn.commit()
|
||||
return session
|
||||
|
||||
def delete_session(self, session_id: str) -> bool:
|
||||
cur = self._conn.cursor()
|
||||
cur.execute(f"DELETE FROM {self.messages_table} WHERE session_id = %s", (session_id,))
|
||||
cur.execute(f"DELETE FROM {self.sessions_table} WHERE session_id = %s", (session_id,))
|
||||
self._conn.commit()
|
||||
return cur.rowcount > 0
|
||||
|
||||
def list_sessions(self, user_id: Optional[str] = None, agent_id: Optional[str] = None,
|
||||
limit: int = 100, offset: int = 0) -> List[ConversationSession]:
|
||||
cur = self._conn.cursor()
|
||||
conditions, params = [], []
|
||||
if user_id:
|
||||
conditions.append("user_id = %s")
|
||||
params.append(user_id)
|
||||
if agent_id:
|
||||
conditions.append("agent_id = %s")
|
||||
params.append(agent_id)
|
||||
where = "WHERE " + " AND ".join(conditions) if conditions else ""
|
||||
params.extend([limit, offset])
|
||||
cur.execute(f"SELECT * FROM {self.sessions_table} {where} ORDER BY updated_at DESC LIMIT %s OFFSET %s", params)
|
||||
cols = [d[0] for d in cur.description]
|
||||
return [ConversationSession(
|
||||
session_id=dict(zip(cols, r))["session_id"],
|
||||
user_id=dict(zip(cols, r))["user_id"],
|
||||
agent_id=dict(zip(cols, r))["agent_id"],
|
||||
name=dict(zip(cols, r))["name"],
|
||||
state=json.loads(dict(zip(cols, r))["state"]) if dict(zip(cols, r))["state"] else None,
|
||||
metadata=json.loads(dict(zip(cols, r))["metadata"]) if dict(zip(cols, r))["metadata"] else None,
|
||||
created_at=dict(zip(cols, r))["created_at"],
|
||||
updated_at=dict(zip(cols, r))["updated_at"],
|
||||
) for r in cur.fetchall()]
|
||||
|
||||
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
|
||||
message.session_id = session_id
|
||||
cur = self._conn.cursor()
|
||||
cur.execute(f"""
|
||||
INSERT INTO {self.messages_table}
|
||||
(id, session_id, role, content, tool_calls, tool_call_id, metadata, created_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
|
||||
""", (
|
||||
message.id, session_id, message.role, message.content,
|
||||
json.dumps(message.tool_calls) if message.tool_calls else None,
|
||||
message.tool_call_id,
|
||||
json.dumps(message.metadata) if message.metadata else None,
|
||||
message.created_at,
|
||||
))
|
||||
self._conn.commit()
|
||||
return message
|
||||
|
||||
def get_messages(self, session_id: str, limit: Optional[int] = None,
|
||||
before: Optional[float] = None, after: Optional[float] = None) -> List[ConversationMessage]:
|
||||
cur = self._conn.cursor()
|
||||
conditions, params = ["session_id = %s"], [session_id]
|
||||
if before:
|
||||
conditions.append("created_at < %s")
|
||||
params.append(before)
|
||||
if after:
|
||||
conditions.append("created_at > %s")
|
||||
params.append(after)
|
||||
where = "WHERE " + " AND ".join(conditions)
|
||||
limit_clause = f"LIMIT {limit}" if limit else ""
|
||||
cur.execute(f"SELECT * FROM {self.messages_table} {where} ORDER BY created_at ASC {limit_clause}", params)
|
||||
cols = [d[0] for d in cur.description]
|
||||
return [ConversationMessage(
|
||||
id=dict(zip(cols, r))["id"],
|
||||
session_id=dict(zip(cols, r))["session_id"],
|
||||
role=dict(zip(cols, r))["role"],
|
||||
content=dict(zip(cols, r))["content"],
|
||||
tool_calls=json.loads(dict(zip(cols, r))["tool_calls"]) if dict(zip(cols, r))["tool_calls"] else None,
|
||||
tool_call_id=dict(zip(cols, r))["tool_call_id"],
|
||||
metadata=json.loads(dict(zip(cols, r))["metadata"]) if dict(zip(cols, r))["metadata"] else None,
|
||||
created_at=dict(zip(cols, r))["created_at"],
|
||||
) for r in cur.fetchall()]
|
||||
|
||||
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
|
||||
cur = self._conn.cursor()
|
||||
if message_ids:
|
||||
placeholders = ",".join(["%s"] * len(message_ids))
|
||||
cur.execute(f"DELETE FROM {self.messages_table} WHERE session_id = %s AND id IN ({placeholders})",
|
||||
[session_id] + message_ids)
|
||||
else:
|
||||
cur.execute(f"DELETE FROM {self.messages_table} WHERE session_id = %s", (session_id,))
|
||||
self._conn.commit()
|
||||
return cur.rowcount
|
||||
|
||||
def close(self) -> None:
|
||||
if self._conn:
|
||||
self._conn.close()
|
||||
self._conn = None
|
||||
@@ -0,0 +1,341 @@
|
||||
"""
|
||||
SQLite implementation of ConversationStore.
|
||||
|
||||
Zero external dependencies - uses built-in sqlite3 module.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
import threading
|
||||
from typing import Any, List, Optional
|
||||
|
||||
from .base import ConversationStore, ConversationSession, ConversationMessage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SQLiteConversationStore(ConversationStore):
|
||||
"""
|
||||
SQLite-based conversation store.
|
||||
|
||||
Zero external dependencies - uses Python's built-in sqlite3.
|
||||
|
||||
Example:
|
||||
store = SQLiteConversationStore(
|
||||
path="./praisonai.db"
|
||||
)
|
||||
"""
|
||||
|
||||
SCHEMA_VERSION = "1.0.0"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
path: str = "praisonai_conversations.db",
|
||||
table_prefix: str = "praison_",
|
||||
auto_create_tables: bool = True,
|
||||
check_same_thread: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize SQLite conversation store.
|
||||
|
||||
Args:
|
||||
path: Path to SQLite database file
|
||||
table_prefix: Prefix for table names
|
||||
auto_create_tables: Create tables if they don't exist
|
||||
check_same_thread: SQLite check_same_thread parameter
|
||||
"""
|
||||
self.path = path
|
||||
self.table_prefix = table_prefix
|
||||
self.sessions_table = f"{table_prefix}sessions"
|
||||
self.messages_table = f"{table_prefix}messages"
|
||||
self._check_same_thread = check_same_thread
|
||||
self._local = threading.local()
|
||||
|
||||
if auto_create_tables:
|
||||
self._create_tables()
|
||||
|
||||
def _get_conn(self) -> sqlite3.Connection:
|
||||
"""Get thread-local connection."""
|
||||
if not hasattr(self._local, "conn") or self._local.conn is None:
|
||||
self._local.conn = sqlite3.connect(
|
||||
self.path,
|
||||
check_same_thread=self._check_same_thread
|
||||
)
|
||||
self._local.conn.row_factory = sqlite3.Row
|
||||
return self._local.conn
|
||||
|
||||
def _create_tables(self) -> None:
|
||||
"""Create tables if they don't exist."""
|
||||
conn = self._get_conn()
|
||||
cur = conn.cursor()
|
||||
|
||||
# Sessions table
|
||||
cur.execute(f"""
|
||||
CREATE TABLE IF NOT EXISTS {self.sessions_table} (
|
||||
session_id TEXT PRIMARY KEY,
|
||||
user_id TEXT,
|
||||
agent_id TEXT,
|
||||
name TEXT,
|
||||
state TEXT,
|
||||
metadata TEXT,
|
||||
created_at REAL,
|
||||
updated_at REAL
|
||||
)
|
||||
""")
|
||||
|
||||
# Sessions indexes
|
||||
cur.execute(f"""
|
||||
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}sessions_user
|
||||
ON {self.sessions_table}(user_id)
|
||||
""")
|
||||
cur.execute(f"""
|
||||
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}sessions_agent
|
||||
ON {self.sessions_table}(agent_id)
|
||||
""")
|
||||
|
||||
# Messages table
|
||||
cur.execute(f"""
|
||||
CREATE TABLE IF NOT EXISTS {self.messages_table} (
|
||||
id TEXT PRIMARY KEY,
|
||||
session_id TEXT NOT NULL,
|
||||
role TEXT NOT NULL,
|
||||
content TEXT,
|
||||
tool_calls TEXT,
|
||||
tool_call_id TEXT,
|
||||
metadata TEXT,
|
||||
created_at REAL,
|
||||
FOREIGN KEY (session_id) REFERENCES {self.sessions_table}(session_id) ON DELETE CASCADE
|
||||
)
|
||||
""")
|
||||
|
||||
# Messages indexes
|
||||
cur.execute(f"""
|
||||
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}messages_session
|
||||
ON {self.messages_table}(session_id)
|
||||
""")
|
||||
cur.execute(f"""
|
||||
CREATE INDEX IF NOT EXISTS idx_{self.table_prefix}messages_created
|
||||
ON {self.messages_table}(session_id, created_at)
|
||||
""")
|
||||
|
||||
# Enable foreign keys
|
||||
cur.execute("PRAGMA foreign_keys = ON")
|
||||
|
||||
conn.commit()
|
||||
logger.info(f"SQLite tables created: {self.sessions_table}, {self.messages_table}")
|
||||
|
||||
def create_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Create a new session."""
|
||||
conn = self._get_conn()
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"""
|
||||
INSERT INTO {self.sessions_table}
|
||||
(session_id, user_id, agent_id, name, state, metadata, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
session.session_id,
|
||||
session.user_id,
|
||||
session.agent_id,
|
||||
session.name,
|
||||
json.dumps(session.state) if session.state else None,
|
||||
json.dumps(session.metadata) if session.metadata else None,
|
||||
session.created_at,
|
||||
session.updated_at,
|
||||
))
|
||||
conn.commit()
|
||||
return session
|
||||
|
||||
def get_session(self, session_id: str) -> Optional[ConversationSession]:
|
||||
"""Get a session by ID."""
|
||||
conn = self._get_conn()
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"""
|
||||
SELECT * FROM {self.sessions_table} WHERE session_id = ?
|
||||
""", (session_id,))
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
return None
|
||||
return ConversationSession(
|
||||
session_id=row["session_id"],
|
||||
user_id=row["user_id"],
|
||||
agent_id=row["agent_id"],
|
||||
name=row["name"],
|
||||
state=json.loads(row["state"]) if row["state"] else None,
|
||||
metadata=json.loads(row["metadata"]) if row["metadata"] else None,
|
||||
created_at=row["created_at"],
|
||||
updated_at=row["updated_at"],
|
||||
)
|
||||
|
||||
def update_session(self, session: ConversationSession) -> ConversationSession:
|
||||
"""Update an existing session."""
|
||||
conn = self._get_conn()
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"""
|
||||
UPDATE {self.sessions_table}
|
||||
SET user_id = ?, agent_id = ?, name = ?,
|
||||
state = ?, metadata = ?, updated_at = ?
|
||||
WHERE session_id = ?
|
||||
""", (
|
||||
session.user_id,
|
||||
session.agent_id,
|
||||
session.name,
|
||||
json.dumps(session.state) if session.state else None,
|
||||
json.dumps(session.metadata) if session.metadata else None,
|
||||
session.updated_at,
|
||||
session.session_id,
|
||||
))
|
||||
conn.commit()
|
||||
return session
|
||||
|
||||
def delete_session(self, session_id: str) -> bool:
|
||||
"""Delete a session and all its messages."""
|
||||
conn = self._get_conn()
|
||||
cur = conn.cursor()
|
||||
# Delete messages first (foreign key)
|
||||
cur.execute(f"DELETE FROM {self.messages_table} WHERE session_id = ?", (session_id,))
|
||||
cur.execute(f"DELETE FROM {self.sessions_table} WHERE session_id = ?", (session_id,))
|
||||
deleted = cur.rowcount > 0
|
||||
conn.commit()
|
||||
return deleted
|
||||
|
||||
def list_sessions(
|
||||
self,
|
||||
user_id: Optional[str] = None,
|
||||
agent_id: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0
|
||||
) -> List[ConversationSession]:
|
||||
"""List sessions, optionally filtered by user or agent."""
|
||||
conn = self._get_conn()
|
||||
cur = conn.cursor()
|
||||
|
||||
conditions = []
|
||||
params: List[Any] = []
|
||||
|
||||
if user_id:
|
||||
conditions.append("user_id = ?")
|
||||
params.append(user_id)
|
||||
if agent_id:
|
||||
conditions.append("agent_id = ?")
|
||||
params.append(agent_id)
|
||||
|
||||
where_clause = ""
|
||||
if conditions:
|
||||
where_clause = "WHERE " + " AND ".join(conditions)
|
||||
|
||||
params.extend([limit, offset])
|
||||
|
||||
cur.execute(f"""
|
||||
SELECT * FROM {self.sessions_table}
|
||||
{where_clause}
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT ? OFFSET ?
|
||||
""", params)
|
||||
|
||||
return [
|
||||
ConversationSession(
|
||||
session_id=row["session_id"],
|
||||
user_id=row["user_id"],
|
||||
agent_id=row["agent_id"],
|
||||
name=row["name"],
|
||||
state=json.loads(row["state"]) if row["state"] else None,
|
||||
metadata=json.loads(row["metadata"]) if row["metadata"] else None,
|
||||
created_at=row["created_at"],
|
||||
updated_at=row["updated_at"],
|
||||
)
|
||||
for row in cur.fetchall()
|
||||
]
|
||||
|
||||
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
|
||||
"""Add a message to a session."""
|
||||
message.session_id = session_id
|
||||
conn = self._get_conn()
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"""
|
||||
INSERT INTO {self.messages_table}
|
||||
(id, session_id, role, content, tool_calls, tool_call_id, metadata, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
message.id,
|
||||
session_id,
|
||||
message.role,
|
||||
message.content,
|
||||
json.dumps(message.tool_calls) if message.tool_calls else None,
|
||||
message.tool_call_id,
|
||||
json.dumps(message.metadata) if message.metadata else None,
|
||||
message.created_at,
|
||||
))
|
||||
conn.commit()
|
||||
return message
|
||||
|
||||
def get_messages(
|
||||
self,
|
||||
session_id: str,
|
||||
limit: Optional[int] = None,
|
||||
before: Optional[float] = None,
|
||||
after: Optional[float] = None
|
||||
) -> List[ConversationMessage]:
|
||||
"""Get messages from a session."""
|
||||
conn = self._get_conn()
|
||||
cur = conn.cursor()
|
||||
|
||||
conditions = ["session_id = ?"]
|
||||
params: List[Any] = [session_id]
|
||||
|
||||
if before:
|
||||
conditions.append("created_at < ?")
|
||||
params.append(before)
|
||||
if after:
|
||||
conditions.append("created_at > ?")
|
||||
params.append(after)
|
||||
|
||||
where_clause = "WHERE " + " AND ".join(conditions)
|
||||
limit_clause = f"LIMIT {limit}" if limit else ""
|
||||
|
||||
cur.execute(f"""
|
||||
SELECT * FROM {self.messages_table}
|
||||
{where_clause}
|
||||
ORDER BY created_at ASC
|
||||
{limit_clause}
|
||||
""", params)
|
||||
|
||||
return [
|
||||
ConversationMessage(
|
||||
id=row["id"],
|
||||
session_id=row["session_id"],
|
||||
role=row["role"],
|
||||
content=row["content"],
|
||||
tool_calls=json.loads(row["tool_calls"]) if row["tool_calls"] else None,
|
||||
tool_call_id=row["tool_call_id"],
|
||||
metadata=json.loads(row["metadata"]) if row["metadata"] else None,
|
||||
created_at=row["created_at"],
|
||||
)
|
||||
for row in cur.fetchall()
|
||||
]
|
||||
|
||||
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
|
||||
"""Delete messages."""
|
||||
conn = self._get_conn()
|
||||
cur = conn.cursor()
|
||||
|
||||
if message_ids:
|
||||
placeholders = ",".join(["?"] * len(message_ids))
|
||||
cur.execute(f"""
|
||||
DELETE FROM {self.messages_table}
|
||||
WHERE session_id = ? AND id IN ({placeholders})
|
||||
""", [session_id] + message_ids)
|
||||
else:
|
||||
cur.execute(f"""
|
||||
DELETE FROM {self.messages_table} WHERE session_id = ?
|
||||
""", (session_id,))
|
||||
|
||||
deleted = cur.rowcount
|
||||
conn.commit()
|
||||
return deleted
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the store."""
|
||||
if hasattr(self._local, "conn") and self._local.conn:
|
||||
self._local.conn.close()
|
||||
self._local.conn = None
|
||||
@@ -0,0 +1,203 @@
|
||||
"""
|
||||
Supabase implementation of ConversationStore.
|
||||
|
||||
Requires: supabase
|
||||
Install: pip install supabase
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import ConversationStore, ConversationSession, ConversationMessage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SupabaseConversationStore(ConversationStore):
|
||||
"""
|
||||
Supabase-based conversation store.
|
||||
|
||||
Uses Supabase's PostgreSQL backend via REST API.
|
||||
|
||||
Example:
|
||||
store = SupabaseConversationStore(
|
||||
url="https://xxx.supabase.co",
|
||||
key="your-anon-key"
|
||||
)
|
||||
"""
|
||||
|
||||
SCHEMA_VERSION = "1.0.0"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: Optional[str] = None,
|
||||
key: Optional[str] = None,
|
||||
table_prefix: str = "praison_",
|
||||
auto_create_tables: bool = False, # Tables should be created via Supabase dashboard
|
||||
):
|
||||
"""
|
||||
Initialize Supabase conversation store.
|
||||
|
||||
Args:
|
||||
url: Supabase project URL
|
||||
key: Supabase anon/service key
|
||||
table_prefix: Prefix for table names
|
||||
auto_create_tables: Not supported - use Supabase dashboard
|
||||
"""
|
||||
try:
|
||||
from supabase import create_client, Client
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"supabase is required for Supabase support. "
|
||||
"Install with: pip install supabase"
|
||||
)
|
||||
|
||||
import os
|
||||
url = url or os.getenv("SUPABASE_URL")
|
||||
key = key or os.getenv("SUPABASE_KEY")
|
||||
|
||||
if not url or not key:
|
||||
raise ValueError("Supabase URL and key are required")
|
||||
|
||||
self._client: Client = create_client(url, key)
|
||||
self.table_prefix = table_prefix
|
||||
self.sessions_table = f"{table_prefix}sessions"
|
||||
self.messages_table = f"{table_prefix}messages"
|
||||
|
||||
if auto_create_tables:
|
||||
logger.warning("auto_create_tables not supported for Supabase. Create tables via dashboard.")
|
||||
|
||||
def create_session(self, session: ConversationSession) -> ConversationSession:
|
||||
data = {
|
||||
"session_id": session.session_id,
|
||||
"user_id": session.user_id,
|
||||
"agent_id": session.agent_id,
|
||||
"name": session.name,
|
||||
"state": session.state,
|
||||
"metadata": session.metadata,
|
||||
"created_at": session.created_at,
|
||||
"updated_at": session.updated_at,
|
||||
}
|
||||
self._client.table(self.sessions_table).insert(data).execute()
|
||||
return session
|
||||
|
||||
def get_session(self, session_id: str) -> Optional[ConversationSession]:
|
||||
result = self._client.table(self.sessions_table).select("*").eq("session_id", session_id).execute()
|
||||
if not result.data:
|
||||
return None
|
||||
row = result.data[0]
|
||||
return ConversationSession(
|
||||
session_id=row["session_id"],
|
||||
user_id=row.get("user_id"),
|
||||
agent_id=row.get("agent_id"),
|
||||
name=row.get("name"),
|
||||
state=row.get("state"),
|
||||
metadata=row.get("metadata"),
|
||||
created_at=row.get("created_at"),
|
||||
updated_at=row.get("updated_at"),
|
||||
)
|
||||
|
||||
def update_session(self, session: ConversationSession) -> ConversationSession:
|
||||
data = {
|
||||
"user_id": session.user_id,
|
||||
"agent_id": session.agent_id,
|
||||
"name": session.name,
|
||||
"state": session.state,
|
||||
"metadata": session.metadata,
|
||||
"updated_at": session.updated_at,
|
||||
}
|
||||
self._client.table(self.sessions_table).update(data).eq("session_id", session.session_id).execute()
|
||||
return session
|
||||
|
||||
def delete_session(self, session_id: str) -> bool:
|
||||
# Delete messages first
|
||||
self._client.table(self.messages_table).delete().eq("session_id", session_id).execute()
|
||||
result = self._client.table(self.sessions_table).delete().eq("session_id", session_id).execute()
|
||||
return len(result.data) > 0 if result.data else False
|
||||
|
||||
def list_sessions(
|
||||
self,
|
||||
user_id: Optional[str] = None,
|
||||
agent_id: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0
|
||||
) -> List[ConversationSession]:
|
||||
query = self._client.table(self.sessions_table).select("*")
|
||||
if user_id:
|
||||
query = query.eq("user_id", user_id)
|
||||
if agent_id:
|
||||
query = query.eq("agent_id", agent_id)
|
||||
query = query.order("updated_at", desc=True).range(offset, offset + limit - 1)
|
||||
result = query.execute()
|
||||
|
||||
return [
|
||||
ConversationSession(
|
||||
session_id=row["session_id"],
|
||||
user_id=row.get("user_id"),
|
||||
agent_id=row.get("agent_id"),
|
||||
name=row.get("name"),
|
||||
state=row.get("state"),
|
||||
metadata=row.get("metadata"),
|
||||
created_at=row.get("created_at"),
|
||||
updated_at=row.get("updated_at"),
|
||||
)
|
||||
for row in (result.data or [])
|
||||
]
|
||||
|
||||
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
|
||||
message.session_id = session_id
|
||||
data = {
|
||||
"id": message.id,
|
||||
"session_id": session_id,
|
||||
"role": message.role,
|
||||
"content": message.content,
|
||||
"tool_calls": message.tool_calls,
|
||||
"tool_call_id": message.tool_call_id,
|
||||
"metadata": message.metadata,
|
||||
"created_at": message.created_at,
|
||||
}
|
||||
self._client.table(self.messages_table).insert(data).execute()
|
||||
return message
|
||||
|
||||
def get_messages(
|
||||
self,
|
||||
session_id: str,
|
||||
limit: Optional[int] = None,
|
||||
before: Optional[float] = None,
|
||||
after: Optional[float] = None
|
||||
) -> List[ConversationMessage]:
|
||||
query = self._client.table(self.messages_table).select("*").eq("session_id", session_id)
|
||||
if before:
|
||||
query = query.lt("created_at", before)
|
||||
if after:
|
||||
query = query.gt("created_at", after)
|
||||
query = query.order("created_at", desc=False)
|
||||
if limit:
|
||||
query = query.limit(limit)
|
||||
result = query.execute()
|
||||
|
||||
return [
|
||||
ConversationMessage(
|
||||
id=row["id"],
|
||||
session_id=row["session_id"],
|
||||
role=row["role"],
|
||||
content=row.get("content", ""),
|
||||
tool_calls=row.get("tool_calls"),
|
||||
tool_call_id=row.get("tool_call_id"),
|
||||
metadata=row.get("metadata"),
|
||||
created_at=row.get("created_at"),
|
||||
)
|
||||
for row in (result.data or [])
|
||||
]
|
||||
|
||||
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
|
||||
if message_ids:
|
||||
result = self._client.table(self.messages_table).delete().eq("session_id", session_id).in_("id", message_ids).execute()
|
||||
else:
|
||||
result = self._client.table(self.messages_table).delete().eq("session_id", session_id).execute()
|
||||
return len(result.data) if result.data else 0
|
||||
|
||||
def close(self) -> None:
|
||||
pass # Supabase client doesn't need explicit close
|
||||
@@ -0,0 +1,287 @@
|
||||
"""
|
||||
SurrealDB implementation of ConversationStore.
|
||||
|
||||
Requires: surrealdb
|
||||
Install: pip install surrealdb
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import List, Optional
|
||||
|
||||
from .base import ConversationStore, ConversationSession, ConversationMessage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SurrealDBConversationStore(ConversationStore):
|
||||
"""
|
||||
SurrealDB-based conversation store.
|
||||
|
||||
SurrealDB is a multi-model database supporting SQL-like queries.
|
||||
|
||||
Example:
|
||||
store = SurrealDBConversationStore(
|
||||
url="ws://localhost:8000/rpc",
|
||||
namespace="praisonai",
|
||||
database="conversations"
|
||||
)
|
||||
"""
|
||||
|
||||
SCHEMA_VERSION = "1.0.0"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: str = "ws://localhost:8000/rpc",
|
||||
namespace: str = "praisonai",
|
||||
database: str = "conversations",
|
||||
username: Optional[str] = None,
|
||||
password: Optional[str] = None,
|
||||
table_prefix: str = "praison_",
|
||||
auto_create_tables: bool = True,
|
||||
):
|
||||
try:
|
||||
from surrealdb import Surreal
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"surrealdb is required for SurrealDB support. "
|
||||
"Install with: pip install surrealdb"
|
||||
)
|
||||
|
||||
self._Surreal = Surreal
|
||||
self.url = url
|
||||
self.namespace = namespace
|
||||
self.database = database
|
||||
self.username = username
|
||||
self.password = password
|
||||
self.table_prefix = table_prefix
|
||||
self.sessions_table = f"{table_prefix}sessions"
|
||||
self.messages_table = f"{table_prefix}messages"
|
||||
self._client = None
|
||||
|
||||
# Note: SurrealDB Python client is async, we'll use sync wrapper
|
||||
self._init_sync()
|
||||
|
||||
def _init_sync(self):
|
||||
"""Initialize synchronously using asyncio."""
|
||||
import asyncio
|
||||
|
||||
async def _init():
|
||||
self._client = self._Surreal(self.url)
|
||||
await self._client.connect()
|
||||
if self.username and self.password:
|
||||
await self._client.signin({"user": self.username, "pass": self.password})
|
||||
await self._client.use(self.namespace, self.database)
|
||||
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
except RuntimeError:
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
|
||||
loop.run_until_complete(_init())
|
||||
|
||||
def _run_sync(self, coro):
|
||||
"""Run async coroutine synchronously."""
|
||||
import asyncio
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
except RuntimeError:
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
return loop.run_until_complete(coro)
|
||||
|
||||
def create_session(self, session: ConversationSession) -> ConversationSession:
|
||||
data = {
|
||||
"session_id": session.session_id,
|
||||
"user_id": session.user_id,
|
||||
"agent_id": session.agent_id,
|
||||
"name": session.name,
|
||||
"state": session.state,
|
||||
"metadata": session.metadata,
|
||||
"created_at": session.created_at,
|
||||
"updated_at": session.updated_at,
|
||||
}
|
||||
self._run_sync(self._client.create(self.sessions_table, data))
|
||||
return session
|
||||
|
||||
def get_session(self, session_id: str) -> Optional[ConversationSession]:
|
||||
result = self._run_sync(
|
||||
self._client.query(
|
||||
f"SELECT * FROM {self.sessions_table} WHERE session_id = $sid",
|
||||
{"sid": session_id}
|
||||
)
|
||||
)
|
||||
if not result or not result[0].get("result"):
|
||||
return None
|
||||
row = result[0]["result"][0]
|
||||
return ConversationSession(
|
||||
session_id=row["session_id"],
|
||||
user_id=row.get("user_id"),
|
||||
agent_id=row.get("agent_id"),
|
||||
name=row.get("name"),
|
||||
state=row.get("state"),
|
||||
metadata=row.get("metadata"),
|
||||
created_at=row.get("created_at"),
|
||||
updated_at=row.get("updated_at"),
|
||||
)
|
||||
|
||||
def update_session(self, session: ConversationSession) -> ConversationSession:
|
||||
self._run_sync(
|
||||
self._client.query(
|
||||
f"""UPDATE {self.sessions_table} SET
|
||||
user_id = $user_id, agent_id = $agent_id, name = $name,
|
||||
state = $state, metadata = $metadata, updated_at = $updated_at
|
||||
WHERE session_id = $session_id""",
|
||||
{
|
||||
"session_id": session.session_id,
|
||||
"user_id": session.user_id,
|
||||
"agent_id": session.agent_id,
|
||||
"name": session.name,
|
||||
"state": session.state,
|
||||
"metadata": session.metadata,
|
||||
"updated_at": session.updated_at,
|
||||
}
|
||||
)
|
||||
)
|
||||
return session
|
||||
|
||||
def delete_session(self, session_id: str) -> bool:
|
||||
self._run_sync(
|
||||
self._client.query(
|
||||
f"DELETE FROM {self.messages_table} WHERE session_id = $sid",
|
||||
{"sid": session_id}
|
||||
)
|
||||
)
|
||||
result = self._run_sync(
|
||||
self._client.query(
|
||||
f"DELETE FROM {self.sessions_table} WHERE session_id = $sid",
|
||||
{"sid": session_id}
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
def list_sessions(
|
||||
self,
|
||||
user_id: Optional[str] = None,
|
||||
agent_id: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0
|
||||
) -> List[ConversationSession]:
|
||||
conditions = []
|
||||
params = {"limit": limit, "offset": offset}
|
||||
|
||||
if user_id:
|
||||
conditions.append("user_id = $user_id")
|
||||
params["user_id"] = user_id
|
||||
if agent_id:
|
||||
conditions.append("agent_id = $agent_id")
|
||||
params["agent_id"] = agent_id
|
||||
|
||||
where = "WHERE " + " AND ".join(conditions) if conditions else ""
|
||||
|
||||
result = self._run_sync(
|
||||
self._client.query(
|
||||
f"SELECT * FROM {self.sessions_table} {where} ORDER BY updated_at DESC LIMIT $limit START $offset",
|
||||
params
|
||||
)
|
||||
)
|
||||
|
||||
if not result or not result[0].get("result"):
|
||||
return []
|
||||
|
||||
return [
|
||||
ConversationSession(
|
||||
session_id=row["session_id"],
|
||||
user_id=row.get("user_id"),
|
||||
agent_id=row.get("agent_id"),
|
||||
name=row.get("name"),
|
||||
state=row.get("state"),
|
||||
metadata=row.get("metadata"),
|
||||
created_at=row.get("created_at"),
|
||||
updated_at=row.get("updated_at"),
|
||||
)
|
||||
for row in result[0]["result"]
|
||||
]
|
||||
|
||||
def add_message(self, session_id: str, message: ConversationMessage) -> ConversationMessage:
|
||||
message.session_id = session_id
|
||||
data = {
|
||||
"id": message.id,
|
||||
"session_id": session_id,
|
||||
"role": message.role,
|
||||
"content": message.content,
|
||||
"tool_calls": message.tool_calls,
|
||||
"tool_call_id": message.tool_call_id,
|
||||
"metadata": message.metadata,
|
||||
"created_at": message.created_at,
|
||||
}
|
||||
self._run_sync(self._client.create(self.messages_table, data))
|
||||
return message
|
||||
|
||||
def get_messages(
|
||||
self,
|
||||
session_id: str,
|
||||
limit: Optional[int] = None,
|
||||
before: Optional[float] = None,
|
||||
after: Optional[float] = None
|
||||
) -> List[ConversationMessage]:
|
||||
conditions = ["session_id = $session_id"]
|
||||
params = {"session_id": session_id}
|
||||
|
||||
if before:
|
||||
conditions.append("created_at < $before")
|
||||
params["before"] = before
|
||||
if after:
|
||||
conditions.append("created_at > $after")
|
||||
params["after"] = after
|
||||
|
||||
where = "WHERE " + " AND ".join(conditions)
|
||||
limit_clause = f"LIMIT {limit}" if limit else ""
|
||||
|
||||
result = self._run_sync(
|
||||
self._client.query(
|
||||
f"SELECT * FROM {self.messages_table} {where} ORDER BY created_at ASC {limit_clause}",
|
||||
params
|
||||
)
|
||||
)
|
||||
|
||||
if not result or not result[0].get("result"):
|
||||
return []
|
||||
|
||||
return [
|
||||
ConversationMessage(
|
||||
id=row["id"],
|
||||
session_id=row["session_id"],
|
||||
role=row["role"],
|
||||
content=row.get("content", ""),
|
||||
tool_calls=row.get("tool_calls"),
|
||||
tool_call_id=row.get("tool_call_id"),
|
||||
metadata=row.get("metadata"),
|
||||
created_at=row.get("created_at"),
|
||||
)
|
||||
for row in result[0]["result"]
|
||||
]
|
||||
|
||||
def delete_messages(self, session_id: str, message_ids: Optional[List[str]] = None) -> int:
|
||||
if message_ids:
|
||||
self._run_sync(
|
||||
self._client.query(
|
||||
f"DELETE FROM {self.messages_table} WHERE session_id = $sid AND id IN $ids",
|
||||
{"sid": session_id, "ids": message_ids}
|
||||
)
|
||||
)
|
||||
return len(message_ids)
|
||||
else:
|
||||
self._run_sync(
|
||||
self._client.query(
|
||||
f"DELETE FROM {self.messages_table} WHERE session_id = $sid",
|
||||
{"sid": session_id}
|
||||
)
|
||||
)
|
||||
return -1
|
||||
|
||||
def close(self) -> None:
|
||||
if self._client:
|
||||
self._run_sync(self._client.close())
|
||||
self._client = None
|
||||
@@ -0,0 +1,301 @@
|
||||
"""
|
||||
Factory functions for creating store instances.
|
||||
|
||||
Provides lazy loading of backend implementations to avoid importing
|
||||
unused dependencies.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from .config import PersistenceConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def create_conversation_store(
|
||||
backend: str,
|
||||
url: Optional[str] = None,
|
||||
**options: Any
|
||||
):
|
||||
"""
|
||||
Create a ConversationStore instance.
|
||||
|
||||
Args:
|
||||
backend: Backend type (postgres, mysql, sqlite, singlestore, supabase, surrealdb)
|
||||
url: Connection URL
|
||||
**options: Backend-specific options
|
||||
|
||||
Returns:
|
||||
ConversationStore instance
|
||||
|
||||
Example:
|
||||
store = create_conversation_store(
|
||||
"postgres",
|
||||
url="postgresql://localhost:5432/praisonai"
|
||||
)
|
||||
"""
|
||||
backend = backend.lower()
|
||||
|
||||
if backend == "postgres":
|
||||
from .conversation.postgres import PostgresConversationStore
|
||||
return PostgresConversationStore(url=url, **options)
|
||||
|
||||
elif backend == "mysql":
|
||||
from .conversation.mysql import MySQLConversationStore
|
||||
return MySQLConversationStore(url=url, **options)
|
||||
|
||||
elif backend == "sqlite":
|
||||
from .conversation.sqlite import SQLiteConversationStore
|
||||
path = url or options.pop("path", None)
|
||||
return SQLiteConversationStore(path=path, **options)
|
||||
|
||||
elif backend == "singlestore":
|
||||
from .conversation.singlestore import SingleStoreConversationStore
|
||||
return SingleStoreConversationStore(url=url, **options)
|
||||
|
||||
elif backend == "supabase":
|
||||
from .conversation.supabase import SupabaseConversationStore
|
||||
return SupabaseConversationStore(url=url, **options)
|
||||
|
||||
elif backend == "surrealdb":
|
||||
from .conversation.surrealdb import SurrealDBConversationStore
|
||||
return SurrealDBConversationStore(url=url, **options)
|
||||
|
||||
elif backend == "json":
|
||||
from .conversation.json_store import JSONConversationStore
|
||||
path = url or options.pop("path", None) or "./praisonai_conversations"
|
||||
return JSONConversationStore(path=path, **options)
|
||||
|
||||
elif backend in ("async_postgres", "asyncpg", "postgres_async"):
|
||||
from .conversation.async_postgres import AsyncPostgresConversationStore
|
||||
return AsyncPostgresConversationStore(url=url, **options)
|
||||
|
||||
elif backend in ("async_sqlite", "aiosqlite", "sqlite_async"):
|
||||
from .conversation.async_sqlite import AsyncSQLiteConversationStore
|
||||
path = url or options.pop("path", None)
|
||||
return AsyncSQLiteConversationStore(path=path, **options)
|
||||
|
||||
elif backend in ("async_mysql", "aiomysql", "mysql_async"):
|
||||
from .conversation.async_mysql import AsyncMySQLConversationStore
|
||||
return AsyncMySQLConversationStore(url=url, **options)
|
||||
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unknown conversation store backend: {backend}. "
|
||||
f"Supported: postgres, mysql, sqlite, json, singlestore, supabase, surrealdb, "
|
||||
f"async_postgres, async_sqlite, async_mysql"
|
||||
)
|
||||
|
||||
|
||||
def create_knowledge_store(
|
||||
backend: str,
|
||||
url: Optional[str] = None,
|
||||
**options: Any
|
||||
):
|
||||
"""
|
||||
Create a KnowledgeStore instance.
|
||||
|
||||
Args:
|
||||
backend: Backend type (qdrant, pinecone, chroma, weaviate, lancedb, milvus, pgvector, redis, cassandra, clickhouse)
|
||||
url: Connection URL
|
||||
**options: Backend-specific options
|
||||
|
||||
Returns:
|
||||
KnowledgeStore instance
|
||||
|
||||
Example:
|
||||
store = create_knowledge_store(
|
||||
"qdrant",
|
||||
url="http://localhost:6333"
|
||||
)
|
||||
"""
|
||||
backend = backend.lower()
|
||||
|
||||
if backend == "qdrant":
|
||||
from .knowledge.qdrant import QdrantKnowledgeStore
|
||||
return QdrantKnowledgeStore(url=url, **options)
|
||||
|
||||
elif backend == "pinecone":
|
||||
from .knowledge.pinecone import PineconeKnowledgeStore
|
||||
return PineconeKnowledgeStore(**options)
|
||||
|
||||
elif backend == "chroma":
|
||||
from .knowledge.chroma import ChromaKnowledgeStore
|
||||
path = url or options.pop("path", None)
|
||||
return ChromaKnowledgeStore(path=path, **options)
|
||||
|
||||
elif backend == "weaviate":
|
||||
from .knowledge.weaviate import WeaviateKnowledgeStore
|
||||
return WeaviateKnowledgeStore(url=url, **options)
|
||||
|
||||
elif backend == "lancedb":
|
||||
from .knowledge.lancedb import LanceDBKnowledgeStore
|
||||
path = url or options.pop("path", None)
|
||||
return LanceDBKnowledgeStore(path=path, **options)
|
||||
|
||||
elif backend == "milvus":
|
||||
from .knowledge.milvus import MilvusKnowledgeStore
|
||||
return MilvusKnowledgeStore(url=url, **options)
|
||||
|
||||
elif backend == "pgvector":
|
||||
from .knowledge.pgvector import PGVectorKnowledgeStore
|
||||
return PGVectorKnowledgeStore(url=url, **options)
|
||||
|
||||
elif backend == "redis":
|
||||
from .knowledge.redis_vector import RedisVectorKnowledgeStore
|
||||
return RedisVectorKnowledgeStore(url=url, **options)
|
||||
|
||||
elif backend == "cassandra":
|
||||
from .knowledge.cassandra import CassandraKnowledgeStore
|
||||
return CassandraKnowledgeStore(**options)
|
||||
|
||||
elif backend == "clickhouse":
|
||||
from .knowledge.clickhouse import ClickHouseKnowledgeStore
|
||||
return ClickHouseKnowledgeStore(**options)
|
||||
|
||||
elif backend == "couchbase":
|
||||
from .knowledge.couchbase import CouchbaseKnowledgeStore
|
||||
return CouchbaseKnowledgeStore(**options)
|
||||
|
||||
elif backend in ("mongodb_vector", "mongodb_atlas", "mongo_vector"):
|
||||
from .knowledge.mongodb_vector import MongoDBVectorKnowledgeStore
|
||||
return MongoDBVectorKnowledgeStore(url=url, **options)
|
||||
|
||||
elif backend in ("singlestore_vector", "singlestore_v"):
|
||||
from .knowledge.singlestore_vector import SingleStoreVectorKnowledgeStore
|
||||
return SingleStoreVectorKnowledgeStore(url=url, **options)
|
||||
|
||||
elif backend in ("surrealdb_vector", "surrealdb_v"):
|
||||
from .knowledge.surrealdb_vector import SurrealDBVectorKnowledgeStore
|
||||
return SurrealDBVectorKnowledgeStore(url=url, **options)
|
||||
|
||||
elif backend in ("upstash_vector", "upstash_v"):
|
||||
from .knowledge.upstash_vector import UpstashVectorKnowledgeStore
|
||||
return UpstashVectorKnowledgeStore(url=url, **options)
|
||||
|
||||
elif backend == "lightrag":
|
||||
from .knowledge.lightrag_adapter import LightRAGKnowledgeStore
|
||||
return LightRAGKnowledgeStore(**options)
|
||||
|
||||
elif backend in ("langchain", "langchain_adapter"):
|
||||
from .knowledge.langchain_adapter import LangChainKnowledgeStore
|
||||
return LangChainKnowledgeStore(**options)
|
||||
|
||||
elif backend in ("llamaindex", "llama_index", "llamaindex_adapter"):
|
||||
from .knowledge.llamaindex_adapter import LlamaIndexKnowledgeStore
|
||||
return LlamaIndexKnowledgeStore(**options)
|
||||
|
||||
elif backend in ("cosmosdb", "cosmos", "azure_cosmos", "cosmosdb_vector"):
|
||||
from .knowledge.cosmosdb_vector import CosmosDBVectorKnowledgeStore
|
||||
return CosmosDBVectorKnowledgeStore(**options)
|
||||
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unknown knowledge store backend: {backend}. "
|
||||
f"Supported: qdrant, pinecone, chroma, weaviate, lancedb, milvus, pgvector, redis, cassandra, clickhouse, "
|
||||
f"couchbase, mongodb_vector, singlestore_vector, surrealdb_vector, upstash_vector, lightrag, langchain, llamaindex, cosmosdb"
|
||||
)
|
||||
|
||||
|
||||
def create_state_store(
|
||||
backend: str,
|
||||
url: Optional[str] = None,
|
||||
**options: Any
|
||||
):
|
||||
"""
|
||||
Create a StateStore instance.
|
||||
|
||||
Args:
|
||||
backend: Backend type (redis, dynamodb, firestore, mongodb, upstash, memory)
|
||||
url: Connection URL
|
||||
**options: Backend-specific options
|
||||
|
||||
Returns:
|
||||
StateStore instance
|
||||
|
||||
Example:
|
||||
store = create_state_store(
|
||||
"redis",
|
||||
url="redis://localhost:6379"
|
||||
)
|
||||
"""
|
||||
backend = backend.lower()
|
||||
|
||||
if backend == "redis":
|
||||
from .state.redis import RedisStateStore
|
||||
return RedisStateStore(url=url, **options)
|
||||
|
||||
elif backend == "dynamodb":
|
||||
from .state.dynamodb import DynamoDBStateStore
|
||||
return DynamoDBStateStore(**options)
|
||||
|
||||
elif backend == "firestore":
|
||||
from .state.firestore import FirestoreStateStore
|
||||
return FirestoreStateStore(**options)
|
||||
|
||||
elif backend == "mongodb":
|
||||
from .state.mongodb import MongoDBStateStore
|
||||
return MongoDBStateStore(url=url, **options)
|
||||
|
||||
elif backend == "upstash":
|
||||
from .state.upstash import UpstashStateStore
|
||||
return UpstashStateStore(url=url, **options)
|
||||
|
||||
elif backend == "memory":
|
||||
from .state.memory import MemoryStateStore
|
||||
return MemoryStateStore(**options)
|
||||
|
||||
elif backend == "gcs":
|
||||
from .state.gcs import GCSStateStore
|
||||
bucket = options.pop("bucket_name", None) or options.pop("bucket", None)
|
||||
if not bucket:
|
||||
raise ValueError("GCS state store requires 'bucket_name' option")
|
||||
return GCSStateStore(bucket_name=bucket, **options)
|
||||
|
||||
elif backend in ("async_mongodb", "motor", "mongodb_async"):
|
||||
from .state.async_mongodb import AsyncMongoDBStateStore
|
||||
return AsyncMongoDBStateStore(url=url, **options)
|
||||
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unknown state store backend: {backend}. "
|
||||
f"Supported: redis, dynamodb, firestore, mongodb, upstash, memory, gcs, async_mongodb"
|
||||
)
|
||||
|
||||
|
||||
def create_stores_from_config(config: PersistenceConfig) -> Dict[str, Any]:
|
||||
"""
|
||||
Create all configured stores from a PersistenceConfig.
|
||||
|
||||
Returns:
|
||||
Dict with keys: conversation, knowledge, state (values may be None)
|
||||
"""
|
||||
stores = {
|
||||
"conversation": None,
|
||||
"knowledge": None,
|
||||
"state": None,
|
||||
}
|
||||
|
||||
if config.conversation_store:
|
||||
stores["conversation"] = create_conversation_store(
|
||||
config.conversation_store,
|
||||
url=config.conversation_url,
|
||||
**config.conversation_options
|
||||
)
|
||||
|
||||
if config.knowledge_store:
|
||||
stores["knowledge"] = create_knowledge_store(
|
||||
config.knowledge_store,
|
||||
url=config.knowledge_url,
|
||||
**config.knowledge_options
|
||||
)
|
||||
|
||||
if config.state_store:
|
||||
stores["state"] = create_state_store(
|
||||
config.state_store,
|
||||
url=config.state_url,
|
||||
**config.state_options
|
||||
)
|
||||
|
||||
return stores
|
||||
@@ -0,0 +1,18 @@
|
||||
"""
|
||||
Agent hooks for automatic persistence integration.
|
||||
|
||||
Provides wrapper functions to add persistence capabilities to PraisonAI agents
|
||||
without modifying the core SDK.
|
||||
"""
|
||||
|
||||
from .agent_hooks import (
|
||||
wrap_agent_with_persistence,
|
||||
PersistentAgent,
|
||||
create_persistent_session,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"wrap_agent_with_persistence",
|
||||
"PersistentAgent",
|
||||
"create_persistent_session",
|
||||
]
|
||||
@@ -0,0 +1,297 @@
|
||||
"""
|
||||
Agent hooks for automatic persistence integration.
|
||||
|
||||
Provides wrapper functions to add persistence capabilities to PraisonAI agents
|
||||
without modifying the core SDK.
|
||||
"""
|
||||
|
||||
import time
|
||||
import uuid
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional, Callable
|
||||
from functools import wraps
|
||||
|
||||
from ..orchestrator import PersistenceOrchestrator
|
||||
from ..conversation.base import ConversationMessage, ConversationSession
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def wrap_agent_with_persistence(
|
||||
agent,
|
||||
orchestrator: PersistenceOrchestrator,
|
||||
session_id: Optional[str] = None,
|
||||
user_id: Optional[str] = None,
|
||||
auto_resume: bool = True,
|
||||
):
|
||||
"""
|
||||
Wrap an agent to add automatic persistence hooks.
|
||||
|
||||
This function wraps the agent's chat method to automatically:
|
||||
1. Load/create session on first message
|
||||
2. Persist each user message
|
||||
3. Persist each assistant response
|
||||
4. Update session metadata after each interaction
|
||||
|
||||
Args:
|
||||
agent: PraisonAI Agent instance
|
||||
orchestrator: PersistenceOrchestrator instance
|
||||
session_id: Session ID (auto-generated if None)
|
||||
user_id: User ID for the session
|
||||
auto_resume: Whether to auto-resume existing session
|
||||
|
||||
Returns:
|
||||
The wrapped agent with persistence capabilities
|
||||
|
||||
Example:
|
||||
from praisonai.persistence import PersistenceOrchestrator
|
||||
from praisonai.persistence.hooks import wrap_agent_with_persistence
|
||||
from praisonaiagents import Agent
|
||||
|
||||
orchestrator = PersistenceOrchestrator(...)
|
||||
agent = Agent(name="Assistant", role="Helper")
|
||||
|
||||
# Wrap agent with persistence
|
||||
agent = wrap_agent_with_persistence(agent, orchestrator, session_id="my-session")
|
||||
|
||||
# Now all chats are automatically persisted
|
||||
response = agent.chat("Hello!")
|
||||
"""
|
||||
session_id = session_id or f"session-{uuid.uuid4().hex[:8]}"
|
||||
user_id = user_id or getattr(agent, 'user_id', 'default')
|
||||
|
||||
# Store original chat method
|
||||
original_chat = agent.chat
|
||||
_session_initialized = [False] # Use list to allow mutation in closure
|
||||
|
||||
@wraps(original_chat)
|
||||
def chat_with_persistence(prompt, *args, **kwargs):
|
||||
nonlocal _session_initialized
|
||||
|
||||
# Initialize session on first call
|
||||
if not _session_initialized[0]:
|
||||
history = orchestrator.on_agent_start(
|
||||
agent,
|
||||
session_id=session_id,
|
||||
user_id=user_id,
|
||||
resume=auto_resume
|
||||
)
|
||||
|
||||
# Inject history into agent's chat_history if resuming
|
||||
if auto_resume and history:
|
||||
agent.chat_history = [
|
||||
{"role": msg.role, "content": msg.content}
|
||||
for msg in history
|
||||
]
|
||||
logger.info(f"Resumed session {session_id} with {len(history)} messages")
|
||||
|
||||
_session_initialized[0] = True
|
||||
|
||||
# Persist user message
|
||||
orchestrator.on_message(session_id, "user", prompt if isinstance(prompt, str) else str(prompt))
|
||||
|
||||
# Call original chat
|
||||
response = original_chat(prompt, *args, **kwargs)
|
||||
|
||||
# Persist assistant response
|
||||
if response:
|
||||
orchestrator.on_message(session_id, "assistant", response)
|
||||
|
||||
return response
|
||||
|
||||
# Replace chat method
|
||||
agent.chat = chat_with_persistence
|
||||
|
||||
# Add persistence-related methods to agent
|
||||
agent._persistence_orchestrator = orchestrator
|
||||
agent._persistence_session_id = session_id
|
||||
|
||||
def get_session():
|
||||
"""Get the current session."""
|
||||
return orchestrator.get_session(session_id)
|
||||
|
||||
def get_messages(limit: Optional[int] = None):
|
||||
"""Get messages from the current session."""
|
||||
return orchestrator.get_messages(session_id, limit=limit)
|
||||
|
||||
def end_session():
|
||||
"""End the current session."""
|
||||
orchestrator.on_agent_end(agent, session_id)
|
||||
|
||||
agent.get_session = get_session
|
||||
agent.get_messages = get_messages
|
||||
agent.end_session = end_session
|
||||
|
||||
return agent
|
||||
|
||||
|
||||
class PersistentAgent:
|
||||
"""
|
||||
A wrapper class that provides persistence capabilities for any agent.
|
||||
|
||||
This is an alternative to wrap_agent_with_persistence that uses composition
|
||||
instead of monkey-patching.
|
||||
|
||||
Example:
|
||||
from praisonai.persistence import PersistenceOrchestrator
|
||||
from praisonai.persistence.hooks import PersistentAgent
|
||||
from praisonaiagents import Agent
|
||||
|
||||
orchestrator = PersistenceOrchestrator(...)
|
||||
base_agent = Agent(name="Assistant", role="Helper")
|
||||
|
||||
agent = PersistentAgent(base_agent, orchestrator, session_id="my-session")
|
||||
response = agent.chat("Hello!")
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
agent,
|
||||
orchestrator: PersistenceOrchestrator,
|
||||
session_id: Optional[str] = None,
|
||||
user_id: Optional[str] = None,
|
||||
auto_resume: bool = True,
|
||||
):
|
||||
self._agent = agent
|
||||
self._orchestrator = orchestrator
|
||||
self._session_id = session_id or f"session-{uuid.uuid4().hex[:8]}"
|
||||
self._user_id = user_id or getattr(agent, 'user_id', 'default')
|
||||
self._auto_resume = auto_resume
|
||||
self._session_initialized = False
|
||||
|
||||
def __getattr__(self, name):
|
||||
"""Delegate attribute access to wrapped agent."""
|
||||
return getattr(self._agent, name)
|
||||
|
||||
def _ensure_session(self):
|
||||
"""Ensure session is initialized."""
|
||||
if not self._session_initialized:
|
||||
history = self._orchestrator.on_agent_start(
|
||||
self._agent,
|
||||
session_id=self._session_id,
|
||||
user_id=self._user_id,
|
||||
resume=self._auto_resume
|
||||
)
|
||||
|
||||
if self._auto_resume and history:
|
||||
self._agent.chat_history = [
|
||||
{"role": msg.role, "content": msg.content}
|
||||
for msg in history
|
||||
]
|
||||
logger.info(f"Resumed session {self._session_id} with {len(history)} messages")
|
||||
|
||||
self._session_initialized = True
|
||||
|
||||
def chat(self, prompt, *args, **kwargs):
|
||||
"""Chat with automatic persistence."""
|
||||
self._ensure_session()
|
||||
|
||||
# Persist user message
|
||||
self._orchestrator.on_message(
|
||||
self._session_id, "user",
|
||||
prompt if isinstance(prompt, str) else str(prompt)
|
||||
)
|
||||
|
||||
# Call agent chat
|
||||
response = self._agent.chat(prompt, *args, **kwargs)
|
||||
|
||||
# Persist assistant response
|
||||
if response:
|
||||
self._orchestrator.on_message(self._session_id, "assistant", response)
|
||||
|
||||
return response
|
||||
|
||||
def get_session(self) -> Optional[ConversationSession]:
|
||||
"""Get the current session."""
|
||||
return self._orchestrator.get_session(self._session_id)
|
||||
|
||||
def get_messages(self, limit: Optional[int] = None) -> List[ConversationMessage]:
|
||||
"""Get messages from the current session."""
|
||||
return self._orchestrator.get_messages(self._session_id, limit=limit)
|
||||
|
||||
def end_session(self):
|
||||
"""End the current session."""
|
||||
self._orchestrator.on_agent_end(self._agent, self._session_id)
|
||||
|
||||
@property
|
||||
def session_id(self) -> str:
|
||||
"""Get the session ID."""
|
||||
return self._session_id
|
||||
|
||||
|
||||
def create_persistent_session(
|
||||
orchestrator: PersistenceOrchestrator,
|
||||
session_id: Optional[str] = None,
|
||||
user_id: Optional[str] = None,
|
||||
):
|
||||
"""
|
||||
Create a context manager for persistent sessions.
|
||||
|
||||
Example:
|
||||
from praisonai.persistence import PersistenceOrchestrator
|
||||
from praisonai.persistence.hooks import create_persistent_session
|
||||
from praisonaiagents import Agent
|
||||
|
||||
orchestrator = PersistenceOrchestrator(...)
|
||||
|
||||
with create_persistent_session(orchestrator, session_id="my-session") as session:
|
||||
agent = Agent(name="Assistant", role="Helper")
|
||||
|
||||
# Messages are persisted within the session context
|
||||
session.persist_message("user", "Hello!")
|
||||
response = agent.chat("Hello!")
|
||||
session.persist_message("assistant", response)
|
||||
"""
|
||||
|
||||
class PersistentSession:
|
||||
def __init__(self, orchestrator, session_id, user_id):
|
||||
self.orchestrator = orchestrator
|
||||
self.session_id = session_id or f"session-{uuid.uuid4().hex[:8]}"
|
||||
self.user_id = user_id or "default"
|
||||
self._started = False
|
||||
|
||||
def __enter__(self):
|
||||
self.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.end()
|
||||
return False
|
||||
|
||||
def start(self, agent=None):
|
||||
"""Start the session."""
|
||||
if not self._started:
|
||||
class MockAgent:
|
||||
name = "session-agent"
|
||||
|
||||
self.orchestrator.on_agent_start(
|
||||
agent or MockAgent(),
|
||||
session_id=self.session_id,
|
||||
user_id=self.user_id
|
||||
)
|
||||
self._started = True
|
||||
|
||||
def end(self, agent=None):
|
||||
"""End the session."""
|
||||
if self._started:
|
||||
class MockAgent:
|
||||
name = "session-agent"
|
||||
|
||||
self.orchestrator.on_agent_end(
|
||||
agent or MockAgent(),
|
||||
self.session_id
|
||||
)
|
||||
|
||||
def persist_message(self, role: str, content: str):
|
||||
"""Persist a message to the session."""
|
||||
self.orchestrator.on_message(self.session_id, role, content)
|
||||
|
||||
def get_messages(self, limit: Optional[int] = None):
|
||||
"""Get messages from the session."""
|
||||
return self.orchestrator.get_messages(self.session_id, limit=limit)
|
||||
|
||||
def get_session(self):
|
||||
"""Get the session object."""
|
||||
return self.orchestrator.get_session(self.session_id)
|
||||
|
||||
return PersistentSession(orchestrator, session_id, user_id)
|
||||
@@ -0,0 +1,26 @@
|
||||
"""
|
||||
KnowledgeStore implementations for vector embeddings and semantic search.
|
||||
|
||||
Supported backends:
|
||||
- Qdrant (qdrant)
|
||||
- Pinecone (pinecone)
|
||||
- ChromaDB (chroma)
|
||||
- Weaviate (weaviate)
|
||||
- LanceDB (lancedb)
|
||||
- Milvus (milvus)
|
||||
- PGVector (pgvector)
|
||||
- Redis Vector (redis)
|
||||
- Cassandra Vector (cassandra)
|
||||
- ClickHouse Vector (clickhouse)
|
||||
"""
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeStore",
|
||||
"KnowledgeDocument",
|
||||
]
|
||||
|
||||
def __getattr__(name: str):
|
||||
if name in ("KnowledgeStore", "KnowledgeDocument"):
|
||||
from .base import KnowledgeStore, KnowledgeDocument
|
||||
return locals()[name]
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -0,0 +1,144 @@
|
||||
"""
|
||||
Base interfaces for KnowledgeStore.
|
||||
|
||||
KnowledgeStore handles vector embeddings and semantic search for RAG.
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
import time
|
||||
import uuid
|
||||
import hashlib
|
||||
|
||||
|
||||
@dataclass
|
||||
class KnowledgeDocument:
|
||||
"""A document with vector embedding for semantic search."""
|
||||
id: str = field(default_factory=lambda: str(uuid.uuid4()))
|
||||
content: str = ""
|
||||
embedding: Optional[List[float]] = None
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
content_hash: Optional[str] = None
|
||||
created_at: float = field(default_factory=time.time)
|
||||
|
||||
def __post_init__(self):
|
||||
if self.content and not self.content_hash:
|
||||
self.content_hash = hashlib.sha256(self.content.encode()).hexdigest()[:16]
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"id": self.id,
|
||||
"content": self.content,
|
||||
"embedding": self.embedding,
|
||||
"metadata": self.metadata,
|
||||
"content_hash": self.content_hash,
|
||||
"created_at": self.created_at,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "KnowledgeDocument":
|
||||
return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__})
|
||||
|
||||
|
||||
class KnowledgeStore(ABC):
|
||||
"""
|
||||
Abstract base class for vector/knowledge persistence.
|
||||
|
||||
Implementations handle document storage and semantic search for different backends:
|
||||
- Qdrant, Pinecone, ChromaDB, Weaviate (dedicated vector DBs)
|
||||
- LanceDB, Milvus (embedded/distributed vector DBs)
|
||||
- PGVector, Redis, Cassandra, ClickHouse (vector extensions)
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def create_collection(
|
||||
self,
|
||||
name: str,
|
||||
dimension: int,
|
||||
distance: str = "cosine",
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""Create a new collection/index for vectors."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def delete_collection(self, name: str) -> bool:
|
||||
"""Delete a collection and all its documents."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def collection_exists(self, name: str) -> bool:
|
||||
"""Check if a collection exists."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def list_collections(self) -> List[str]:
|
||||
"""List all collections."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def insert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert documents into a collection. Returns list of IDs."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def upsert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert or update documents. Returns list of IDs."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def search(
|
||||
self,
|
||||
collection: str,
|
||||
query_embedding: List[float],
|
||||
limit: int = 5,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
score_threshold: Optional[float] = None
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Search for similar documents by embedding."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def get(
|
||||
self,
|
||||
collection: str,
|
||||
ids: List[str]
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Get documents by IDs."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def delete(
|
||||
self,
|
||||
collection: str,
|
||||
ids: Optional[List[str]] = None,
|
||||
filters: Optional[Dict[str, Any]] = None
|
||||
) -> int:
|
||||
"""Delete documents by IDs or filters. Returns count deleted."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def count(self, collection: str) -> int:
|
||||
"""Count documents in a collection."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def close(self) -> None:
|
||||
"""Close the store and release resources."""
|
||||
raise NotImplementedError
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.close()
|
||||
return False
|
||||
@@ -0,0 +1,232 @@
|
||||
"""
|
||||
Cassandra Vector implementation of KnowledgeStore.
|
||||
|
||||
Requires: cassandra-driver
|
||||
Install: pip install cassandra-driver
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import KnowledgeStore, KnowledgeDocument
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CassandraKnowledgeStore(KnowledgeStore):
|
||||
"""
|
||||
Cassandra-based knowledge store using vector search (SAI).
|
||||
|
||||
Requires Cassandra 5.0+ or DataStax Astra with vector search.
|
||||
|
||||
Example:
|
||||
store = CassandraKnowledgeStore(
|
||||
hosts=["localhost"],
|
||||
keyspace="praisonai"
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hosts: Optional[List[str]] = None,
|
||||
keyspace: str = "praisonai",
|
||||
username: Optional[str] = None,
|
||||
password: Optional[str] = None,
|
||||
port: int = 9042,
|
||||
secure_connect_bundle: Optional[str] = None,
|
||||
):
|
||||
try:
|
||||
from cassandra.cluster import Cluster
|
||||
from cassandra.auth import PlainTextAuthProvider
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"cassandra-driver is required for Cassandra support. "
|
||||
"Install with: pip install cassandra-driver"
|
||||
)
|
||||
|
||||
self._Cluster = Cluster
|
||||
self.keyspace = keyspace
|
||||
|
||||
auth = None
|
||||
if username and password:
|
||||
auth = PlainTextAuthProvider(username=username, password=password)
|
||||
|
||||
if secure_connect_bundle:
|
||||
# Astra DB connection
|
||||
from cassandra.cluster import Cluster
|
||||
cloud_config = {"secure_connect_bundle": secure_connect_bundle}
|
||||
self._cluster = Cluster(cloud=cloud_config, auth_provider=auth)
|
||||
else:
|
||||
hosts = hosts or ["localhost"]
|
||||
self._cluster = Cluster(hosts, port=port, auth_provider=auth)
|
||||
|
||||
self._session = self._cluster.connect()
|
||||
|
||||
# Create keyspace if not exists
|
||||
self._session.execute(f"""
|
||||
CREATE KEYSPACE IF NOT EXISTS {keyspace}
|
||||
WITH replication = {{'class': 'SimpleStrategy', 'replication_factor': 1}}
|
||||
""")
|
||||
self._session.set_keyspace(keyspace)
|
||||
logger.info(f"Connected to Cassandra keyspace: {keyspace}")
|
||||
|
||||
def create_collection(
|
||||
self,
|
||||
name: str,
|
||||
dimension: int,
|
||||
distance: str = "cosine",
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""Create a new table with vector column."""
|
||||
similarity_map = {"cosine": "COSINE", "euclidean": "EUCLIDEAN", "dot": "DOT_PRODUCT"}
|
||||
|
||||
self._session.execute(f"""
|
||||
CREATE TABLE IF NOT EXISTS {name} (
|
||||
id text PRIMARY KEY,
|
||||
content text,
|
||||
content_hash text,
|
||||
created_at double,
|
||||
embedding vector<float, {dimension}>
|
||||
)
|
||||
""")
|
||||
|
||||
# Create SAI index for vector search
|
||||
self._session.execute(f"""
|
||||
CREATE CUSTOM INDEX IF NOT EXISTS {name}_embedding_idx ON {name} (embedding)
|
||||
USING 'StorageAttachedIndex'
|
||||
WITH OPTIONS = {{'similarity_function': '{similarity_map.get(distance, "COSINE")}'}}
|
||||
""")
|
||||
logger.info(f"Created Cassandra table: {name}")
|
||||
|
||||
def delete_collection(self, name: str) -> bool:
|
||||
"""Delete a table."""
|
||||
try:
|
||||
self._session.execute(f"DROP TABLE IF EXISTS {name}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to delete table {name}: {e}")
|
||||
return False
|
||||
|
||||
def collection_exists(self, name: str) -> bool:
|
||||
"""Check if a table exists."""
|
||||
result = self._session.execute("""
|
||||
SELECT table_name FROM system_schema.tables
|
||||
WHERE keyspace_name = %s AND table_name = %s
|
||||
""", (self.keyspace, name))
|
||||
return len(list(result)) > 0
|
||||
|
||||
def list_collections(self) -> List[str]:
|
||||
"""List all tables."""
|
||||
result = self._session.execute("""
|
||||
SELECT table_name FROM system_schema.tables WHERE keyspace_name = %s
|
||||
""", (self.keyspace,))
|
||||
return [row.table_name for row in result]
|
||||
|
||||
def insert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert documents."""
|
||||
ids = []
|
||||
for doc in documents:
|
||||
if doc.embedding is None:
|
||||
raise ValueError(f"Document {doc.id} has no embedding")
|
||||
|
||||
self._session.execute(f"""
|
||||
INSERT INTO {collection} (id, content, content_hash, created_at, embedding)
|
||||
VALUES (%s, %s, %s, %s, %s)
|
||||
""", (doc.id, doc.content, doc.content_hash or "", doc.created_at, doc.embedding))
|
||||
ids.append(doc.id)
|
||||
|
||||
return ids
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert or update documents."""
|
||||
return self.insert(collection, documents) # Cassandra INSERT is upsert
|
||||
|
||||
def search(
|
||||
self,
|
||||
collection: str,
|
||||
query_embedding: List[float],
|
||||
limit: int = 5,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
score_threshold: Optional[float] = None
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Search for similar documents using ANN."""
|
||||
result = self._session.execute(f"""
|
||||
SELECT id, content, content_hash, created_at, similarity_cosine(embedding, %s) as score
|
||||
FROM {collection}
|
||||
ORDER BY embedding ANN OF %s
|
||||
LIMIT %s
|
||||
""", (query_embedding, query_embedding, limit))
|
||||
|
||||
documents = []
|
||||
for row in result:
|
||||
if score_threshold and row.score < score_threshold:
|
||||
continue
|
||||
|
||||
doc = KnowledgeDocument(
|
||||
id=row.id,
|
||||
content=row.content,
|
||||
embedding=None,
|
||||
metadata={},
|
||||
content_hash=row.content_hash,
|
||||
created_at=row.created_at,
|
||||
)
|
||||
documents.append(doc)
|
||||
|
||||
return documents
|
||||
|
||||
def get(
|
||||
self,
|
||||
collection: str,
|
||||
ids: List[str]
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Get documents by IDs."""
|
||||
documents = []
|
||||
for doc_id in ids:
|
||||
result = self._session.execute(f"""
|
||||
SELECT id, content, content_hash, created_at, embedding
|
||||
FROM {collection} WHERE id = %s
|
||||
""", (doc_id,))
|
||||
|
||||
for row in result:
|
||||
doc = KnowledgeDocument(
|
||||
id=row.id,
|
||||
content=row.content,
|
||||
embedding=list(row.embedding) if row.embedding else None,
|
||||
metadata={},
|
||||
content_hash=row.content_hash,
|
||||
created_at=row.created_at,
|
||||
)
|
||||
documents.append(doc)
|
||||
|
||||
return documents
|
||||
|
||||
def delete(
|
||||
self,
|
||||
collection: str,
|
||||
ids: Optional[List[str]] = None,
|
||||
filters: Optional[Dict[str, Any]] = None
|
||||
) -> int:
|
||||
"""Delete documents."""
|
||||
if ids:
|
||||
for doc_id in ids:
|
||||
self._session.execute(f"DELETE FROM {collection} WHERE id = %s", (doc_id,))
|
||||
return len(ids)
|
||||
return 0
|
||||
|
||||
def count(self, collection: str) -> int:
|
||||
"""Count documents."""
|
||||
result = self._session.execute(f"SELECT COUNT(*) FROM {collection}")
|
||||
return result.one()[0]
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the store."""
|
||||
if self._cluster:
|
||||
self._cluster.shutdown()
|
||||
@@ -0,0 +1,295 @@
|
||||
"""
|
||||
ChromaDB implementation of KnowledgeStore.
|
||||
|
||||
Requires: chromadb
|
||||
Install: pip install chromadb
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import KnowledgeStore, KnowledgeDocument
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ChromaKnowledgeStore(KnowledgeStore):
|
||||
"""
|
||||
ChromaDB-based knowledge store for vector search.
|
||||
|
||||
Zero-config embedded vector database.
|
||||
|
||||
Example:
|
||||
store = ChromaKnowledgeStore(
|
||||
path="./chroma_db"
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
path: Optional[str] = None,
|
||||
host: Optional[str] = None,
|
||||
port: int = 8000,
|
||||
persistent: bool = True,
|
||||
):
|
||||
"""
|
||||
Initialize ChromaDB knowledge store.
|
||||
|
||||
Args:
|
||||
path: Path for persistent storage (embedded mode)
|
||||
host: Chroma server host (client mode)
|
||||
port: Chroma server port
|
||||
persistent: Use persistent storage
|
||||
"""
|
||||
try:
|
||||
import chromadb
|
||||
from chromadb.config import Settings
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"chromadb is required for ChromaDB support. "
|
||||
"Install with: pip install chromadb"
|
||||
)
|
||||
|
||||
self._chromadb = chromadb
|
||||
|
||||
if host:
|
||||
# Client mode
|
||||
self._client = chromadb.HttpClient(host=host, port=port)
|
||||
elif path and persistent:
|
||||
# Persistent embedded mode
|
||||
self._client = chromadb.PersistentClient(
|
||||
path=path,
|
||||
settings=Settings(anonymized_telemetry=False)
|
||||
)
|
||||
else:
|
||||
# In-memory mode
|
||||
self._client = chromadb.Client(
|
||||
settings=Settings(anonymized_telemetry=False)
|
||||
)
|
||||
|
||||
self._collections: Dict[str, Any] = {}
|
||||
logger.info(f"ChromaDB initialized (path={path}, host={host})")
|
||||
|
||||
def _get_collection(self, name: str):
|
||||
"""Get or cache a collection."""
|
||||
if name not in self._collections:
|
||||
self._collections[name] = self._client.get_collection(name)
|
||||
return self._collections[name]
|
||||
|
||||
def create_collection(
|
||||
self,
|
||||
name: str,
|
||||
dimension: int,
|
||||
distance: str = "cosine",
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""Create a new collection."""
|
||||
distance_map = {
|
||||
"cosine": "cosine",
|
||||
"euclidean": "l2",
|
||||
"dot": "ip",
|
||||
}
|
||||
|
||||
self._client.create_collection(
|
||||
name=name,
|
||||
metadata={"hnsw:space": distance_map.get(distance, "cosine"), **(metadata or {})}
|
||||
)
|
||||
logger.info(f"Created ChromaDB collection: {name}")
|
||||
|
||||
def delete_collection(self, name: str) -> bool:
|
||||
"""Delete a collection."""
|
||||
try:
|
||||
self._client.delete_collection(name)
|
||||
if name in self._collections:
|
||||
del self._collections[name]
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to delete collection {name}: {e}")
|
||||
return False
|
||||
|
||||
def collection_exists(self, name: str) -> bool:
|
||||
"""Check if a collection exists."""
|
||||
try:
|
||||
self._client.get_collection(name)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def list_collections(self) -> List[str]:
|
||||
"""List all collections."""
|
||||
return [c.name for c in self._client.list_collections()]
|
||||
|
||||
def insert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert documents into a collection."""
|
||||
col = self._get_collection(collection)
|
||||
|
||||
ids = []
|
||||
embeddings = []
|
||||
contents = []
|
||||
metadatas = []
|
||||
|
||||
for doc in documents:
|
||||
if doc.embedding is None:
|
||||
raise ValueError(f"Document {doc.id} has no embedding")
|
||||
ids.append(doc.id)
|
||||
embeddings.append(doc.embedding)
|
||||
contents.append(doc.content)
|
||||
metadatas.append({
|
||||
"content_hash": doc.content_hash or "",
|
||||
"created_at": doc.created_at,
|
||||
**(doc.metadata or {})
|
||||
})
|
||||
|
||||
col.add(
|
||||
ids=ids,
|
||||
embeddings=embeddings,
|
||||
documents=contents,
|
||||
metadatas=metadatas,
|
||||
)
|
||||
|
||||
return ids
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert or update documents."""
|
||||
col = self._get_collection(collection)
|
||||
|
||||
ids = []
|
||||
embeddings = []
|
||||
contents = []
|
||||
metadatas = []
|
||||
|
||||
for doc in documents:
|
||||
if doc.embedding is None:
|
||||
raise ValueError(f"Document {doc.id} has no embedding")
|
||||
ids.append(doc.id)
|
||||
embeddings.append(doc.embedding)
|
||||
contents.append(doc.content)
|
||||
metadatas.append({
|
||||
"content_hash": doc.content_hash or "",
|
||||
"created_at": doc.created_at,
|
||||
**(doc.metadata or {})
|
||||
})
|
||||
|
||||
col.upsert(
|
||||
ids=ids,
|
||||
embeddings=embeddings,
|
||||
documents=contents,
|
||||
metadatas=metadatas,
|
||||
)
|
||||
|
||||
return ids
|
||||
|
||||
def search(
|
||||
self,
|
||||
collection: str,
|
||||
query_embedding: List[float],
|
||||
limit: int = 5,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
score_threshold: Optional[float] = None
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Search for similar documents."""
|
||||
col = self._get_collection(collection)
|
||||
|
||||
where = None
|
||||
if filters:
|
||||
where = {k: v for k, v in filters.items()}
|
||||
|
||||
results = col.query(
|
||||
query_embeddings=[query_embedding],
|
||||
n_results=limit,
|
||||
where=where,
|
||||
include=["documents", "metadatas", "distances"],
|
||||
)
|
||||
|
||||
documents = []
|
||||
if results["ids"] and results["ids"][0]:
|
||||
for i, doc_id in enumerate(results["ids"][0]):
|
||||
distance = results["distances"][0][i] if results.get("distances") else None
|
||||
|
||||
# Convert distance to similarity score (for cosine: 1 - distance)
|
||||
if score_threshold is not None and distance is not None:
|
||||
score = 1 - distance
|
||||
if score < score_threshold:
|
||||
continue
|
||||
|
||||
metadata = results["metadatas"][0][i] if results.get("metadatas") else {}
|
||||
content_hash = metadata.pop("content_hash", None)
|
||||
created_at = metadata.pop("created_at", 0)
|
||||
|
||||
doc = KnowledgeDocument(
|
||||
id=doc_id,
|
||||
content=results["documents"][0][i] if results.get("documents") else "",
|
||||
embedding=None,
|
||||
metadata=metadata,
|
||||
content_hash=content_hash,
|
||||
created_at=created_at,
|
||||
)
|
||||
documents.append(doc)
|
||||
|
||||
return documents
|
||||
|
||||
def get(
|
||||
self,
|
||||
collection: str,
|
||||
ids: List[str]
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Get documents by IDs."""
|
||||
col = self._get_collection(collection)
|
||||
|
||||
results = col.get(
|
||||
ids=ids,
|
||||
include=["documents", "metadatas", "embeddings"],
|
||||
)
|
||||
|
||||
documents = []
|
||||
if results["ids"]:
|
||||
for i, doc_id in enumerate(results["ids"]):
|
||||
metadata = results["metadatas"][i] if results.get("metadatas") else {}
|
||||
content_hash = metadata.pop("content_hash", None)
|
||||
created_at = metadata.pop("created_at", 0)
|
||||
|
||||
doc = KnowledgeDocument(
|
||||
id=doc_id,
|
||||
content=results["documents"][i] if results.get("documents") else "",
|
||||
embedding=results["embeddings"][i] if results.get("embeddings") else None,
|
||||
metadata=metadata,
|
||||
content_hash=content_hash,
|
||||
created_at=created_at,
|
||||
)
|
||||
documents.append(doc)
|
||||
|
||||
return documents
|
||||
|
||||
def delete(
|
||||
self,
|
||||
collection: str,
|
||||
ids: Optional[List[str]] = None,
|
||||
filters: Optional[Dict[str, Any]] = None
|
||||
) -> int:
|
||||
"""Delete documents."""
|
||||
col = self._get_collection(collection)
|
||||
|
||||
if ids:
|
||||
col.delete(ids=ids)
|
||||
return len(ids)
|
||||
elif filters:
|
||||
col.delete(where=filters)
|
||||
return -1 # Unknown count
|
||||
return 0
|
||||
|
||||
def count(self, collection: str) -> int:
|
||||
"""Count documents in a collection."""
|
||||
col = self._get_collection(collection)
|
||||
return col.count()
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the store."""
|
||||
self._collections.clear()
|
||||
@@ -0,0 +1,242 @@
|
||||
"""
|
||||
ClickHouse Vector implementation of KnowledgeStore.
|
||||
|
||||
Requires: clickhouse-connect
|
||||
Install: pip install clickhouse-connect
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import KnowledgeStore, KnowledgeDocument
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ClickHouseKnowledgeStore(KnowledgeStore):
|
||||
"""
|
||||
ClickHouse-based knowledge store using vector search.
|
||||
|
||||
Example:
|
||||
store = ClickHouseKnowledgeStore(
|
||||
host="localhost",
|
||||
port=8123
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str = "localhost",
|
||||
port: int = 8123,
|
||||
username: str = "default",
|
||||
password: str = "",
|
||||
database: str = "praisonai",
|
||||
secure: bool = False,
|
||||
):
|
||||
try:
|
||||
import clickhouse_connect
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"clickhouse-connect is required for ClickHouse support. "
|
||||
"Install with: pip install clickhouse-connect"
|
||||
)
|
||||
|
||||
self._client = clickhouse_connect.get_client(
|
||||
host=host,
|
||||
port=port,
|
||||
username=username,
|
||||
password=password,
|
||||
database=database,
|
||||
secure=secure,
|
||||
)
|
||||
self.database = database
|
||||
|
||||
# Create database if not exists
|
||||
self._client.command(f"CREATE DATABASE IF NOT EXISTS {database}")
|
||||
logger.info(f"Connected to ClickHouse database: {database}")
|
||||
|
||||
def _table_name(self, collection: str) -> str:
|
||||
return f"{self.database}.praison_vec_{collection}"
|
||||
|
||||
def create_collection(
|
||||
self,
|
||||
name: str,
|
||||
dimension: int,
|
||||
distance: str = "cosine",
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""Create a new table with vector column."""
|
||||
table = self._table_name(name)
|
||||
|
||||
self._client.command(f"""
|
||||
CREATE TABLE IF NOT EXISTS {table} (
|
||||
id String,
|
||||
content String,
|
||||
content_hash String,
|
||||
created_at Float64,
|
||||
embedding Array(Float32)
|
||||
) ENGINE = MergeTree()
|
||||
ORDER BY id
|
||||
""")
|
||||
logger.info(f"Created ClickHouse table: {table}")
|
||||
|
||||
def delete_collection(self, name: str) -> bool:
|
||||
"""Delete a table."""
|
||||
table = self._table_name(name)
|
||||
try:
|
||||
self._client.command(f"DROP TABLE IF EXISTS {table}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to delete table {table}: {e}")
|
||||
return False
|
||||
|
||||
def collection_exists(self, name: str) -> bool:
|
||||
"""Check if a table exists."""
|
||||
table = self._table_name(name)
|
||||
result = self._client.query(f"EXISTS TABLE {table}")
|
||||
return result.result_rows[0][0] == 1
|
||||
|
||||
def list_collections(self) -> List[str]:
|
||||
"""List all collections."""
|
||||
result = self._client.query(f"""
|
||||
SELECT name FROM system.tables
|
||||
WHERE database = '{self.database}' AND name LIKE 'praison_vec_%'
|
||||
""")
|
||||
return [row[0].replace("praison_vec_", "") for row in result.result_rows]
|
||||
|
||||
def insert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert documents."""
|
||||
table = self._table_name(collection)
|
||||
|
||||
data = []
|
||||
ids = []
|
||||
for doc in documents:
|
||||
if doc.embedding is None:
|
||||
raise ValueError(f"Document {doc.id} has no embedding")
|
||||
data.append([
|
||||
doc.id,
|
||||
doc.content,
|
||||
doc.content_hash or "",
|
||||
doc.created_at,
|
||||
doc.embedding,
|
||||
])
|
||||
ids.append(doc.id)
|
||||
|
||||
self._client.insert(
|
||||
table,
|
||||
data,
|
||||
column_names=["id", "content", "content_hash", "created_at", "embedding"]
|
||||
)
|
||||
return ids
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert or update documents."""
|
||||
# ClickHouse doesn't have native upsert for MergeTree
|
||||
# Delete then insert
|
||||
ids = [doc.id for doc in documents]
|
||||
self.delete(collection, ids)
|
||||
return self.insert(collection, documents)
|
||||
|
||||
def search(
|
||||
self,
|
||||
collection: str,
|
||||
query_embedding: List[float],
|
||||
limit: int = 5,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
score_threshold: Optional[float] = None
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Search for similar documents using cosine distance."""
|
||||
table = self._table_name(collection)
|
||||
|
||||
# Convert embedding to string for query
|
||||
embedding_str = "[" + ",".join(map(str, query_embedding)) + "]"
|
||||
|
||||
query = f"""
|
||||
SELECT
|
||||
id, content, content_hash, created_at,
|
||||
1 - cosineDistance(embedding, {embedding_str}) as score
|
||||
FROM {table}
|
||||
"""
|
||||
|
||||
if score_threshold:
|
||||
query += f" WHERE 1 - cosineDistance(embedding, {embedding_str}) >= {score_threshold}"
|
||||
|
||||
query += f" ORDER BY score DESC LIMIT {limit}"
|
||||
|
||||
result = self._client.query(query)
|
||||
|
||||
documents = []
|
||||
for row in result.result_rows:
|
||||
doc = KnowledgeDocument(
|
||||
id=row[0],
|
||||
content=row[1],
|
||||
embedding=None,
|
||||
metadata={},
|
||||
content_hash=row[2],
|
||||
created_at=row[3],
|
||||
)
|
||||
documents.append(doc)
|
||||
|
||||
return documents
|
||||
|
||||
def get(
|
||||
self,
|
||||
collection: str,
|
||||
ids: List[str]
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Get documents by IDs."""
|
||||
table = self._table_name(collection)
|
||||
|
||||
id_list = ", ".join([f"'{i}'" for i in ids])
|
||||
result = self._client.query(f"""
|
||||
SELECT id, content, content_hash, created_at, embedding
|
||||
FROM {table} WHERE id IN ({id_list})
|
||||
""")
|
||||
|
||||
documents = []
|
||||
for row in result.result_rows:
|
||||
doc = KnowledgeDocument(
|
||||
id=row[0],
|
||||
content=row[1],
|
||||
embedding=list(row[4]) if row[4] else None,
|
||||
metadata={},
|
||||
content_hash=row[2],
|
||||
created_at=row[3],
|
||||
)
|
||||
documents.append(doc)
|
||||
|
||||
return documents
|
||||
|
||||
def delete(
|
||||
self,
|
||||
collection: str,
|
||||
ids: Optional[List[str]] = None,
|
||||
filters: Optional[Dict[str, Any]] = None
|
||||
) -> int:
|
||||
"""Delete documents."""
|
||||
table = self._table_name(collection)
|
||||
|
||||
if ids:
|
||||
id_list = ", ".join([f"'{i}'" for i in ids])
|
||||
self._client.command(f"ALTER TABLE {table} DELETE WHERE id IN ({id_list})")
|
||||
return len(ids)
|
||||
return 0
|
||||
|
||||
def count(self, collection: str) -> int:
|
||||
"""Count documents."""
|
||||
table = self._table_name(collection)
|
||||
result = self._client.query(f"SELECT COUNT(*) FROM {table}")
|
||||
return result.result_rows[0][0]
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the store."""
|
||||
if self._client:
|
||||
self._client.close()
|
||||
@@ -0,0 +1,438 @@
|
||||
"""
|
||||
Azure Cosmos DB Vector implementation of KnowledgeStore.
|
||||
|
||||
Supports both MongoDB API and vCore modes.
|
||||
|
||||
Requires: pymongo (for MongoDB API) or azure-cosmos (for SQL API)
|
||||
Install: pip install pymongo # or pip install azure-cosmos
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import KnowledgeStore, KnowledgeDocument
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CosmosDBVectorKnowledgeStore(KnowledgeStore):
|
||||
"""
|
||||
Azure Cosmos DB vector store for knowledge/RAG.
|
||||
|
||||
Supports MongoDB API with vector search capabilities.
|
||||
|
||||
Example:
|
||||
store = CosmosDBVectorKnowledgeStore(
|
||||
connection_string="mongodb+srv://...",
|
||||
database="praisonai",
|
||||
collection="vectors"
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
connection_string: Optional[str] = None,
|
||||
database: str = "praisonai",
|
||||
collection: str = "vectors",
|
||||
index_name: str = "vector_index",
|
||||
embedding_field: str = "embedding",
|
||||
text_field: str = "content",
|
||||
embedding_dim: int = 1536,
|
||||
api_mode: str = "mongodb", # "mongodb" or "sql"
|
||||
):
|
||||
"""
|
||||
Initialize Cosmos DB Vector store.
|
||||
|
||||
Args:
|
||||
connection_string: Azure Cosmos DB connection string
|
||||
database: Database name
|
||||
collection: Collection name
|
||||
index_name: Vector search index name
|
||||
embedding_field: Field name for embeddings
|
||||
text_field: Field name for text content
|
||||
embedding_dim: Embedding dimension
|
||||
api_mode: API mode ("mongodb" or "sql")
|
||||
"""
|
||||
self.connection_string = connection_string
|
||||
self.database_name = database
|
||||
self.collection_name = collection
|
||||
self.index_name = index_name
|
||||
self.embedding_field = embedding_field
|
||||
self.text_field = text_field
|
||||
self.embedding_dim = embedding_dim
|
||||
self.api_mode = api_mode
|
||||
|
||||
self._client = None
|
||||
self._db = None
|
||||
self._collection = None
|
||||
self._initialized = False
|
||||
|
||||
def _init_client(self):
|
||||
"""Initialize Cosmos DB client lazily."""
|
||||
if self._initialized:
|
||||
return
|
||||
|
||||
if not self.connection_string:
|
||||
import os
|
||||
self.connection_string = os.getenv("COSMOS_CONNECTION_STRING")
|
||||
if not self.connection_string:
|
||||
raise ValueError(
|
||||
"connection_string required. Set COSMOS_CONNECTION_STRING env var."
|
||||
)
|
||||
|
||||
if self.api_mode == "mongodb":
|
||||
self._init_mongodb_client()
|
||||
else:
|
||||
self._init_sql_client()
|
||||
|
||||
self._initialized = True
|
||||
|
||||
def _init_mongodb_client(self):
|
||||
"""Initialize MongoDB API client."""
|
||||
try:
|
||||
from pymongo import MongoClient
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"pymongo is required for Cosmos DB MongoDB API. "
|
||||
"Install with: pip install pymongo"
|
||||
)
|
||||
|
||||
self._client = MongoClient(self.connection_string)
|
||||
self._db = self._client[self.database_name]
|
||||
self._collection = self._db[self.collection_name]
|
||||
|
||||
def _init_sql_client(self):
|
||||
"""Initialize SQL API client."""
|
||||
try:
|
||||
from azure.cosmos import CosmosClient
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"azure-cosmos is required for Cosmos DB SQL API. "
|
||||
"Install with: pip install azure-cosmos"
|
||||
)
|
||||
|
||||
self._client = CosmosClient.from_connection_string(self.connection_string)
|
||||
self._db = self._client.get_database_client(self.database_name)
|
||||
self._collection = self._db.get_container_client(self.collection_name)
|
||||
|
||||
def create_collection(
|
||||
self,
|
||||
name: str,
|
||||
dimension: int = 1536,
|
||||
distance: str = "cosine",
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""Create collection with vector index."""
|
||||
self._init_client()
|
||||
|
||||
if self.api_mode == "mongodb":
|
||||
# Create vector search index via MongoDB command
|
||||
try:
|
||||
self._db.command({
|
||||
"createIndexes": name,
|
||||
"indexes": [{
|
||||
"name": self.index_name,
|
||||
"key": {self.embedding_field: "cosmosSearch"},
|
||||
"cosmosSearchOptions": {
|
||||
"kind": "vector-ivf",
|
||||
"numLists": 100,
|
||||
"similarity": distance,
|
||||
"dimensions": dimension
|
||||
}
|
||||
}]
|
||||
})
|
||||
except Exception as e:
|
||||
logger.warning(f"Index creation may require manual setup: {e}")
|
||||
|
||||
def delete_collection(self, name: str) -> bool:
|
||||
"""Delete collection."""
|
||||
self._init_client()
|
||||
try:
|
||||
if self.api_mode == "mongodb":
|
||||
self._db.drop_collection(name)
|
||||
else:
|
||||
self._db.delete_container(name)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete collection: {e}")
|
||||
return False
|
||||
|
||||
def collection_exists(self, name: str) -> bool:
|
||||
"""Check if collection exists."""
|
||||
self._init_client()
|
||||
if self.api_mode == "mongodb":
|
||||
return name in self._db.list_collection_names()
|
||||
return True # SQL API containers are pre-created
|
||||
|
||||
def list_collections(self) -> List[str]:
|
||||
"""List collections."""
|
||||
self._init_client()
|
||||
if self.api_mode == "mongodb":
|
||||
return self._db.list_collection_names()
|
||||
return [c["id"] for c in self._db.list_containers()]
|
||||
|
||||
def insert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert documents."""
|
||||
self._init_client()
|
||||
|
||||
coll = self._db[collection] if self.api_mode == "mongodb" else self._db.get_container_client(collection)
|
||||
ids = []
|
||||
|
||||
for doc in documents:
|
||||
item = {
|
||||
"_id" if self.api_mode == "mongodb" else "id": doc.id,
|
||||
self.text_field: doc.content,
|
||||
self.embedding_field: doc.embedding,
|
||||
"metadata": doc.metadata or {},
|
||||
"content_hash": doc.content_hash,
|
||||
"created_at": doc.created_at,
|
||||
}
|
||||
|
||||
try:
|
||||
if self.api_mode == "mongodb":
|
||||
coll.insert_one(item)
|
||||
else:
|
||||
coll.create_item(item)
|
||||
ids.append(doc.id)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to insert {doc.id}: {e}")
|
||||
|
||||
return ids
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Upsert documents."""
|
||||
self._init_client()
|
||||
|
||||
coll = self._db[collection] if self.api_mode == "mongodb" else self._db.get_container_client(collection)
|
||||
ids = []
|
||||
|
||||
for doc in documents:
|
||||
item = {
|
||||
"_id" if self.api_mode == "mongodb" else "id": doc.id,
|
||||
self.text_field: doc.content,
|
||||
self.embedding_field: doc.embedding,
|
||||
"metadata": doc.metadata or {},
|
||||
"content_hash": doc.content_hash,
|
||||
"created_at": doc.created_at,
|
||||
}
|
||||
|
||||
try:
|
||||
if self.api_mode == "mongodb":
|
||||
coll.replace_one({"_id": doc.id}, item, upsert=True)
|
||||
else:
|
||||
coll.upsert_item(item)
|
||||
ids.append(doc.id)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to upsert {doc.id}: {e}")
|
||||
|
||||
return ids
|
||||
|
||||
def search(
|
||||
self,
|
||||
collection: str,
|
||||
query_embedding: List[float],
|
||||
limit: int = 5,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
score_threshold: Optional[float] = None
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Search for similar documents using vector search."""
|
||||
self._init_client()
|
||||
|
||||
if self.api_mode == "mongodb":
|
||||
return self._search_mongodb(collection, query_embedding, limit, filters, score_threshold)
|
||||
else:
|
||||
return self._search_sql(collection, query_embedding, limit, filters, score_threshold)
|
||||
|
||||
def _search_mongodb(
|
||||
self,
|
||||
collection: str,
|
||||
query_embedding: List[float],
|
||||
limit: int,
|
||||
filters: Optional[Dict[str, Any]],
|
||||
score_threshold: Optional[float]
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""MongoDB API vector search."""
|
||||
coll = self._db[collection]
|
||||
|
||||
pipeline = [
|
||||
{
|
||||
"$search": {
|
||||
"cosmosSearch": {
|
||||
"vector": query_embedding,
|
||||
"path": self.embedding_field,
|
||||
"k": limit
|
||||
},
|
||||
"returnStoredSource": True
|
||||
}
|
||||
},
|
||||
{
|
||||
"$project": {
|
||||
"similarityScore": {"$meta": "searchScore"},
|
||||
self.text_field: 1,
|
||||
"metadata": 1,
|
||||
"content_hash": 1,
|
||||
"created_at": 1
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
if filters:
|
||||
pipeline.insert(1, {"$match": filters})
|
||||
|
||||
try:
|
||||
results = list(coll.aggregate(pipeline))
|
||||
|
||||
documents = []
|
||||
for doc in results:
|
||||
score = doc.get("similarityScore", 0)
|
||||
if score_threshold and score < score_threshold:
|
||||
continue
|
||||
|
||||
documents.append(KnowledgeDocument(
|
||||
id=str(doc.get("_id", "")),
|
||||
content=doc.get(self.text_field, ""),
|
||||
metadata={**(doc.get("metadata") or {}), "score": score},
|
||||
content_hash=doc.get("content_hash"),
|
||||
created_at=doc.get("created_at", 0)
|
||||
))
|
||||
|
||||
return documents
|
||||
except Exception as e:
|
||||
logger.error(f"Vector search failed: {e}")
|
||||
return []
|
||||
|
||||
def _search_sql(
|
||||
self,
|
||||
collection: str,
|
||||
query_embedding: List[float],
|
||||
limit: int,
|
||||
filters: Optional[Dict[str, Any]],
|
||||
score_threshold: Optional[float]
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""SQL API vector search (requires DiskANN index)."""
|
||||
coll = self._db.get_container_client(collection)
|
||||
|
||||
# SQL API uses VectorDistance function
|
||||
query = f"""
|
||||
SELECT TOP {limit} c.id, c.{self.text_field}, c.metadata, c.content_hash, c.created_at,
|
||||
VectorDistance(c.{self.embedding_field}, @embedding) AS score
|
||||
FROM c
|
||||
ORDER BY VectorDistance(c.{self.embedding_field}, @embedding)
|
||||
"""
|
||||
|
||||
try:
|
||||
results = list(coll.query_items(
|
||||
query=query,
|
||||
parameters=[{"name": "@embedding", "value": query_embedding}],
|
||||
enable_cross_partition_query=True
|
||||
))
|
||||
|
||||
documents = []
|
||||
for doc in results:
|
||||
score = doc.get("score", 0)
|
||||
if score_threshold and score < score_threshold:
|
||||
continue
|
||||
|
||||
documents.append(KnowledgeDocument(
|
||||
id=doc.get("id", ""),
|
||||
content=doc.get(self.text_field, ""),
|
||||
metadata={**(doc.get("metadata") or {}), "score": score},
|
||||
content_hash=doc.get("content_hash"),
|
||||
created_at=doc.get("created_at", 0)
|
||||
))
|
||||
|
||||
return documents
|
||||
except Exception as e:
|
||||
logger.error(f"Vector search failed: {e}")
|
||||
return []
|
||||
|
||||
def get(
|
||||
self,
|
||||
collection: str,
|
||||
ids: List[str]
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Get documents by IDs."""
|
||||
self._init_client()
|
||||
|
||||
coll = self._db[collection] if self.api_mode == "mongodb" else self._db.get_container_client(collection)
|
||||
documents = []
|
||||
|
||||
for doc_id in ids:
|
||||
try:
|
||||
if self.api_mode == "mongodb":
|
||||
doc = coll.find_one({"_id": doc_id})
|
||||
else:
|
||||
doc = coll.read_item(item=doc_id, partition_key=doc_id)
|
||||
|
||||
if doc:
|
||||
documents.append(KnowledgeDocument(
|
||||
id=str(doc.get("_id" if self.api_mode == "mongodb" else "id", "")),
|
||||
content=doc.get(self.text_field, ""),
|
||||
embedding=doc.get(self.embedding_field),
|
||||
metadata=doc.get("metadata"),
|
||||
content_hash=doc.get("content_hash"),
|
||||
created_at=doc.get("created_at", 0)
|
||||
))
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to get {doc_id}: {e}")
|
||||
|
||||
return documents
|
||||
|
||||
def delete(
|
||||
self,
|
||||
collection: str,
|
||||
ids: Optional[List[str]] = None,
|
||||
filters: Optional[Dict[str, Any]] = None
|
||||
) -> int:
|
||||
"""Delete documents."""
|
||||
self._init_client()
|
||||
|
||||
coll = self._db[collection] if self.api_mode == "mongodb" else self._db.get_container_client(collection)
|
||||
|
||||
if ids:
|
||||
count = 0
|
||||
for doc_id in ids:
|
||||
try:
|
||||
if self.api_mode == "mongodb":
|
||||
coll.delete_one({"_id": doc_id})
|
||||
else:
|
||||
coll.delete_item(item=doc_id, partition_key=doc_id)
|
||||
count += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to delete {doc_id}: {e}")
|
||||
return count
|
||||
|
||||
return 0
|
||||
|
||||
def count(self, collection: str) -> int:
|
||||
"""Count documents."""
|
||||
self._init_client()
|
||||
|
||||
if self.api_mode == "mongodb":
|
||||
return self._db[collection].count_documents({})
|
||||
else:
|
||||
# SQL API count
|
||||
coll = self._db.get_container_client(collection)
|
||||
result = list(coll.query_items(
|
||||
query="SELECT VALUE COUNT(1) FROM c",
|
||||
enable_cross_partition_query=True
|
||||
))
|
||||
return result[0] if result else 0
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the connection."""
|
||||
if self._client:
|
||||
if self.api_mode == "mongodb":
|
||||
self._client.close()
|
||||
self._client = None
|
||||
self._db = None
|
||||
self._collection = None
|
||||
self._initialized = False
|
||||
@@ -0,0 +1,286 @@
|
||||
"""
|
||||
Couchbase implementation of KnowledgeStore.
|
||||
|
||||
Requires: couchbase
|
||||
Install: pip install couchbase
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import KnowledgeStore, KnowledgeDocument
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CouchbaseKnowledgeStore(KnowledgeStore):
|
||||
"""
|
||||
Couchbase vector store for knowledge/RAG.
|
||||
|
||||
Uses Couchbase's vector search capabilities.
|
||||
|
||||
Example:
|
||||
store = CouchbaseKnowledgeStore(
|
||||
connection_string="couchbase://localhost",
|
||||
username="admin",
|
||||
password="password",
|
||||
bucket_name="praisonai"
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
connection_string: str = "couchbase://localhost",
|
||||
username: str = "Administrator",
|
||||
password: str = "",
|
||||
bucket_name: str = "praisonai",
|
||||
scope_name: str = "_default",
|
||||
collection_name: str = "vectors",
|
||||
index_name: str = "vector_index",
|
||||
vector_field: str = "embedding",
|
||||
text_field: str = "content",
|
||||
embedding_dim: int = 1536,
|
||||
):
|
||||
"""
|
||||
Initialize Couchbase knowledge store.
|
||||
|
||||
Args:
|
||||
connection_string: Couchbase connection string
|
||||
username: Couchbase username
|
||||
password: Couchbase password
|
||||
bucket_name: Bucket name
|
||||
scope_name: Scope name
|
||||
collection_name: Collection name
|
||||
index_name: Vector search index name
|
||||
vector_field: Field name for embeddings
|
||||
text_field: Field name for text content
|
||||
embedding_dim: Embedding dimension
|
||||
"""
|
||||
self.connection_string = connection_string
|
||||
self.username = username
|
||||
self.password = password
|
||||
self.bucket_name = bucket_name
|
||||
self.scope_name = scope_name
|
||||
self.collection_name = collection_name
|
||||
self.index_name = index_name
|
||||
self.vector_field = vector_field
|
||||
self.text_field = text_field
|
||||
self.embedding_dim = embedding_dim
|
||||
|
||||
self._cluster = None
|
||||
self._bucket = None
|
||||
self._collection = None
|
||||
self._initialized = False
|
||||
|
||||
def _init_client(self):
|
||||
"""Initialize Couchbase client lazily."""
|
||||
if self._initialized:
|
||||
return
|
||||
|
||||
try:
|
||||
from couchbase.cluster import Cluster
|
||||
from couchbase.options import ClusterOptions
|
||||
from couchbase.auth import PasswordAuthenticator
|
||||
from datetime import timedelta
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"couchbase is required for Couchbase support. "
|
||||
"Install with: pip install couchbase"
|
||||
)
|
||||
|
||||
auth = PasswordAuthenticator(self.username, self.password)
|
||||
options = ClusterOptions(auth)
|
||||
|
||||
self._cluster = Cluster(self.connection_string, options)
|
||||
self._cluster.wait_until_ready(timedelta(seconds=10))
|
||||
|
||||
self._bucket = self._cluster.bucket(self.bucket_name)
|
||||
scope = self._bucket.scope(self.scope_name)
|
||||
self._collection = scope.collection(self.collection_name)
|
||||
|
||||
self._initialized = True
|
||||
|
||||
def insert(
|
||||
self,
|
||||
collection_name: str,
|
||||
documents: List[KnowledgeDocument],
|
||||
embeddings: Optional[List[List[float]]] = None,
|
||||
) -> List[str]:
|
||||
"""Insert documents into the store."""
|
||||
self._init_client()
|
||||
|
||||
ids = []
|
||||
for i, doc in enumerate(documents):
|
||||
doc_id = doc.id or f"doc_{i}_{hash(doc.content)}"
|
||||
|
||||
data = {
|
||||
self.text_field: doc.content,
|
||||
"metadata": doc.metadata or {},
|
||||
}
|
||||
|
||||
if embeddings and i < len(embeddings):
|
||||
data[self.vector_field] = embeddings[i]
|
||||
|
||||
self._collection.upsert(doc_id, data)
|
||||
ids.append(doc_id)
|
||||
|
||||
return ids
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
collection_name: str,
|
||||
documents: List[KnowledgeDocument],
|
||||
embeddings: Optional[List[List[float]]] = None,
|
||||
) -> List[str]:
|
||||
"""Upsert documents (same as insert for Couchbase)."""
|
||||
return self.insert(collection_name, documents, embeddings)
|
||||
|
||||
def search(
|
||||
self,
|
||||
collection_name: str,
|
||||
query_embedding: List[float],
|
||||
limit: int = 10,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Search for similar documents using vector search."""
|
||||
self._init_client()
|
||||
|
||||
try:
|
||||
from couchbase.vector_search import VectorQuery, VectorSearch
|
||||
except ImportError:
|
||||
logger.warning("Vector search not available in this Couchbase version")
|
||||
return []
|
||||
|
||||
vector_query = VectorQuery(
|
||||
self.vector_field,
|
||||
query_embedding,
|
||||
num_candidates=limit * 2
|
||||
)
|
||||
|
||||
search_req = VectorSearch.from_vector_query(vector_query)
|
||||
|
||||
try:
|
||||
scope = self._bucket.scope(self.scope_name)
|
||||
result = scope.search(
|
||||
self.index_name,
|
||||
search_req,
|
||||
limit=limit
|
||||
)
|
||||
|
||||
documents = []
|
||||
for row in result.rows():
|
||||
doc_id = row.id
|
||||
try:
|
||||
doc_data = self._collection.get(doc_id).content_as[dict]
|
||||
documents.append(KnowledgeDocument(
|
||||
id=doc_id,
|
||||
content=doc_data.get(self.text_field, ""),
|
||||
metadata=doc_data.get("metadata", {}),
|
||||
score=row.score if hasattr(row, 'score') else None
|
||||
))
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to fetch document {doc_id}: {e}")
|
||||
|
||||
return documents
|
||||
except Exception as e:
|
||||
logger.error(f"Vector search failed: {e}")
|
||||
return []
|
||||
|
||||
def delete(
|
||||
self,
|
||||
collection_name: str,
|
||||
ids: Optional[List[str]] = None,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
) -> int:
|
||||
"""Delete documents by ID."""
|
||||
self._init_client()
|
||||
|
||||
if not ids:
|
||||
return 0
|
||||
|
||||
count = 0
|
||||
for doc_id in ids:
|
||||
try:
|
||||
self._collection.remove(doc_id)
|
||||
count += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to delete {doc_id}: {e}")
|
||||
|
||||
return count
|
||||
|
||||
def get(
|
||||
self,
|
||||
collection_name: str,
|
||||
ids: List[str],
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Get documents by ID."""
|
||||
self._init_client()
|
||||
|
||||
documents = []
|
||||
for doc_id in ids:
|
||||
try:
|
||||
doc_data = self._collection.get(doc_id).content_as[dict]
|
||||
documents.append(KnowledgeDocument(
|
||||
id=doc_id,
|
||||
content=doc_data.get(self.text_field, ""),
|
||||
metadata=doc_data.get("metadata", {})
|
||||
))
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to get {doc_id}: {e}")
|
||||
|
||||
return documents
|
||||
|
||||
def count(self, collection_name: str) -> int:
|
||||
"""Count documents in collection."""
|
||||
self._init_client()
|
||||
|
||||
try:
|
||||
query = f"SELECT COUNT(*) as count FROM `{self.bucket_name}`.`{self.scope_name}`.`{self.collection_name}`"
|
||||
result = self._cluster.query(query)
|
||||
for row in result:
|
||||
return row.get("count", 0)
|
||||
except Exception as e:
|
||||
logger.error(f"Count query failed: {e}")
|
||||
|
||||
return 0
|
||||
|
||||
def create_collection(self, collection_name: str, **kwargs) -> bool:
|
||||
"""Create collection (Couchbase collections are pre-created)."""
|
||||
self._init_client()
|
||||
return True
|
||||
|
||||
def delete_collection(self, collection_name: str) -> bool:
|
||||
"""Delete all documents in collection."""
|
||||
self._init_client()
|
||||
|
||||
try:
|
||||
query = f"DELETE FROM `{self.bucket_name}`.`{self.scope_name}`.`{self.collection_name}`"
|
||||
self._cluster.query(query)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Delete collection failed: {e}")
|
||||
return False
|
||||
|
||||
def list_collections(self) -> List[str]:
|
||||
"""List available collections."""
|
||||
self._init_client()
|
||||
|
||||
try:
|
||||
collections = self._bucket.collections().get_all_scopes()
|
||||
result = []
|
||||
for scope in collections:
|
||||
for coll in scope.collections:
|
||||
result.append(f"{scope.name}.{coll.name}")
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"List collections failed: {e}")
|
||||
return []
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the connection."""
|
||||
if self._cluster:
|
||||
self._cluster.close()
|
||||
self._cluster = None
|
||||
self._bucket = None
|
||||
self._collection = None
|
||||
self._initialized = False
|
||||
@@ -0,0 +1,216 @@
|
||||
"""
|
||||
LanceDB implementation of KnowledgeStore.
|
||||
|
||||
Requires: lancedb
|
||||
Install: pip install lancedb
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import KnowledgeStore, KnowledgeDocument
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LanceDBKnowledgeStore(KnowledgeStore):
|
||||
"""
|
||||
LanceDB-based knowledge store for vector search.
|
||||
|
||||
Embedded serverless vector database.
|
||||
|
||||
Example:
|
||||
store = LanceDBKnowledgeStore(
|
||||
path="./lancedb"
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
path: str = "./lancedb",
|
||||
uri: Optional[str] = None,
|
||||
):
|
||||
try:
|
||||
import lancedb
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"lancedb is required for LanceDB support. "
|
||||
"Install with: pip install lancedb"
|
||||
)
|
||||
|
||||
self._lancedb = lancedb
|
||||
self._db = lancedb.connect(uri or path)
|
||||
self._tables: Dict[str, Any] = {}
|
||||
logger.info(f"Connected to LanceDB at {uri or path}")
|
||||
|
||||
def create_collection(
|
||||
self,
|
||||
name: str,
|
||||
dimension: int,
|
||||
distance: str = "cosine",
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""Create a new table."""
|
||||
import pyarrow as pa
|
||||
|
||||
schema = pa.schema([
|
||||
pa.field("id", pa.string()),
|
||||
pa.field("content", pa.string()),
|
||||
pa.field("content_hash", pa.string()),
|
||||
pa.field("created_at", pa.float64()),
|
||||
pa.field("vector", pa.list_(pa.float32(), dimension)),
|
||||
])
|
||||
|
||||
self._db.create_table(name, schema=schema)
|
||||
logger.info(f"Created LanceDB table: {name}")
|
||||
|
||||
def delete_collection(self, name: str) -> bool:
|
||||
"""Delete a table."""
|
||||
try:
|
||||
self._db.drop_table(name)
|
||||
if name in self._tables:
|
||||
del self._tables[name]
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to delete table {name}: {e}")
|
||||
return False
|
||||
|
||||
def collection_exists(self, name: str) -> bool:
|
||||
"""Check if a table exists."""
|
||||
return name in self._db.table_names()
|
||||
|
||||
def list_collections(self) -> List[str]:
|
||||
"""List all tables."""
|
||||
return self._db.table_names()
|
||||
|
||||
def _get_table(self, name: str):
|
||||
"""Get or cache a table."""
|
||||
if name not in self._tables:
|
||||
self._tables[name] = self._db.open_table(name)
|
||||
return self._tables[name]
|
||||
|
||||
def insert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert documents."""
|
||||
table = self._get_table(collection)
|
||||
|
||||
data = []
|
||||
ids = []
|
||||
for doc in documents:
|
||||
if doc.embedding is None:
|
||||
raise ValueError(f"Document {doc.id} has no embedding")
|
||||
|
||||
data.append({
|
||||
"id": doc.id,
|
||||
"content": doc.content,
|
||||
"content_hash": doc.content_hash or "",
|
||||
"created_at": doc.created_at,
|
||||
"vector": doc.embedding,
|
||||
})
|
||||
ids.append(doc.id)
|
||||
|
||||
table.add(data)
|
||||
return ids
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert or update documents."""
|
||||
# LanceDB doesn't have native upsert, delete then insert
|
||||
ids = [doc.id for doc in documents]
|
||||
self.delete(collection, ids)
|
||||
return self.insert(collection, documents)
|
||||
|
||||
def search(
|
||||
self,
|
||||
collection: str,
|
||||
query_embedding: List[float],
|
||||
limit: int = 5,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
score_threshold: Optional[float] = None
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Search for similar documents."""
|
||||
table = self._get_table(collection)
|
||||
|
||||
query = table.search(query_embedding).limit(limit)
|
||||
|
||||
if filters:
|
||||
where_clauses = [f"{k} = '{v}'" for k, v in filters.items()]
|
||||
query = query.where(" AND ".join(where_clauses))
|
||||
|
||||
results = query.to_pandas()
|
||||
|
||||
documents = []
|
||||
for _, row in results.iterrows():
|
||||
if score_threshold and row.get("_distance", 1) > (1 - score_threshold):
|
||||
continue
|
||||
|
||||
doc = KnowledgeDocument(
|
||||
id=row["id"],
|
||||
content=row["content"],
|
||||
embedding=None,
|
||||
metadata={},
|
||||
content_hash=row.get("content_hash"),
|
||||
created_at=row.get("created_at", 0),
|
||||
)
|
||||
documents.append(doc)
|
||||
|
||||
return documents
|
||||
|
||||
def get(
|
||||
self,
|
||||
collection: str,
|
||||
ids: List[str]
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Get documents by IDs."""
|
||||
table = self._get_table(collection)
|
||||
|
||||
id_list = ", ".join([f"'{i}'" for i in ids])
|
||||
results = table.search().where(f"id IN ({id_list})").to_pandas()
|
||||
|
||||
documents = []
|
||||
for _, row in results.iterrows():
|
||||
doc = KnowledgeDocument(
|
||||
id=row["id"],
|
||||
content=row["content"],
|
||||
embedding=list(row["vector"]) if "vector" in row else None,
|
||||
metadata={},
|
||||
content_hash=row.get("content_hash"),
|
||||
created_at=row.get("created_at", 0),
|
||||
)
|
||||
documents.append(doc)
|
||||
|
||||
return documents
|
||||
|
||||
def delete(
|
||||
self,
|
||||
collection: str,
|
||||
ids: Optional[List[str]] = None,
|
||||
filters: Optional[Dict[str, Any]] = None
|
||||
) -> int:
|
||||
"""Delete documents."""
|
||||
table = self._get_table(collection)
|
||||
|
||||
if ids:
|
||||
id_list = ", ".join([f"'{i}'" for i in ids])
|
||||
table.delete(f"id IN ({id_list})")
|
||||
return len(ids)
|
||||
elif filters:
|
||||
where_clauses = [f"{k} = '{v}'" for k, v in filters.items()]
|
||||
table.delete(" AND ".join(where_clauses))
|
||||
return -1
|
||||
return 0
|
||||
|
||||
def count(self, collection: str) -> int:
|
||||
"""Count documents."""
|
||||
table = self._get_table(collection)
|
||||
return table.count_rows()
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the store."""
|
||||
self._tables.clear()
|
||||
@@ -0,0 +1,291 @@
|
||||
"""
|
||||
LangChain VectorStore adapter implementation of KnowledgeStore.
|
||||
|
||||
Wraps any LangChain VectorStore for use with PraisonAI.
|
||||
|
||||
Requires: langchain-core
|
||||
Install: pip install langchain-core
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import KnowledgeStore, KnowledgeDocument
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LangChainKnowledgeStore(KnowledgeStore):
|
||||
"""
|
||||
LangChain VectorStore adapter for knowledge/RAG.
|
||||
|
||||
Wraps any LangChain VectorStore for use with PraisonAI's interface.
|
||||
|
||||
Example:
|
||||
from langchain_chroma import Chroma
|
||||
|
||||
lc_store = Chroma(persist_directory="./chroma_db")
|
||||
store = LangChainKnowledgeStore(vectorstore=lc_store)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vectorstore: Any = None,
|
||||
vectorstore_cls: Optional[str] = None,
|
||||
vectorstore_kwargs: Optional[Dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Initialize LangChain adapter.
|
||||
|
||||
Args:
|
||||
vectorstore: Pre-initialized LangChain VectorStore instance
|
||||
vectorstore_cls: Class name to import (e.g., "langchain_chroma.Chroma")
|
||||
vectorstore_kwargs: Kwargs for vectorstore initialization
|
||||
"""
|
||||
self._vectorstore = vectorstore
|
||||
self._vectorstore_cls = vectorstore_cls
|
||||
self._vectorstore_kwargs = vectorstore_kwargs or {}
|
||||
self._initialized = vectorstore is not None
|
||||
|
||||
def _init_client(self):
|
||||
"""Initialize vectorstore lazily."""
|
||||
if self._initialized:
|
||||
return
|
||||
|
||||
if not self._vectorstore_cls:
|
||||
raise ValueError(
|
||||
"Either vectorstore or vectorstore_cls must be provided"
|
||||
)
|
||||
|
||||
# Dynamic import of vectorstore class
|
||||
parts = self._vectorstore_cls.rsplit(".", 1)
|
||||
if len(parts) == 2:
|
||||
module_name, class_name = parts
|
||||
import importlib
|
||||
module = importlib.import_module(module_name)
|
||||
cls = getattr(module, class_name)
|
||||
self._vectorstore = cls(**self._vectorstore_kwargs)
|
||||
else:
|
||||
raise ValueError(f"Invalid vectorstore_cls: {self._vectorstore_cls}")
|
||||
|
||||
self._initialized = True
|
||||
|
||||
def create_collection(
|
||||
self,
|
||||
name: str,
|
||||
dimension: int = 1536,
|
||||
distance: str = "cosine",
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""Create collection (handled by underlying vectorstore)."""
|
||||
self._init_client()
|
||||
logger.info(f"Collection creation handled by LangChain vectorstore")
|
||||
|
||||
def delete_collection(self, name: str) -> bool:
|
||||
"""Delete collection."""
|
||||
self._init_client()
|
||||
try:
|
||||
if hasattr(self._vectorstore, 'delete_collection'):
|
||||
self._vectorstore.delete_collection()
|
||||
return True
|
||||
logger.warning("Underlying vectorstore doesn't support delete_collection")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete collection: {e}")
|
||||
return False
|
||||
|
||||
def collection_exists(self, name: str) -> bool:
|
||||
"""Check if collection exists."""
|
||||
self._init_client()
|
||||
return self._vectorstore is not None
|
||||
|
||||
def list_collections(self) -> List[str]:
|
||||
"""List collections."""
|
||||
self._init_client()
|
||||
if hasattr(self._vectorstore, 'list_collections'):
|
||||
return self._vectorstore.list_collections()
|
||||
return ["default"]
|
||||
|
||||
def insert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert documents."""
|
||||
self._init_client()
|
||||
|
||||
try:
|
||||
from langchain_core.documents import Document as LCDocument
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"langchain-core is required. Install with: pip install langchain-core"
|
||||
)
|
||||
|
||||
lc_docs = []
|
||||
ids = []
|
||||
for doc in documents:
|
||||
lc_doc = LCDocument(
|
||||
page_content=doc.content,
|
||||
metadata={
|
||||
**(doc.metadata or {}),
|
||||
"id": doc.id,
|
||||
"content_hash": doc.content_hash,
|
||||
"created_at": doc.created_at,
|
||||
}
|
||||
)
|
||||
lc_docs.append(lc_doc)
|
||||
ids.append(doc.id)
|
||||
|
||||
if hasattr(self._vectorstore, 'add_documents'):
|
||||
self._vectorstore.add_documents(lc_docs, ids=ids)
|
||||
else:
|
||||
logger.warning("Vectorstore doesn't support add_documents")
|
||||
|
||||
return ids
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Upsert documents."""
|
||||
# Most LangChain vectorstores handle upsert via add_documents
|
||||
return self.insert(collection, documents)
|
||||
|
||||
def search(
|
||||
self,
|
||||
collection: str,
|
||||
query_embedding: List[float],
|
||||
limit: int = 5,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
score_threshold: Optional[float] = None
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Search for similar documents."""
|
||||
self._init_client()
|
||||
|
||||
try:
|
||||
if hasattr(self._vectorstore, 'similarity_search_by_vector'):
|
||||
results = self._vectorstore.similarity_search_by_vector(
|
||||
query_embedding,
|
||||
k=limit,
|
||||
filter=filters
|
||||
)
|
||||
else:
|
||||
logger.warning("Vectorstore doesn't support vector search")
|
||||
return []
|
||||
|
||||
documents = []
|
||||
for lc_doc in results:
|
||||
metadata = lc_doc.metadata or {}
|
||||
documents.append(KnowledgeDocument(
|
||||
id=metadata.get("id", ""),
|
||||
content=lc_doc.page_content,
|
||||
metadata=metadata,
|
||||
content_hash=metadata.get("content_hash"),
|
||||
created_at=metadata.get("created_at", 0)
|
||||
))
|
||||
|
||||
return documents
|
||||
except Exception as e:
|
||||
logger.error(f"Search failed: {e}")
|
||||
return []
|
||||
|
||||
def similarity_search(
|
||||
self,
|
||||
query_text: str,
|
||||
limit: int = 5,
|
||||
filters: Optional[Dict[str, Any]] = None
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""
|
||||
Text-based similarity search (native LangChain method).
|
||||
|
||||
Args:
|
||||
query_text: Query string
|
||||
limit: Maximum results
|
||||
filters: Metadata filters
|
||||
|
||||
Returns:
|
||||
List of matching documents
|
||||
"""
|
||||
self._init_client()
|
||||
|
||||
try:
|
||||
results = self._vectorstore.similarity_search(
|
||||
query_text,
|
||||
k=limit,
|
||||
filter=filters
|
||||
)
|
||||
|
||||
documents = []
|
||||
for lc_doc in results:
|
||||
metadata = lc_doc.metadata or {}
|
||||
documents.append(KnowledgeDocument(
|
||||
id=metadata.get("id", ""),
|
||||
content=lc_doc.page_content,
|
||||
metadata=metadata,
|
||||
content_hash=metadata.get("content_hash"),
|
||||
created_at=metadata.get("created_at", 0)
|
||||
))
|
||||
|
||||
return documents
|
||||
except Exception as e:
|
||||
logger.error(f"Similarity search failed: {e}")
|
||||
return []
|
||||
|
||||
def get(
|
||||
self,
|
||||
collection: str,
|
||||
ids: List[str]
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Get documents by IDs."""
|
||||
self._init_client()
|
||||
|
||||
if hasattr(self._vectorstore, 'get'):
|
||||
try:
|
||||
results = self._vectorstore.get(ids)
|
||||
documents = []
|
||||
for lc_doc in results:
|
||||
metadata = lc_doc.metadata or {}
|
||||
documents.append(KnowledgeDocument(
|
||||
id=metadata.get("id", ""),
|
||||
content=lc_doc.page_content,
|
||||
metadata=metadata
|
||||
))
|
||||
return documents
|
||||
except Exception as e:
|
||||
logger.warning(f"Get by ID failed: {e}")
|
||||
|
||||
return []
|
||||
|
||||
def delete(
|
||||
self,
|
||||
collection: str,
|
||||
ids: Optional[List[str]] = None,
|
||||
filters: Optional[Dict[str, Any]] = None
|
||||
) -> int:
|
||||
"""Delete documents."""
|
||||
self._init_client()
|
||||
|
||||
if ids and hasattr(self._vectorstore, 'delete'):
|
||||
try:
|
||||
self._vectorstore.delete(ids)
|
||||
return len(ids)
|
||||
except Exception as e:
|
||||
logger.error(f"Delete failed: {e}")
|
||||
|
||||
return 0
|
||||
|
||||
def count(self, collection: str) -> int:
|
||||
"""Count documents."""
|
||||
self._init_client()
|
||||
|
||||
if hasattr(self._vectorstore, '__len__'):
|
||||
return len(self._vectorstore)
|
||||
|
||||
logger.warning("Vectorstore doesn't support counting")
|
||||
return -1
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the client."""
|
||||
self._vectorstore = None
|
||||
self._initialized = False
|
||||
@@ -0,0 +1,212 @@
|
||||
"""
|
||||
LightRAG adapter implementation of KnowledgeStore.
|
||||
|
||||
Requires: lightrag
|
||||
Install: pip install lightrag
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import KnowledgeStore, KnowledgeDocument
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LightRAGKnowledgeStore(KnowledgeStore):
|
||||
"""
|
||||
LightRAG adapter for knowledge/RAG.
|
||||
|
||||
Wraps LightRAG for use with PraisonAI's KnowledgeStore interface.
|
||||
|
||||
Example:
|
||||
store = LightRAGKnowledgeStore(
|
||||
working_dir="./lightrag_data"
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
working_dir: str = "./lightrag_data",
|
||||
llm_model: str = "gpt-4o-mini",
|
||||
embedding_model: str = "text-embedding-3-small",
|
||||
**kwargs
|
||||
):
|
||||
"""
|
||||
Initialize LightRAG adapter.
|
||||
|
||||
Args:
|
||||
working_dir: Directory for LightRAG data
|
||||
llm_model: LLM model for graph extraction
|
||||
embedding_model: Embedding model name
|
||||
**kwargs: Additional LightRAG options
|
||||
"""
|
||||
self.working_dir = working_dir
|
||||
self.llm_model = llm_model
|
||||
self.embedding_model = embedding_model
|
||||
self.kwargs = kwargs
|
||||
|
||||
self._rag = None
|
||||
self._initialized = False
|
||||
|
||||
def _init_client(self):
|
||||
"""Initialize LightRAG client lazily."""
|
||||
if self._initialized:
|
||||
return
|
||||
|
||||
try:
|
||||
from lightrag import LightRAG, QueryParam
|
||||
from lightrag.llm import gpt_4o_mini_complete, openai_embedding
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"lightrag is required for LightRAG support. "
|
||||
"Install with: pip install lightrag"
|
||||
)
|
||||
|
||||
self._QueryParam = QueryParam
|
||||
self._rag = LightRAG(
|
||||
working_dir=self.working_dir,
|
||||
llm_model_func=gpt_4o_mini_complete,
|
||||
embedding_func=openai_embedding,
|
||||
**self.kwargs
|
||||
)
|
||||
self._initialized = True
|
||||
|
||||
def create_collection(
|
||||
self,
|
||||
name: str,
|
||||
dimension: int = 1536,
|
||||
distance: str = "cosine",
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""Create collection (LightRAG uses working directory)."""
|
||||
self._init_client()
|
||||
logger.info(f"LightRAG uses working_dir: {self.working_dir}")
|
||||
|
||||
def delete_collection(self, name: str) -> bool:
|
||||
"""Delete collection data."""
|
||||
import shutil
|
||||
import os
|
||||
|
||||
try:
|
||||
if os.path.exists(self.working_dir):
|
||||
shutil.rmtree(self.working_dir)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete collection: {e}")
|
||||
return False
|
||||
|
||||
def collection_exists(self, name: str) -> bool:
|
||||
"""Check if working directory exists."""
|
||||
import os
|
||||
return os.path.exists(self.working_dir)
|
||||
|
||||
def list_collections(self) -> List[str]:
|
||||
"""List collections (single collection per working_dir)."""
|
||||
import os
|
||||
if os.path.exists(self.working_dir):
|
||||
return ["default"]
|
||||
return []
|
||||
|
||||
def insert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert documents into LightRAG."""
|
||||
self._init_client()
|
||||
|
||||
ids = []
|
||||
for doc in documents:
|
||||
try:
|
||||
self._rag.insert(doc.content)
|
||||
ids.append(doc.id)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to insert document {doc.id}: {e}")
|
||||
|
||||
return ids
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Upsert documents (same as insert for LightRAG)."""
|
||||
return self.insert(collection, documents)
|
||||
|
||||
def search(
|
||||
self,
|
||||
collection: str,
|
||||
query_embedding: List[float],
|
||||
limit: int = 5,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
score_threshold: Optional[float] = None
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Search using LightRAG's query method."""
|
||||
self._init_client()
|
||||
|
||||
# LightRAG uses text query, not embeddings directly
|
||||
# This is a limitation - we need the original query text
|
||||
logger.warning(
|
||||
"LightRAG search requires text query, not embeddings. "
|
||||
"Use query() method directly for best results."
|
||||
)
|
||||
return []
|
||||
|
||||
def query(
|
||||
self,
|
||||
query_text: str,
|
||||
mode: str = "hybrid",
|
||||
limit: int = 5
|
||||
) -> str:
|
||||
"""
|
||||
Query LightRAG with text (native method).
|
||||
|
||||
Args:
|
||||
query_text: The query string
|
||||
mode: Query mode (naive, local, global, hybrid)
|
||||
limit: Maximum results
|
||||
|
||||
Returns:
|
||||
Query response as string
|
||||
"""
|
||||
self._init_client()
|
||||
|
||||
try:
|
||||
result = self._rag.query(
|
||||
query_text,
|
||||
param=self._QueryParam(mode=mode, top_k=limit)
|
||||
)
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"Query failed: {e}")
|
||||
return ""
|
||||
|
||||
def get(
|
||||
self,
|
||||
collection: str,
|
||||
ids: List[str]
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Get documents by IDs (not supported by LightRAG)."""
|
||||
logger.warning("LightRAG does not support get by ID")
|
||||
return []
|
||||
|
||||
def delete(
|
||||
self,
|
||||
collection: str,
|
||||
ids: Optional[List[str]] = None,
|
||||
filters: Optional[Dict[str, Any]] = None
|
||||
) -> int:
|
||||
"""Delete documents (not directly supported)."""
|
||||
logger.warning("LightRAG does not support individual document deletion")
|
||||
return 0
|
||||
|
||||
def count(self, collection: str) -> int:
|
||||
"""Count documents (not directly supported)."""
|
||||
logger.warning("LightRAG does not support document counting")
|
||||
return -1
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the client."""
|
||||
self._rag = None
|
||||
self._initialized = False
|
||||
@@ -0,0 +1,256 @@
|
||||
"""
|
||||
LlamaIndex VectorStore adapter implementation of KnowledgeStore.
|
||||
|
||||
Wraps any LlamaIndex VectorStore for use with PraisonAI.
|
||||
|
||||
Requires: llama-index-core
|
||||
Install: pip install llama-index-core
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import KnowledgeStore, KnowledgeDocument
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LlamaIndexKnowledgeStore(KnowledgeStore):
|
||||
"""
|
||||
LlamaIndex VectorStore adapter for knowledge/RAG.
|
||||
|
||||
Wraps any LlamaIndex VectorStore for use with PraisonAI's interface.
|
||||
|
||||
Example:
|
||||
from llama_index.vector_stores.chroma import ChromaVectorStore
|
||||
|
||||
li_store = ChromaVectorStore(chroma_collection=collection)
|
||||
store = LlamaIndexKnowledgeStore(vector_store=li_store)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vector_store: Any = None,
|
||||
vector_store_cls: Optional[str] = None,
|
||||
vector_store_kwargs: Optional[Dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Initialize LlamaIndex adapter.
|
||||
|
||||
Args:
|
||||
vector_store: Pre-initialized LlamaIndex VectorStore instance
|
||||
vector_store_cls: Class name to import (e.g., "llama_index.vector_stores.chroma.ChromaVectorStore")
|
||||
vector_store_kwargs: Kwargs for vector_store initialization
|
||||
"""
|
||||
self._vector_store = vector_store
|
||||
self._vector_store_cls = vector_store_cls
|
||||
self._vector_store_kwargs = vector_store_kwargs or {}
|
||||
self._initialized = vector_store is not None
|
||||
|
||||
def _init_client(self):
|
||||
"""Initialize vector store lazily."""
|
||||
if self._initialized:
|
||||
return
|
||||
|
||||
if not self._vector_store_cls:
|
||||
raise ValueError(
|
||||
"Either vector_store or vector_store_cls must be provided"
|
||||
)
|
||||
|
||||
# Dynamic import of vector store class
|
||||
parts = self._vector_store_cls.rsplit(".", 1)
|
||||
if len(parts) == 2:
|
||||
module_name, class_name = parts
|
||||
import importlib
|
||||
module = importlib.import_module(module_name)
|
||||
cls = getattr(module, class_name)
|
||||
self._vector_store = cls(**self._vector_store_kwargs)
|
||||
else:
|
||||
raise ValueError(f"Invalid vector_store_cls: {self._vector_store_cls}")
|
||||
|
||||
self._initialized = True
|
||||
|
||||
def create_collection(
|
||||
self,
|
||||
name: str,
|
||||
dimension: int = 1536,
|
||||
distance: str = "cosine",
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""Create collection (handled by underlying vector store)."""
|
||||
self._init_client()
|
||||
logger.info("Collection creation handled by LlamaIndex vector store")
|
||||
|
||||
def delete_collection(self, name: str) -> bool:
|
||||
"""Delete collection."""
|
||||
self._init_client()
|
||||
try:
|
||||
if hasattr(self._vector_store, 'delete'):
|
||||
self._vector_store.delete(delete_all=True)
|
||||
return True
|
||||
logger.warning("Underlying vector store doesn't support delete")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete collection: {e}")
|
||||
return False
|
||||
|
||||
def collection_exists(self, name: str) -> bool:
|
||||
"""Check if collection exists."""
|
||||
self._init_client()
|
||||
return self._vector_store is not None
|
||||
|
||||
def list_collections(self) -> List[str]:
|
||||
"""List collections."""
|
||||
self._init_client()
|
||||
return ["default"]
|
||||
|
||||
def insert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert documents."""
|
||||
self._init_client()
|
||||
|
||||
try:
|
||||
from llama_index.core.schema import TextNode
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"llama-index-core is required. Install with: pip install llama-index-core"
|
||||
)
|
||||
|
||||
nodes = []
|
||||
ids = []
|
||||
for doc in documents:
|
||||
node = TextNode(
|
||||
text=doc.content,
|
||||
id_=doc.id,
|
||||
metadata={
|
||||
**(doc.metadata or {}),
|
||||
"content_hash": doc.content_hash,
|
||||
"created_at": doc.created_at,
|
||||
},
|
||||
embedding=doc.embedding
|
||||
)
|
||||
nodes.append(node)
|
||||
ids.append(doc.id)
|
||||
|
||||
if hasattr(self._vector_store, 'add'):
|
||||
self._vector_store.add(nodes)
|
||||
else:
|
||||
logger.warning("Vector store doesn't support add")
|
||||
|
||||
return ids
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Upsert documents."""
|
||||
return self.insert(collection, documents)
|
||||
|
||||
def search(
|
||||
self,
|
||||
collection: str,
|
||||
query_embedding: List[float],
|
||||
limit: int = 5,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
score_threshold: Optional[float] = None
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Search for similar documents."""
|
||||
self._init_client()
|
||||
|
||||
try:
|
||||
from llama_index.core.vector_stores import VectorStoreQuery
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"llama-index-core is required. Install with: pip install llama-index-core"
|
||||
)
|
||||
|
||||
try:
|
||||
query = VectorStoreQuery(
|
||||
query_embedding=query_embedding,
|
||||
similarity_top_k=limit,
|
||||
)
|
||||
|
||||
result = self._vector_store.query(query)
|
||||
|
||||
documents = []
|
||||
for i, node in enumerate(result.nodes or []):
|
||||
score = result.similarities[i] if result.similarities else None
|
||||
if score_threshold and score and score < score_threshold:
|
||||
continue
|
||||
|
||||
metadata = node.metadata or {}
|
||||
if score:
|
||||
metadata["score"] = score
|
||||
|
||||
documents.append(KnowledgeDocument(
|
||||
id=node.node_id or node.id_,
|
||||
content=node.text or node.get_content(),
|
||||
embedding=node.embedding,
|
||||
metadata=metadata,
|
||||
content_hash=metadata.get("content_hash"),
|
||||
created_at=metadata.get("created_at", 0)
|
||||
))
|
||||
|
||||
return documents
|
||||
except Exception as e:
|
||||
logger.error(f"Search failed: {e}")
|
||||
return []
|
||||
|
||||
def get(
|
||||
self,
|
||||
collection: str,
|
||||
ids: List[str]
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Get documents by IDs."""
|
||||
self._init_client()
|
||||
|
||||
documents = []
|
||||
for doc_id in ids:
|
||||
try:
|
||||
if hasattr(self._vector_store, 'get'):
|
||||
node = self._vector_store.get(doc_id)
|
||||
if node:
|
||||
metadata = node.metadata or {}
|
||||
documents.append(KnowledgeDocument(
|
||||
id=node.node_id or node.id_,
|
||||
content=node.text or node.get_content(),
|
||||
embedding=node.embedding,
|
||||
metadata=metadata
|
||||
))
|
||||
except Exception as e:
|
||||
logger.warning(f"Get by ID failed for {doc_id}: {e}")
|
||||
|
||||
return documents
|
||||
|
||||
def delete(
|
||||
self,
|
||||
collection: str,
|
||||
ids: Optional[List[str]] = None,
|
||||
filters: Optional[Dict[str, Any]] = None
|
||||
) -> int:
|
||||
"""Delete documents."""
|
||||
self._init_client()
|
||||
|
||||
if ids and hasattr(self._vector_store, 'delete'):
|
||||
try:
|
||||
self._vector_store.delete(ref_doc_id=ids[0] if len(ids) == 1 else None)
|
||||
return len(ids)
|
||||
except Exception as e:
|
||||
logger.error(f"Delete failed: {e}")
|
||||
|
||||
return 0
|
||||
|
||||
def count(self, collection: str) -> int:
|
||||
"""Count documents."""
|
||||
self._init_client()
|
||||
logger.warning("LlamaIndex vector stores don't typically support counting")
|
||||
return -1
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the client."""
|
||||
self._vector_store = None
|
||||
self._initialized = False
|
||||
@@ -0,0 +1,277 @@
|
||||
"""
|
||||
Milvus implementation of KnowledgeStore.
|
||||
|
||||
Requires: pymilvus
|
||||
Install: pip install pymilvus
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import KnowledgeStore, KnowledgeDocument
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MilvusKnowledgeStore(KnowledgeStore):
|
||||
"""
|
||||
Milvus-based knowledge store for vector search.
|
||||
|
||||
Example:
|
||||
store = MilvusKnowledgeStore(
|
||||
host="localhost",
|
||||
port=19530
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: Optional[str] = None,
|
||||
host: str = "localhost",
|
||||
port: int = 19530,
|
||||
token: Optional[str] = None,
|
||||
db_name: str = "default",
|
||||
):
|
||||
try:
|
||||
from pymilvus import connections, Collection, FieldSchema, CollectionSchema, DataType, utility
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"pymilvus is required for Milvus support. "
|
||||
"Install with: pip install pymilvus"
|
||||
)
|
||||
|
||||
self._pymilvus = {
|
||||
"connections": connections,
|
||||
"Collection": Collection,
|
||||
"FieldSchema": FieldSchema,
|
||||
"CollectionSchema": CollectionSchema,
|
||||
"DataType": DataType,
|
||||
"utility": utility,
|
||||
}
|
||||
|
||||
if url:
|
||||
# Parse milvus://host:port
|
||||
from urllib.parse import urlparse
|
||||
parsed = urlparse(url)
|
||||
host = parsed.hostname or host
|
||||
port = parsed.port or port
|
||||
|
||||
connections.connect(
|
||||
alias="default",
|
||||
host=host,
|
||||
port=port,
|
||||
token=token,
|
||||
db_name=db_name,
|
||||
)
|
||||
logger.info(f"Connected to Milvus at {host}:{port}")
|
||||
|
||||
def create_collection(
|
||||
self,
|
||||
name: str,
|
||||
dimension: int,
|
||||
distance: str = "cosine",
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""Create a new collection."""
|
||||
FieldSchema = self._pymilvus["FieldSchema"]
|
||||
CollectionSchema = self._pymilvus["CollectionSchema"]
|
||||
DataType = self._pymilvus["DataType"]
|
||||
Collection = self._pymilvus["Collection"]
|
||||
|
||||
fields = [
|
||||
FieldSchema(name="id", dtype=DataType.VARCHAR, is_primary=True, max_length=256),
|
||||
FieldSchema(name="content", dtype=DataType.VARCHAR, max_length=65535),
|
||||
FieldSchema(name="content_hash", dtype=DataType.VARCHAR, max_length=64),
|
||||
FieldSchema(name="created_at", dtype=DataType.DOUBLE),
|
||||
FieldSchema(name="vector", dtype=DataType.FLOAT_VECTOR, dim=dimension),
|
||||
]
|
||||
|
||||
schema = CollectionSchema(fields=fields, description="PraisonAI knowledge store")
|
||||
collection = Collection(name=name, schema=schema)
|
||||
|
||||
# Create index
|
||||
metric_map = {"cosine": "COSINE", "euclidean": "L2", "dot": "IP"}
|
||||
index_params = {
|
||||
"metric_type": metric_map.get(distance, "COSINE"),
|
||||
"index_type": "HNSW",
|
||||
"params": {"M": 16, "efConstruction": 256}
|
||||
}
|
||||
collection.create_index(field_name="vector", index_params=index_params)
|
||||
logger.info(f"Created Milvus collection: {name}")
|
||||
|
||||
def delete_collection(self, name: str) -> bool:
|
||||
"""Delete a collection."""
|
||||
utility = self._pymilvus["utility"]
|
||||
try:
|
||||
utility.drop_collection(name)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to delete collection {name}: {e}")
|
||||
return False
|
||||
|
||||
def collection_exists(self, name: str) -> bool:
|
||||
"""Check if a collection exists."""
|
||||
utility = self._pymilvus["utility"]
|
||||
return utility.has_collection(name)
|
||||
|
||||
def list_collections(self) -> List[str]:
|
||||
"""List all collections."""
|
||||
utility = self._pymilvus["utility"]
|
||||
return utility.list_collections()
|
||||
|
||||
def insert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert documents."""
|
||||
Collection = self._pymilvus["Collection"]
|
||||
col = Collection(collection)
|
||||
col.load()
|
||||
|
||||
data = [
|
||||
[doc.id for doc in documents],
|
||||
[doc.content for doc in documents],
|
||||
[doc.content_hash or "" for doc in documents],
|
||||
[doc.created_at for doc in documents],
|
||||
[doc.embedding for doc in documents],
|
||||
]
|
||||
|
||||
col.insert(data)
|
||||
col.flush()
|
||||
return [doc.id for doc in documents]
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert or update documents."""
|
||||
Collection = self._pymilvus["Collection"]
|
||||
col = Collection(collection)
|
||||
col.load()
|
||||
|
||||
data = [
|
||||
[doc.id for doc in documents],
|
||||
[doc.content for doc in documents],
|
||||
[doc.content_hash or "" for doc in documents],
|
||||
[doc.created_at for doc in documents],
|
||||
[doc.embedding for doc in documents],
|
||||
]
|
||||
|
||||
col.upsert(data)
|
||||
col.flush()
|
||||
return [doc.id for doc in documents]
|
||||
|
||||
def search(
|
||||
self,
|
||||
collection: str,
|
||||
query_embedding: List[float],
|
||||
limit: int = 5,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
score_threshold: Optional[float] = None
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Search for similar documents."""
|
||||
Collection = self._pymilvus["Collection"]
|
||||
col = Collection(collection)
|
||||
col.load()
|
||||
|
||||
search_params = {"metric_type": "COSINE", "params": {"ef": 64}}
|
||||
|
||||
expr = None
|
||||
if filters:
|
||||
conditions = [f'{k} == "{v}"' for k, v in filters.items()]
|
||||
expr = " and ".join(conditions)
|
||||
|
||||
results = col.search(
|
||||
data=[query_embedding],
|
||||
anns_field="vector",
|
||||
param=search_params,
|
||||
limit=limit,
|
||||
expr=expr,
|
||||
output_fields=["id", "content", "content_hash", "created_at"],
|
||||
)
|
||||
|
||||
documents = []
|
||||
for hits in results:
|
||||
for hit in hits:
|
||||
if score_threshold and hit.score < score_threshold:
|
||||
continue
|
||||
|
||||
doc = KnowledgeDocument(
|
||||
id=hit.entity.get("id"),
|
||||
content=hit.entity.get("content", ""),
|
||||
embedding=None,
|
||||
metadata={},
|
||||
content_hash=hit.entity.get("content_hash"),
|
||||
created_at=hit.entity.get("created_at", 0),
|
||||
)
|
||||
documents.append(doc)
|
||||
|
||||
return documents
|
||||
|
||||
def get(
|
||||
self,
|
||||
collection: str,
|
||||
ids: List[str]
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Get documents by IDs."""
|
||||
Collection = self._pymilvus["Collection"]
|
||||
col = Collection(collection)
|
||||
col.load()
|
||||
|
||||
id_list = ", ".join([f'"{i}"' for i in ids])
|
||||
expr = f"id in [{id_list}]"
|
||||
|
||||
results = col.query(
|
||||
expr=expr,
|
||||
output_fields=["id", "content", "content_hash", "created_at", "vector"],
|
||||
)
|
||||
|
||||
documents = []
|
||||
for row in results:
|
||||
doc = KnowledgeDocument(
|
||||
id=row["id"],
|
||||
content=row.get("content", ""),
|
||||
embedding=row.get("vector"),
|
||||
metadata={},
|
||||
content_hash=row.get("content_hash"),
|
||||
created_at=row.get("created_at", 0),
|
||||
)
|
||||
documents.append(doc)
|
||||
|
||||
return documents
|
||||
|
||||
def delete(
|
||||
self,
|
||||
collection: str,
|
||||
ids: Optional[List[str]] = None,
|
||||
filters: Optional[Dict[str, Any]] = None
|
||||
) -> int:
|
||||
"""Delete documents."""
|
||||
Collection = self._pymilvus["Collection"]
|
||||
col = Collection(collection)
|
||||
col.load()
|
||||
|
||||
if ids:
|
||||
id_list = ", ".join([f'"{i}"' for i in ids])
|
||||
expr = f"id in [{id_list}]"
|
||||
col.delete(expr)
|
||||
return len(ids)
|
||||
elif filters:
|
||||
conditions = [f'{k} == "{v}"' for k, v in filters.items()]
|
||||
expr = " and ".join(conditions)
|
||||
col.delete(expr)
|
||||
return -1
|
||||
return 0
|
||||
|
||||
def count(self, collection: str) -> int:
|
||||
"""Count documents."""
|
||||
Collection = self._pymilvus["Collection"]
|
||||
col = Collection(collection)
|
||||
return col.num_entities
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the store."""
|
||||
connections = self._pymilvus["connections"]
|
||||
connections.disconnect("default")
|
||||
@@ -0,0 +1,306 @@
|
||||
"""
|
||||
MongoDB Vector Search implementation of KnowledgeStore.
|
||||
|
||||
Requires: pymongo
|
||||
Install: pip install pymongo
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import KnowledgeStore, KnowledgeDocument
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MongoDBVectorKnowledgeStore(KnowledgeStore):
|
||||
"""
|
||||
MongoDB Atlas Vector Search store for knowledge/RAG.
|
||||
|
||||
Uses MongoDB Atlas Vector Search capabilities.
|
||||
|
||||
Example:
|
||||
store = MongoDBVectorKnowledgeStore(
|
||||
url="mongodb+srv://...",
|
||||
database="praisonai",
|
||||
collection="vectors"
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: str = "mongodb://localhost:27017",
|
||||
database: str = "praisonai",
|
||||
collection: str = "vectors",
|
||||
index_name: str = "vector_index",
|
||||
embedding_field: str = "embedding",
|
||||
text_field: str = "content",
|
||||
embedding_dim: int = 1536,
|
||||
):
|
||||
"""
|
||||
Initialize MongoDB Vector store.
|
||||
|
||||
Args:
|
||||
url: MongoDB connection URL
|
||||
database: Database name
|
||||
collection: Collection name
|
||||
index_name: Vector search index name
|
||||
embedding_field: Field name for embeddings
|
||||
text_field: Field name for text content
|
||||
embedding_dim: Embedding dimension
|
||||
"""
|
||||
self.url = url
|
||||
self.database_name = database
|
||||
self.collection_name = collection
|
||||
self.index_name = index_name
|
||||
self.embedding_field = embedding_field
|
||||
self.text_field = text_field
|
||||
self.embedding_dim = embedding_dim
|
||||
|
||||
self._client = None
|
||||
self._db = None
|
||||
self._collection = None
|
||||
self._initialized = False
|
||||
|
||||
def _init_client(self):
|
||||
"""Initialize MongoDB client lazily."""
|
||||
if self._initialized:
|
||||
return
|
||||
|
||||
try:
|
||||
from pymongo import MongoClient
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"pymongo is required for MongoDB Vector support. "
|
||||
"Install with: pip install pymongo"
|
||||
)
|
||||
|
||||
self._client = MongoClient(self.url)
|
||||
self._db = self._client[self.database_name]
|
||||
self._collection = self._db[self.collection_name]
|
||||
self._initialized = True
|
||||
|
||||
def create_collection(
|
||||
self,
|
||||
name: str,
|
||||
dimension: int = 1536,
|
||||
distance: str = "cosine",
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""Create collection (MongoDB creates on first insert)."""
|
||||
self._init_client()
|
||||
# MongoDB creates collections automatically on first insert
|
||||
# Vector search index must be created via Atlas UI or API
|
||||
logger.info(
|
||||
f"Collection '{name}' will be created on first insert. "
|
||||
f"Note: Vector search index must be created via MongoDB Atlas."
|
||||
)
|
||||
|
||||
def delete_collection(self, name: str) -> bool:
|
||||
"""Delete a collection."""
|
||||
self._init_client()
|
||||
try:
|
||||
self._db.drop_collection(name)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete collection {name}: {e}")
|
||||
return False
|
||||
|
||||
def collection_exists(self, name: str) -> bool:
|
||||
"""Check if collection exists."""
|
||||
self._init_client()
|
||||
return name in self._db.list_collection_names()
|
||||
|
||||
def list_collections(self) -> List[str]:
|
||||
"""List all collections."""
|
||||
self._init_client()
|
||||
return self._db.list_collection_names()
|
||||
|
||||
def insert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert documents into collection."""
|
||||
self._init_client()
|
||||
|
||||
coll = self._db[collection]
|
||||
ids = []
|
||||
|
||||
for doc in documents:
|
||||
mongo_doc = {
|
||||
"_id": doc.id,
|
||||
self.text_field: doc.content,
|
||||
"metadata": doc.metadata or {},
|
||||
"content_hash": doc.content_hash,
|
||||
"created_at": doc.created_at,
|
||||
}
|
||||
|
||||
if doc.embedding:
|
||||
mongo_doc[self.embedding_field] = doc.embedding
|
||||
|
||||
try:
|
||||
coll.insert_one(mongo_doc)
|
||||
ids.append(doc.id)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to insert document {doc.id}: {e}")
|
||||
|
||||
return ids
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Upsert documents into collection."""
|
||||
self._init_client()
|
||||
|
||||
coll = self._db[collection]
|
||||
ids = []
|
||||
|
||||
for doc in documents:
|
||||
mongo_doc = {
|
||||
self.text_field: doc.content,
|
||||
"metadata": doc.metadata or {},
|
||||
"content_hash": doc.content_hash,
|
||||
"created_at": doc.created_at,
|
||||
}
|
||||
|
||||
if doc.embedding:
|
||||
mongo_doc[self.embedding_field] = doc.embedding
|
||||
|
||||
try:
|
||||
coll.replace_one(
|
||||
{"_id": doc.id},
|
||||
{"_id": doc.id, **mongo_doc},
|
||||
upsert=True
|
||||
)
|
||||
ids.append(doc.id)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to upsert document {doc.id}: {e}")
|
||||
|
||||
return ids
|
||||
|
||||
def search(
|
||||
self,
|
||||
collection: str,
|
||||
query_embedding: List[float],
|
||||
limit: int = 5,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
score_threshold: Optional[float] = None
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Search for similar documents using vector search."""
|
||||
self._init_client()
|
||||
|
||||
coll = self._db[collection]
|
||||
|
||||
# Build aggregation pipeline for vector search
|
||||
pipeline = [
|
||||
{
|
||||
"$vectorSearch": {
|
||||
"index": self.index_name,
|
||||
"path": self.embedding_field,
|
||||
"queryVector": query_embedding,
|
||||
"numCandidates": limit * 10,
|
||||
"limit": limit
|
||||
}
|
||||
},
|
||||
{
|
||||
"$project": {
|
||||
"_id": 1,
|
||||
self.text_field: 1,
|
||||
"metadata": 1,
|
||||
"content_hash": 1,
|
||||
"created_at": 1,
|
||||
"score": {"$meta": "vectorSearchScore"}
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
# Add filter if provided
|
||||
if filters:
|
||||
pipeline[0]["$vectorSearch"]["filter"] = filters
|
||||
|
||||
try:
|
||||
results = list(coll.aggregate(pipeline))
|
||||
|
||||
documents = []
|
||||
for doc in results:
|
||||
score = doc.get("score", 0)
|
||||
if score_threshold and score < score_threshold:
|
||||
continue
|
||||
|
||||
documents.append(KnowledgeDocument(
|
||||
id=str(doc["_id"]),
|
||||
content=doc.get(self.text_field, ""),
|
||||
metadata={**(doc.get("metadata") or {}), "score": score},
|
||||
content_hash=doc.get("content_hash"),
|
||||
created_at=doc.get("created_at", 0)
|
||||
))
|
||||
|
||||
return documents
|
||||
except Exception as e:
|
||||
logger.error(f"Vector search failed: {e}")
|
||||
return []
|
||||
|
||||
def get(
|
||||
self,
|
||||
collection: str,
|
||||
ids: List[str]
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Get documents by IDs."""
|
||||
self._init_client()
|
||||
|
||||
coll = self._db[collection]
|
||||
documents = []
|
||||
|
||||
for doc_id in ids:
|
||||
try:
|
||||
doc = coll.find_one({"_id": doc_id})
|
||||
if doc:
|
||||
documents.append(KnowledgeDocument(
|
||||
id=str(doc["_id"]),
|
||||
content=doc.get(self.text_field, ""),
|
||||
embedding=doc.get(self.embedding_field),
|
||||
metadata=doc.get("metadata"),
|
||||
content_hash=doc.get("content_hash"),
|
||||
created_at=doc.get("created_at", 0)
|
||||
))
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to get document {doc_id}: {e}")
|
||||
|
||||
return documents
|
||||
|
||||
def delete(
|
||||
self,
|
||||
collection: str,
|
||||
ids: Optional[List[str]] = None,
|
||||
filters: Optional[Dict[str, Any]] = None
|
||||
) -> int:
|
||||
"""Delete documents by IDs or filters."""
|
||||
self._init_client()
|
||||
|
||||
coll = self._db[collection]
|
||||
|
||||
if ids:
|
||||
result = coll.delete_many({"_id": {"$in": ids}})
|
||||
return result.deleted_count
|
||||
elif filters:
|
||||
result = coll.delete_many(filters)
|
||||
return result.deleted_count
|
||||
else:
|
||||
return 0
|
||||
|
||||
def count(self, collection: str) -> int:
|
||||
"""Count documents in collection."""
|
||||
self._init_client()
|
||||
return self._db[collection].count_documents({})
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the connection."""
|
||||
if self._client:
|
||||
self._client.close()
|
||||
self._client = None
|
||||
self._db = None
|
||||
self._collection = None
|
||||
self._initialized = False
|
||||
@@ -0,0 +1,335 @@
|
||||
"""
|
||||
PGVector implementation of KnowledgeStore.
|
||||
|
||||
Requires: pgvector, psycopg2-binary
|
||||
Install: pip install pgvector psycopg2-binary
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import KnowledgeStore, KnowledgeDocument
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PGVectorKnowledgeStore(KnowledgeStore):
|
||||
"""
|
||||
PGVector-based knowledge store for vector search.
|
||||
|
||||
Uses PostgreSQL with pgvector extension.
|
||||
|
||||
Example:
|
||||
store = PGVectorKnowledgeStore(
|
||||
url="postgresql://localhost:5432/praisonai"
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: Optional[str] = None,
|
||||
host: str = "localhost",
|
||||
port: int = 5432,
|
||||
database: str = "praisonai",
|
||||
user: str = "postgres",
|
||||
password: str = "",
|
||||
schema: str = "public",
|
||||
auto_create_extension: bool = True,
|
||||
):
|
||||
try:
|
||||
import psycopg2
|
||||
from psycopg2 import pool as pg_pool
|
||||
from psycopg2.extras import RealDictCursor
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"psycopg2 is required for PGVector support. "
|
||||
"Install with: pip install psycopg2-binary"
|
||||
)
|
||||
|
||||
self._psycopg2 = psycopg2
|
||||
self._RealDictCursor = RealDictCursor
|
||||
self.schema = schema
|
||||
|
||||
if url:
|
||||
self._pool = pg_pool.ThreadedConnectionPool(1, 5, url)
|
||||
else:
|
||||
self._pool = pg_pool.ThreadedConnectionPool(
|
||||
1, 5,
|
||||
host=host, port=port, database=database,
|
||||
user=user, password=password,
|
||||
)
|
||||
|
||||
if auto_create_extension:
|
||||
self._create_extension()
|
||||
|
||||
def _get_conn(self):
|
||||
return self._pool.getconn()
|
||||
|
||||
def _put_conn(self, conn):
|
||||
self._pool.putconn(conn)
|
||||
|
||||
def _create_extension(self):
|
||||
"""Create pgvector extension if not exists."""
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
||||
conn.commit()
|
||||
finally:
|
||||
self._put_conn(conn)
|
||||
|
||||
def _table_name(self, collection: str) -> str:
|
||||
return f"{self.schema}.praison_vec_{collection}"
|
||||
|
||||
def create_collection(
|
||||
self,
|
||||
name: str,
|
||||
dimension: int,
|
||||
distance: str = "cosine",
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""Create a new collection table."""
|
||||
table = self._table_name(name)
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(f"""
|
||||
CREATE TABLE IF NOT EXISTS {table} (
|
||||
id VARCHAR(255) PRIMARY KEY,
|
||||
content TEXT,
|
||||
content_hash VARCHAR(64),
|
||||
created_at DOUBLE PRECISION,
|
||||
metadata JSONB,
|
||||
embedding vector({dimension})
|
||||
)
|
||||
""")
|
||||
|
||||
# Create index based on distance metric
|
||||
op_map = {"cosine": "vector_cosine_ops", "euclidean": "vector_l2_ops", "dot": "vector_ip_ops"}
|
||||
ops = op_map.get(distance, "vector_cosine_ops")
|
||||
|
||||
cur.execute(f"""
|
||||
CREATE INDEX IF NOT EXISTS idx_{name}_embedding
|
||||
ON {table} USING hnsw (embedding {ops})
|
||||
""")
|
||||
conn.commit()
|
||||
logger.info(f"Created PGVector table: {table}")
|
||||
finally:
|
||||
self._put_conn(conn)
|
||||
|
||||
def delete_collection(self, name: str) -> bool:
|
||||
"""Delete a collection table."""
|
||||
table = self._table_name(name)
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(f"DROP TABLE IF EXISTS {table}")
|
||||
conn.commit()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to delete table {table}: {e}")
|
||||
return False
|
||||
finally:
|
||||
self._put_conn(conn)
|
||||
|
||||
def collection_exists(self, name: str) -> bool:
|
||||
"""Check if a collection exists."""
|
||||
table = self._table_name(name)
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
SELECT EXISTS (
|
||||
SELECT FROM information_schema.tables
|
||||
WHERE table_schema = %s AND table_name = %s
|
||||
)
|
||||
""", (self.schema, f"praison_vec_{name}"))
|
||||
return cur.fetchone()[0]
|
||||
finally:
|
||||
self._put_conn(conn)
|
||||
|
||||
def list_collections(self) -> List[str]:
|
||||
"""List all collections."""
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
SELECT table_name FROM information_schema.tables
|
||||
WHERE table_schema = %s AND table_name LIKE 'praison_vec_%%'
|
||||
""", (self.schema,))
|
||||
return [row[0].replace("praison_vec_", "") for row in cur.fetchall()]
|
||||
finally:
|
||||
self._put_conn(conn)
|
||||
|
||||
def insert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert documents."""
|
||||
table = self._table_name(collection)
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
for doc in documents:
|
||||
if doc.embedding is None:
|
||||
raise ValueError(f"Document {doc.id} has no embedding")
|
||||
cur.execute(f"""
|
||||
INSERT INTO {table} (id, content, content_hash, created_at, metadata, embedding)
|
||||
VALUES (%s, %s, %s, %s, %s, %s)
|
||||
""", (
|
||||
doc.id, doc.content, doc.content_hash, doc.created_at,
|
||||
json.dumps(doc.metadata) if doc.metadata else None,
|
||||
doc.embedding,
|
||||
))
|
||||
conn.commit()
|
||||
return [doc.id for doc in documents]
|
||||
finally:
|
||||
self._put_conn(conn)
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert or update documents."""
|
||||
table = self._table_name(collection)
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
for doc in documents:
|
||||
if doc.embedding is None:
|
||||
raise ValueError(f"Document {doc.id} has no embedding")
|
||||
cur.execute(f"""
|
||||
INSERT INTO {table} (id, content, content_hash, created_at, metadata, embedding)
|
||||
VALUES (%s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
content = EXCLUDED.content,
|
||||
content_hash = EXCLUDED.content_hash,
|
||||
created_at = EXCLUDED.created_at,
|
||||
metadata = EXCLUDED.metadata,
|
||||
embedding = EXCLUDED.embedding
|
||||
""", (
|
||||
doc.id, doc.content, doc.content_hash, doc.created_at,
|
||||
json.dumps(doc.metadata) if doc.metadata else None,
|
||||
doc.embedding,
|
||||
))
|
||||
conn.commit()
|
||||
return [doc.id for doc in documents]
|
||||
finally:
|
||||
self._put_conn(conn)
|
||||
|
||||
def search(
|
||||
self,
|
||||
collection: str,
|
||||
query_embedding: List[float],
|
||||
limit: int = 5,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
score_threshold: Optional[float] = None
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Search for similar documents."""
|
||||
table = self._table_name(collection)
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
with conn.cursor(cursor_factory=self._RealDictCursor) as cur:
|
||||
# Use cosine distance operator <=>
|
||||
query = f"""
|
||||
SELECT id, content, content_hash, created_at, metadata,
|
||||
1 - (embedding <=> %s::vector) as score
|
||||
FROM {table}
|
||||
"""
|
||||
params = [query_embedding]
|
||||
|
||||
if score_threshold:
|
||||
query += " WHERE 1 - (embedding <=> %s::vector) >= %s"
|
||||
params.extend([query_embedding, score_threshold])
|
||||
|
||||
query += " ORDER BY embedding <=> %s::vector LIMIT %s"
|
||||
params.extend([query_embedding, limit])
|
||||
|
||||
cur.execute(query, params)
|
||||
|
||||
documents = []
|
||||
for row in cur.fetchall():
|
||||
doc = KnowledgeDocument(
|
||||
id=row["id"],
|
||||
content=row["content"],
|
||||
embedding=None,
|
||||
metadata=row["metadata"] or {},
|
||||
content_hash=row["content_hash"],
|
||||
created_at=row["created_at"],
|
||||
)
|
||||
documents.append(doc)
|
||||
return documents
|
||||
finally:
|
||||
self._put_conn(conn)
|
||||
|
||||
def get(
|
||||
self,
|
||||
collection: str,
|
||||
ids: List[str]
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Get documents by IDs."""
|
||||
table = self._table_name(collection)
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
with conn.cursor(cursor_factory=self._RealDictCursor) as cur:
|
||||
placeholders = ",".join(["%s"] * len(ids))
|
||||
cur.execute(f"""
|
||||
SELECT id, content, content_hash, created_at, metadata, embedding::float[]
|
||||
FROM {table} WHERE id IN ({placeholders})
|
||||
""", ids)
|
||||
|
||||
documents = []
|
||||
for row in cur.fetchall():
|
||||
doc = KnowledgeDocument(
|
||||
id=row["id"],
|
||||
content=row["content"],
|
||||
embedding=row["embedding"],
|
||||
metadata=row["metadata"] or {},
|
||||
content_hash=row["content_hash"],
|
||||
created_at=row["created_at"],
|
||||
)
|
||||
documents.append(doc)
|
||||
return documents
|
||||
finally:
|
||||
self._put_conn(conn)
|
||||
|
||||
def delete(
|
||||
self,
|
||||
collection: str,
|
||||
ids: Optional[List[str]] = None,
|
||||
filters: Optional[Dict[str, Any]] = None
|
||||
) -> int:
|
||||
"""Delete documents."""
|
||||
table = self._table_name(collection)
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
if ids:
|
||||
placeholders = ",".join(["%s"] * len(ids))
|
||||
cur.execute(f"DELETE FROM {table} WHERE id IN ({placeholders})", ids)
|
||||
deleted = cur.rowcount
|
||||
else:
|
||||
deleted = 0
|
||||
conn.commit()
|
||||
return deleted
|
||||
finally:
|
||||
self._put_conn(conn)
|
||||
|
||||
def count(self, collection: str) -> int:
|
||||
"""Count documents."""
|
||||
table = self._table_name(collection)
|
||||
conn = self._get_conn()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(f"SELECT COUNT(*) FROM {table}")
|
||||
return cur.fetchone()[0]
|
||||
finally:
|
||||
self._put_conn(conn)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the store."""
|
||||
if self._pool:
|
||||
self._pool.closeall()
|
||||
@@ -0,0 +1,253 @@
|
||||
"""
|
||||
Pinecone implementation of KnowledgeStore.
|
||||
|
||||
Requires: pinecone-client
|
||||
Install: pip install pinecone-client
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import KnowledgeStore, KnowledgeDocument
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PineconeKnowledgeStore(KnowledgeStore):
|
||||
"""
|
||||
Pinecone-based knowledge store for vector search.
|
||||
|
||||
Cloud-only managed vector database.
|
||||
|
||||
Example:
|
||||
store = PineconeKnowledgeStore(
|
||||
api_key="your-api-key",
|
||||
index_name="praisonai"
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_key: Optional[str] = None,
|
||||
index_name: Optional[str] = None,
|
||||
environment: Optional[str] = None,
|
||||
namespace: str = "",
|
||||
):
|
||||
try:
|
||||
from pinecone import Pinecone
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"pinecone-client is required for Pinecone support. "
|
||||
"Install with: pip install pinecone-client"
|
||||
)
|
||||
|
||||
api_key = api_key or os.getenv("PINECONE_API_KEY")
|
||||
if not api_key:
|
||||
raise ValueError("Pinecone API key is required")
|
||||
|
||||
self._pc = Pinecone(api_key=api_key)
|
||||
self._index_name = index_name or os.getenv("PINECONE_INDEX", "praisonai")
|
||||
self._namespace = namespace
|
||||
self._index = None
|
||||
|
||||
# Try to connect to existing index
|
||||
try:
|
||||
self._index = self._pc.Index(self._index_name)
|
||||
logger.info(f"Connected to Pinecone index: {self._index_name}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Index {self._index_name} not found: {e}")
|
||||
|
||||
def create_collection(
|
||||
self,
|
||||
name: str,
|
||||
dimension: int,
|
||||
distance: str = "cosine",
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""Create a new index (Pinecone calls them indexes, not collections)."""
|
||||
from pinecone import ServerlessSpec
|
||||
|
||||
metric_map = {"cosine": "cosine", "euclidean": "euclidean", "dot": "dotproduct"}
|
||||
|
||||
self._pc.create_index(
|
||||
name=name,
|
||||
dimension=dimension,
|
||||
metric=metric_map.get(distance, "cosine"),
|
||||
spec=ServerlessSpec(cloud="aws", region="us-east-1")
|
||||
)
|
||||
self._index_name = name
|
||||
self._index = self._pc.Index(name)
|
||||
logger.info(f"Created Pinecone index: {name}")
|
||||
|
||||
def delete_collection(self, name: str) -> bool:
|
||||
"""Delete an index."""
|
||||
try:
|
||||
self._pc.delete_index(name)
|
||||
if name == self._index_name:
|
||||
self._index = None
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to delete index {name}: {e}")
|
||||
return False
|
||||
|
||||
def collection_exists(self, name: str) -> bool:
|
||||
"""Check if an index exists."""
|
||||
try:
|
||||
indexes = self._pc.list_indexes()
|
||||
return any(idx.name == name for idx in indexes)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def list_collections(self) -> List[str]:
|
||||
"""List all indexes."""
|
||||
indexes = self._pc.list_indexes()
|
||||
return [idx.name for idx in indexes]
|
||||
|
||||
def insert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert documents."""
|
||||
if collection != self._index_name:
|
||||
self._index = self._pc.Index(collection)
|
||||
self._index_name = collection
|
||||
|
||||
vectors = []
|
||||
ids = []
|
||||
|
||||
for doc in documents:
|
||||
if doc.embedding is None:
|
||||
raise ValueError(f"Document {doc.id} has no embedding")
|
||||
|
||||
vectors.append({
|
||||
"id": doc.id,
|
||||
"values": doc.embedding,
|
||||
"metadata": {
|
||||
"content": doc.content,
|
||||
"content_hash": doc.content_hash or "",
|
||||
"created_at": doc.created_at,
|
||||
**(doc.metadata or {})
|
||||
}
|
||||
})
|
||||
ids.append(doc.id)
|
||||
|
||||
self._index.upsert(vectors=vectors, namespace=self._namespace)
|
||||
return ids
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert or update documents."""
|
||||
return self.insert(collection, documents)
|
||||
|
||||
def search(
|
||||
self,
|
||||
collection: str,
|
||||
query_embedding: List[float],
|
||||
limit: int = 5,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
score_threshold: Optional[float] = None
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Search for similar documents."""
|
||||
if collection != self._index_name:
|
||||
self._index = self._pc.Index(collection)
|
||||
self._index_name = collection
|
||||
|
||||
results = self._index.query(
|
||||
vector=query_embedding,
|
||||
top_k=limit,
|
||||
filter=filters,
|
||||
include_metadata=True,
|
||||
namespace=self._namespace,
|
||||
)
|
||||
|
||||
documents = []
|
||||
for match in results.matches:
|
||||
if score_threshold and match.score < score_threshold:
|
||||
continue
|
||||
|
||||
metadata = match.metadata or {}
|
||||
content = metadata.pop("content", "")
|
||||
content_hash = metadata.pop("content_hash", None)
|
||||
created_at = metadata.pop("created_at", 0)
|
||||
|
||||
doc = KnowledgeDocument(
|
||||
id=match.id,
|
||||
content=content,
|
||||
embedding=None,
|
||||
metadata=metadata,
|
||||
content_hash=content_hash,
|
||||
created_at=created_at,
|
||||
)
|
||||
documents.append(doc)
|
||||
|
||||
return documents
|
||||
|
||||
def get(
|
||||
self,
|
||||
collection: str,
|
||||
ids: List[str]
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Get documents by IDs."""
|
||||
if collection != self._index_name:
|
||||
self._index = self._pc.Index(collection)
|
||||
self._index_name = collection
|
||||
|
||||
results = self._index.fetch(ids=ids, namespace=self._namespace)
|
||||
|
||||
documents = []
|
||||
for doc_id, vector in results.vectors.items():
|
||||
metadata = vector.metadata or {}
|
||||
content = metadata.pop("content", "")
|
||||
content_hash = metadata.pop("content_hash", None)
|
||||
created_at = metadata.pop("created_at", 0)
|
||||
|
||||
doc = KnowledgeDocument(
|
||||
id=doc_id,
|
||||
content=content,
|
||||
embedding=list(vector.values),
|
||||
metadata=metadata,
|
||||
content_hash=content_hash,
|
||||
created_at=created_at,
|
||||
)
|
||||
documents.append(doc)
|
||||
|
||||
return documents
|
||||
|
||||
def delete(
|
||||
self,
|
||||
collection: str,
|
||||
ids: Optional[List[str]] = None,
|
||||
filters: Optional[Dict[str, Any]] = None
|
||||
) -> int:
|
||||
"""Delete documents."""
|
||||
if collection != self._index_name:
|
||||
self._index = self._pc.Index(collection)
|
||||
self._index_name = collection
|
||||
|
||||
if ids:
|
||||
self._index.delete(ids=ids, namespace=self._namespace)
|
||||
return len(ids)
|
||||
elif filters:
|
||||
self._index.delete(filter=filters, namespace=self._namespace)
|
||||
return -1
|
||||
return 0
|
||||
|
||||
def count(self, collection: str) -> int:
|
||||
"""Count documents."""
|
||||
if collection != self._index_name:
|
||||
self._index = self._pc.Index(collection)
|
||||
self._index_name = collection
|
||||
|
||||
stats = self._index.describe_index_stats()
|
||||
if self._namespace:
|
||||
return stats.namespaces.get(self._namespace, {}).get("vector_count", 0)
|
||||
return stats.total_vector_count
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the store."""
|
||||
self._index = None
|
||||
@@ -0,0 +1,301 @@
|
||||
"""
|
||||
Qdrant implementation of KnowledgeStore.
|
||||
|
||||
Requires: qdrant-client
|
||||
Install: pip install qdrant-client
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import KnowledgeStore, KnowledgeDocument
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class QdrantKnowledgeStore(KnowledgeStore):
|
||||
"""
|
||||
Qdrant-based knowledge store for vector search.
|
||||
|
||||
Example:
|
||||
store = QdrantKnowledgeStore(
|
||||
url="http://localhost:6333"
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: Optional[str] = None,
|
||||
host: str = "localhost",
|
||||
port: int = 6333,
|
||||
grpc_port: int = 6334,
|
||||
api_key: Optional[str] = None,
|
||||
prefer_grpc: bool = False,
|
||||
timeout: int = 30,
|
||||
):
|
||||
"""
|
||||
Initialize Qdrant knowledge store.
|
||||
|
||||
Args:
|
||||
url: Full URL (overrides host/port)
|
||||
host: Qdrant host
|
||||
port: Qdrant HTTP port
|
||||
grpc_port: Qdrant gRPC port
|
||||
api_key: API key for Qdrant Cloud
|
||||
prefer_grpc: Use gRPC instead of HTTP
|
||||
timeout: Request timeout in seconds
|
||||
"""
|
||||
try:
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.models import (
|
||||
Distance, VectorParams, PointStruct,
|
||||
Filter, FieldCondition, MatchValue
|
||||
)
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"qdrant-client is required for Qdrant support. "
|
||||
"Install with: pip install qdrant-client"
|
||||
)
|
||||
|
||||
self._QdrantClient = QdrantClient
|
||||
self._Distance = Distance
|
||||
self._VectorParams = VectorParams
|
||||
self._PointStruct = PointStruct
|
||||
self._Filter = Filter
|
||||
self._FieldCondition = FieldCondition
|
||||
self._MatchValue = MatchValue
|
||||
|
||||
if url:
|
||||
self._client = QdrantClient(url=url, api_key=api_key, timeout=timeout)
|
||||
else:
|
||||
self._client = QdrantClient(
|
||||
host=host,
|
||||
port=port,
|
||||
grpc_port=grpc_port,
|
||||
api_key=api_key,
|
||||
prefer_grpc=prefer_grpc,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
logger.info(f"Connected to Qdrant at {url or f'{host}:{port}'}")
|
||||
|
||||
def _get_distance(self, distance: str) -> Any:
|
||||
"""Convert distance string to Qdrant Distance enum."""
|
||||
distance_map = {
|
||||
"cosine": self._Distance.COSINE,
|
||||
"euclidean": self._Distance.EUCLID,
|
||||
"dot": self._Distance.DOT,
|
||||
"manhattan": self._Distance.MANHATTAN,
|
||||
}
|
||||
return distance_map.get(distance.lower(), self._Distance.COSINE)
|
||||
|
||||
def create_collection(
|
||||
self,
|
||||
name: str,
|
||||
dimension: int,
|
||||
distance: str = "cosine",
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""Create a new collection."""
|
||||
self._client.create_collection(
|
||||
collection_name=name,
|
||||
vectors_config=self._VectorParams(
|
||||
size=dimension,
|
||||
distance=self._get_distance(distance)
|
||||
)
|
||||
)
|
||||
logger.info(f"Created Qdrant collection: {name} (dim={dimension}, distance={distance})")
|
||||
|
||||
def delete_collection(self, name: str) -> bool:
|
||||
"""Delete a collection."""
|
||||
try:
|
||||
self._client.delete_collection(collection_name=name)
|
||||
logger.info(f"Deleted Qdrant collection: {name}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to delete collection {name}: {e}")
|
||||
return False
|
||||
|
||||
def collection_exists(self, name: str) -> bool:
|
||||
"""Check if a collection exists."""
|
||||
try:
|
||||
collections = self._client.get_collections()
|
||||
return any(c.name == name for c in collections.collections)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def list_collections(self) -> List[str]:
|
||||
"""List all collections."""
|
||||
collections = self._client.get_collections()
|
||||
return [c.name for c in collections.collections]
|
||||
|
||||
def insert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert documents into a collection."""
|
||||
points = []
|
||||
ids = []
|
||||
|
||||
for i, doc in enumerate(documents):
|
||||
if doc.embedding is None:
|
||||
raise ValueError(f"Document {doc.id} has no embedding")
|
||||
|
||||
point = self._PointStruct(
|
||||
id=i, # Qdrant prefers numeric IDs for performance
|
||||
vector=doc.embedding,
|
||||
payload={
|
||||
"doc_id": doc.id,
|
||||
"content": doc.content,
|
||||
"content_hash": doc.content_hash,
|
||||
"created_at": doc.created_at,
|
||||
**(doc.metadata or {})
|
||||
}
|
||||
)
|
||||
points.append(point)
|
||||
ids.append(doc.id)
|
||||
|
||||
self._client.upsert(collection_name=collection, points=points)
|
||||
logger.debug(f"Inserted {len(documents)} documents into {collection}")
|
||||
return ids
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert or update documents."""
|
||||
return self.insert(collection, documents)
|
||||
|
||||
def search(
|
||||
self,
|
||||
collection: str,
|
||||
query_embedding: List[float],
|
||||
limit: int = 5,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
score_threshold: Optional[float] = None
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Search for similar documents."""
|
||||
search_filter = None
|
||||
if filters:
|
||||
conditions = [
|
||||
self._FieldCondition(
|
||||
key=k,
|
||||
match=self._MatchValue(value=v)
|
||||
)
|
||||
for k, v in filters.items()
|
||||
]
|
||||
search_filter = self._Filter(must=conditions)
|
||||
|
||||
results = self._client.query_points(
|
||||
collection_name=collection,
|
||||
query=query_embedding,
|
||||
limit=limit,
|
||||
query_filter=search_filter,
|
||||
score_threshold=score_threshold,
|
||||
)
|
||||
|
||||
documents = []
|
||||
for point in results.points:
|
||||
payload = point.payload or {}
|
||||
doc = KnowledgeDocument(
|
||||
id=payload.get("doc_id", str(point.id)),
|
||||
content=payload.get("content", ""),
|
||||
embedding=None, # Don't return embeddings by default
|
||||
metadata={
|
||||
k: v for k, v in payload.items()
|
||||
if k not in ("doc_id", "content", "content_hash", "created_at")
|
||||
},
|
||||
content_hash=payload.get("content_hash"),
|
||||
created_at=payload.get("created_at", 0),
|
||||
)
|
||||
documents.append(doc)
|
||||
|
||||
return documents
|
||||
|
||||
def get(
|
||||
self,
|
||||
collection: str,
|
||||
ids: List[str]
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Get documents by IDs."""
|
||||
# Qdrant uses numeric IDs internally, so we search by doc_id in payload
|
||||
documents = []
|
||||
for doc_id in ids:
|
||||
results = self._client.scroll(
|
||||
collection_name=collection,
|
||||
scroll_filter=self._Filter(
|
||||
must=[
|
||||
self._FieldCondition(
|
||||
key="doc_id",
|
||||
match=self._MatchValue(value=doc_id)
|
||||
)
|
||||
]
|
||||
),
|
||||
limit=1,
|
||||
)
|
||||
for point in results[0]:
|
||||
payload = point.payload or {}
|
||||
doc = KnowledgeDocument(
|
||||
id=payload.get("doc_id", str(point.id)),
|
||||
content=payload.get("content", ""),
|
||||
embedding=list(point.vector) if point.vector else None,
|
||||
metadata={
|
||||
k: v for k, v in payload.items()
|
||||
if k not in ("doc_id", "content", "content_hash", "created_at")
|
||||
},
|
||||
content_hash=payload.get("content_hash"),
|
||||
created_at=payload.get("created_at", 0),
|
||||
)
|
||||
documents.append(doc)
|
||||
|
||||
return documents
|
||||
|
||||
def delete(
|
||||
self,
|
||||
collection: str,
|
||||
ids: Optional[List[str]] = None,
|
||||
filters: Optional[Dict[str, Any]] = None
|
||||
) -> int:
|
||||
"""Delete documents by IDs or filters."""
|
||||
if ids:
|
||||
# Delete by doc_id in payload
|
||||
for doc_id in ids:
|
||||
self._client.delete(
|
||||
collection_name=collection,
|
||||
points_selector=self._Filter(
|
||||
must=[
|
||||
self._FieldCondition(
|
||||
key="doc_id",
|
||||
match=self._MatchValue(value=doc_id)
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
return len(ids)
|
||||
elif filters:
|
||||
conditions = [
|
||||
self._FieldCondition(
|
||||
key=k,
|
||||
match=self._MatchValue(value=v)
|
||||
)
|
||||
for k, v in filters.items()
|
||||
]
|
||||
self._client.delete(
|
||||
collection_name=collection,
|
||||
points_selector=self._Filter(must=conditions)
|
||||
)
|
||||
return -1 # Unknown count
|
||||
return 0
|
||||
|
||||
def count(self, collection: str) -> int:
|
||||
"""Count documents in a collection."""
|
||||
info = self._client.get_collection(collection_name=collection)
|
||||
return info.points_count or 0
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the store."""
|
||||
if self._client:
|
||||
self._client.close()
|
||||
self._client = None
|
||||
@@ -0,0 +1,291 @@
|
||||
"""
|
||||
Redis Vector implementation of KnowledgeStore.
|
||||
|
||||
Requires: redis
|
||||
Install: pip install redis
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import KnowledgeStore, KnowledgeDocument
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RedisVectorKnowledgeStore(KnowledgeStore):
|
||||
"""
|
||||
Redis-based knowledge store using RediSearch for vector search.
|
||||
|
||||
Requires Redis Stack or Redis with RediSearch module.
|
||||
|
||||
Example:
|
||||
store = RedisVectorKnowledgeStore(
|
||||
url="redis://localhost:6379"
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: Optional[str] = None,
|
||||
host: str = "localhost",
|
||||
port: int = 6379,
|
||||
password: Optional[str] = None,
|
||||
prefix: str = "praison:vec:",
|
||||
):
|
||||
try:
|
||||
import redis
|
||||
from redis.commands.search.field import VectorField, TextField, NumericField
|
||||
from redis.commands.search.indexDefinition import IndexDefinition, IndexType
|
||||
from redis.commands.search.query import Query
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"redis is required for Redis Vector support. "
|
||||
"Install with: pip install redis"
|
||||
)
|
||||
|
||||
self._redis = redis
|
||||
self._VectorField = VectorField
|
||||
self._TextField = TextField
|
||||
self._NumericField = NumericField
|
||||
self._IndexDefinition = IndexDefinition
|
||||
self._IndexType = IndexType
|
||||
self._Query = Query
|
||||
|
||||
self.prefix = prefix
|
||||
|
||||
if url:
|
||||
self._client = redis.from_url(url, decode_responses=False)
|
||||
else:
|
||||
self._client = redis.Redis(
|
||||
host=host, port=port, password=password, decode_responses=False
|
||||
)
|
||||
|
||||
self._client.ping()
|
||||
logger.info(f"Connected to Redis at {url or f'{host}:{port}'}")
|
||||
|
||||
def _index_name(self, collection: str) -> str:
|
||||
return f"{self.prefix}{collection}:idx"
|
||||
|
||||
def _doc_key(self, collection: str, doc_id: str) -> str:
|
||||
return f"{self.prefix}{collection}:{doc_id}"
|
||||
|
||||
def create_collection(
|
||||
self,
|
||||
name: str,
|
||||
dimension: int,
|
||||
distance: str = "cosine",
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""Create a new index."""
|
||||
index_name = self._index_name(name)
|
||||
|
||||
distance_map = {"cosine": "COSINE", "euclidean": "L2", "dot": "IP"}
|
||||
|
||||
schema = (
|
||||
self._TextField("content"),
|
||||
self._TextField("content_hash"),
|
||||
self._NumericField("created_at"),
|
||||
self._VectorField(
|
||||
"embedding",
|
||||
"HNSW",
|
||||
{
|
||||
"TYPE": "FLOAT32",
|
||||
"DIM": dimension,
|
||||
"DISTANCE_METRIC": distance_map.get(distance, "COSINE"),
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
definition = self._IndexDefinition(
|
||||
prefix=[f"{self.prefix}{name}:"],
|
||||
index_type=self._IndexType.HASH,
|
||||
)
|
||||
|
||||
try:
|
||||
self._client.ft(index_name).create_index(schema, definition=definition)
|
||||
logger.info(f"Created Redis index: {index_name}")
|
||||
except Exception as e:
|
||||
if "Index already exists" not in str(e):
|
||||
raise
|
||||
|
||||
def delete_collection(self, name: str) -> bool:
|
||||
"""Delete an index and all documents."""
|
||||
index_name = self._index_name(name)
|
||||
try:
|
||||
self._client.ft(index_name).dropindex(delete_documents=True)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to delete index {index_name}: {e}")
|
||||
return False
|
||||
|
||||
def collection_exists(self, name: str) -> bool:
|
||||
"""Check if an index exists."""
|
||||
index_name = self._index_name(name)
|
||||
try:
|
||||
self._client.ft(index_name).info()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def list_collections(self) -> List[str]:
|
||||
"""List all collections."""
|
||||
try:
|
||||
indexes = self._client.execute_command("FT._LIST")
|
||||
prefix = f"{self.prefix}"
|
||||
return [
|
||||
idx.decode().replace(prefix, "").replace(":idx", "")
|
||||
for idx in indexes
|
||||
if idx.decode().startswith(prefix)
|
||||
]
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def insert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert documents."""
|
||||
import numpy as np
|
||||
|
||||
ids = []
|
||||
pipe = self._client.pipeline()
|
||||
|
||||
for doc in documents:
|
||||
if doc.embedding is None:
|
||||
raise ValueError(f"Document {doc.id} has no embedding")
|
||||
|
||||
key = self._doc_key(collection, doc.id)
|
||||
embedding_bytes = np.array(doc.embedding, dtype=np.float32).tobytes()
|
||||
|
||||
pipe.hset(key, mapping={
|
||||
"content": doc.content,
|
||||
"content_hash": doc.content_hash or "",
|
||||
"created_at": doc.created_at,
|
||||
"embedding": embedding_bytes,
|
||||
})
|
||||
ids.append(doc.id)
|
||||
|
||||
pipe.execute()
|
||||
return ids
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert or update documents."""
|
||||
return self.insert(collection, documents)
|
||||
|
||||
def search(
|
||||
self,
|
||||
collection: str,
|
||||
query_embedding: List[float],
|
||||
limit: int = 5,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
score_threshold: Optional[float] = None
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Search for similar documents."""
|
||||
import numpy as np
|
||||
|
||||
index_name = self._index_name(collection)
|
||||
query_bytes = np.array(query_embedding, dtype=np.float32).tobytes()
|
||||
|
||||
query_str = f"*=>[KNN {limit} @embedding $vec AS score]"
|
||||
|
||||
query = (
|
||||
self._Query(query_str)
|
||||
.return_fields("content", "content_hash", "created_at", "score")
|
||||
.sort_by("score")
|
||||
.dialect(2)
|
||||
)
|
||||
|
||||
results = self._client.ft(index_name).search(
|
||||
query, query_params={"vec": query_bytes}
|
||||
)
|
||||
|
||||
documents = []
|
||||
for doc in results.docs:
|
||||
score = float(doc.score) if hasattr(doc, "score") else 0
|
||||
# Redis returns distance, convert to similarity for cosine
|
||||
similarity = 1 - score
|
||||
|
||||
if score_threshold and similarity < score_threshold:
|
||||
continue
|
||||
|
||||
doc_id = doc.id.decode() if isinstance(doc.id, bytes) else doc.id
|
||||
doc_id = doc_id.split(":")[-1] # Remove prefix
|
||||
|
||||
knowledge_doc = KnowledgeDocument(
|
||||
id=doc_id,
|
||||
content=doc.content.decode() if isinstance(doc.content, bytes) else doc.content,
|
||||
embedding=None,
|
||||
metadata={},
|
||||
content_hash=doc.content_hash.decode() if isinstance(doc.content_hash, bytes) else doc.content_hash,
|
||||
created_at=float(doc.created_at) if doc.created_at else 0,
|
||||
)
|
||||
documents.append(knowledge_doc)
|
||||
|
||||
return documents
|
||||
|
||||
def get(
|
||||
self,
|
||||
collection: str,
|
||||
ids: List[str]
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Get documents by IDs."""
|
||||
import numpy as np
|
||||
|
||||
documents = []
|
||||
for doc_id in ids:
|
||||
key = self._doc_key(collection, doc_id)
|
||||
data = self._client.hgetall(key)
|
||||
|
||||
if data:
|
||||
embedding = None
|
||||
if b"embedding" in data:
|
||||
embedding = np.frombuffer(data[b"embedding"], dtype=np.float32).tolist()
|
||||
|
||||
doc = KnowledgeDocument(
|
||||
id=doc_id,
|
||||
content=data.get(b"content", b"").decode(),
|
||||
embedding=embedding,
|
||||
metadata={},
|
||||
content_hash=data.get(b"content_hash", b"").decode(),
|
||||
created_at=float(data.get(b"created_at", 0)),
|
||||
)
|
||||
documents.append(doc)
|
||||
|
||||
return documents
|
||||
|
||||
def delete(
|
||||
self,
|
||||
collection: str,
|
||||
ids: Optional[List[str]] = None,
|
||||
filters: Optional[Dict[str, Any]] = None
|
||||
) -> int:
|
||||
"""Delete documents."""
|
||||
if ids:
|
||||
pipe = self._client.pipeline()
|
||||
for doc_id in ids:
|
||||
key = self._doc_key(collection, doc_id)
|
||||
pipe.delete(key)
|
||||
pipe.execute()
|
||||
return len(ids)
|
||||
return 0
|
||||
|
||||
def count(self, collection: str) -> int:
|
||||
"""Count documents."""
|
||||
index_name = self._index_name(collection)
|
||||
try:
|
||||
info = self._client.ft(index_name).info()
|
||||
return int(info.get("num_docs", 0))
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the store."""
|
||||
if self._client:
|
||||
self._client.close()
|
||||
@@ -0,0 +1,299 @@
|
||||
"""
|
||||
SingleStore Vector implementation of KnowledgeStore.
|
||||
|
||||
Requires: singlestoredb
|
||||
Install: pip install singlestoredb
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import KnowledgeStore, KnowledgeDocument
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SingleStoreVectorKnowledgeStore(KnowledgeStore):
|
||||
"""
|
||||
SingleStore vector store for knowledge/RAG.
|
||||
|
||||
Uses SingleStore's native vector capabilities.
|
||||
|
||||
Example:
|
||||
store = SingleStoreVectorKnowledgeStore(
|
||||
url="singlestoredb://user:pass@host:3306/db"
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: Optional[str] = None,
|
||||
host: str = "localhost",
|
||||
port: int = 3306,
|
||||
database: str = "praisonai",
|
||||
user: str = "root",
|
||||
password: str = "",
|
||||
table_prefix: str = "praisonai_",
|
||||
embedding_dim: int = 1536,
|
||||
):
|
||||
"""
|
||||
Initialize SingleStore Vector store.
|
||||
|
||||
Args:
|
||||
url: SingleStore connection URL
|
||||
host: Database host
|
||||
port: Database port
|
||||
database: Database name
|
||||
user: Database user
|
||||
password: Database password
|
||||
table_prefix: Prefix for table names
|
||||
embedding_dim: Embedding dimension
|
||||
"""
|
||||
self.url = url
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.database = database
|
||||
self.user = user
|
||||
self.password = password
|
||||
self.table_prefix = table_prefix
|
||||
self.embedding_dim = embedding_dim
|
||||
|
||||
self._conn = None
|
||||
self._initialized = False
|
||||
|
||||
def _init_client(self):
|
||||
"""Initialize SingleStore client lazily."""
|
||||
if self._initialized:
|
||||
return
|
||||
|
||||
try:
|
||||
import singlestoredb as s2
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"singlestoredb is required for SingleStore Vector support. "
|
||||
"Install with: pip install singlestoredb"
|
||||
)
|
||||
|
||||
if self.url:
|
||||
self._conn = s2.connect(self.url)
|
||||
else:
|
||||
self._conn = s2.connect(
|
||||
host=self.host,
|
||||
port=self.port,
|
||||
database=self.database,
|
||||
user=self.user,
|
||||
password=self.password
|
||||
)
|
||||
|
||||
self._initialized = True
|
||||
|
||||
def create_collection(
|
||||
self,
|
||||
name: str,
|
||||
dimension: int = 1536,
|
||||
distance: str = "cosine",
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""Create a vector table."""
|
||||
self._init_client()
|
||||
|
||||
table_name = f"{self.table_prefix}{name}"
|
||||
|
||||
with self._conn.cursor() as cur:
|
||||
cur.execute(f"""
|
||||
CREATE TABLE IF NOT EXISTS {table_name} (
|
||||
id VARCHAR(255) PRIMARY KEY,
|
||||
content TEXT,
|
||||
embedding VECTOR({dimension}),
|
||||
metadata JSON,
|
||||
content_hash VARCHAR(64),
|
||||
created_at DOUBLE
|
||||
)
|
||||
""")
|
||||
|
||||
def delete_collection(self, name: str) -> bool:
|
||||
"""Delete a vector table."""
|
||||
self._init_client()
|
||||
|
||||
table_name = f"{self.table_prefix}{name}"
|
||||
try:
|
||||
with self._conn.cursor() as cur:
|
||||
cur.execute(f"DROP TABLE IF EXISTS {table_name}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete collection {name}: {e}")
|
||||
return False
|
||||
|
||||
def collection_exists(self, name: str) -> bool:
|
||||
"""Check if table exists."""
|
||||
self._init_client()
|
||||
|
||||
table_name = f"{self.table_prefix}{name}"
|
||||
with self._conn.cursor() as cur:
|
||||
cur.execute(f"""
|
||||
SELECT COUNT(*) FROM information_schema.tables
|
||||
WHERE table_schema = %s AND table_name = %s
|
||||
""", (self.database, table_name))
|
||||
return cur.fetchone()[0] > 0
|
||||
|
||||
def list_collections(self) -> List[str]:
|
||||
"""List all vector tables."""
|
||||
self._init_client()
|
||||
|
||||
with self._conn.cursor() as cur:
|
||||
cur.execute(f"""
|
||||
SELECT table_name FROM information_schema.tables
|
||||
WHERE table_schema = %s AND table_name LIKE %s
|
||||
""", (self.database, f"{self.table_prefix}%"))
|
||||
return [row[0][len(self.table_prefix):] for row in cur.fetchall()]
|
||||
|
||||
def insert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert documents."""
|
||||
self._init_client()
|
||||
|
||||
table_name = f"{self.table_prefix}{collection}"
|
||||
ids = []
|
||||
|
||||
with self._conn.cursor() as cur:
|
||||
for doc in documents:
|
||||
embedding_str = str(doc.embedding) if doc.embedding else None
|
||||
cur.execute(f"""
|
||||
INSERT INTO {table_name} (id, content, embedding, metadata, content_hash, created_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s)
|
||||
""", (doc.id, doc.content, embedding_str,
|
||||
str(doc.metadata) if doc.metadata else None,
|
||||
doc.content_hash, doc.created_at))
|
||||
ids.append(doc.id)
|
||||
|
||||
return ids
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Upsert documents."""
|
||||
self._init_client()
|
||||
|
||||
table_name = f"{self.table_prefix}{collection}"
|
||||
ids = []
|
||||
|
||||
with self._conn.cursor() as cur:
|
||||
for doc in documents:
|
||||
embedding_str = str(doc.embedding) if doc.embedding else None
|
||||
cur.execute(f"""
|
||||
REPLACE INTO {table_name} (id, content, embedding, metadata, content_hash, created_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s)
|
||||
""", (doc.id, doc.content, embedding_str,
|
||||
str(doc.metadata) if doc.metadata else None,
|
||||
doc.content_hash, doc.created_at))
|
||||
ids.append(doc.id)
|
||||
|
||||
return ids
|
||||
|
||||
def search(
|
||||
self,
|
||||
collection: str,
|
||||
query_embedding: List[float],
|
||||
limit: int = 5,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
score_threshold: Optional[float] = None
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Search for similar documents using vector search."""
|
||||
self._init_client()
|
||||
|
||||
table_name = f"{self.table_prefix}{collection}"
|
||||
embedding_str = str(query_embedding)
|
||||
|
||||
with self._conn.cursor() as cur:
|
||||
cur.execute(f"""
|
||||
SELECT id, content, metadata, content_hash, created_at,
|
||||
DOT_PRODUCT(embedding, %s) as score
|
||||
FROM {table_name}
|
||||
ORDER BY score DESC
|
||||
LIMIT %s
|
||||
""", (embedding_str, limit))
|
||||
|
||||
documents = []
|
||||
for row in cur.fetchall():
|
||||
score = row[5] if len(row) > 5 else 0
|
||||
if score_threshold and score < score_threshold:
|
||||
continue
|
||||
|
||||
documents.append(KnowledgeDocument(
|
||||
id=row[0],
|
||||
content=row[1],
|
||||
metadata=eval(row[2]) if row[2] else None,
|
||||
content_hash=row[3],
|
||||
created_at=row[4]
|
||||
))
|
||||
|
||||
return documents
|
||||
|
||||
def get(
|
||||
self,
|
||||
collection: str,
|
||||
ids: List[str]
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Get documents by IDs."""
|
||||
self._init_client()
|
||||
|
||||
table_name = f"{self.table_prefix}{collection}"
|
||||
placeholders = ','.join(['%s'] * len(ids))
|
||||
|
||||
with self._conn.cursor() as cur:
|
||||
cur.execute(f"""
|
||||
SELECT id, content, metadata, content_hash, created_at
|
||||
FROM {table_name}
|
||||
WHERE id IN ({placeholders})
|
||||
""", ids)
|
||||
|
||||
return [
|
||||
KnowledgeDocument(
|
||||
id=row[0],
|
||||
content=row[1],
|
||||
metadata=eval(row[2]) if row[2] else None,
|
||||
content_hash=row[3],
|
||||
created_at=row[4]
|
||||
)
|
||||
for row in cur.fetchall()
|
||||
]
|
||||
|
||||
def delete(
|
||||
self,
|
||||
collection: str,
|
||||
ids: Optional[List[str]] = None,
|
||||
filters: Optional[Dict[str, Any]] = None
|
||||
) -> int:
|
||||
"""Delete documents."""
|
||||
self._init_client()
|
||||
|
||||
table_name = f"{self.table_prefix}{collection}"
|
||||
|
||||
with self._conn.cursor() as cur:
|
||||
if ids:
|
||||
placeholders = ','.join(['%s'] * len(ids))
|
||||
cur.execute(f"DELETE FROM {table_name} WHERE id IN ({placeholders})", ids)
|
||||
else:
|
||||
cur.execute(f"DELETE FROM {table_name}")
|
||||
return cur.rowcount
|
||||
|
||||
def count(self, collection: str) -> int:
|
||||
"""Count documents."""
|
||||
self._init_client()
|
||||
|
||||
table_name = f"{self.table_prefix}{collection}"
|
||||
with self._conn.cursor() as cur:
|
||||
cur.execute(f"SELECT COUNT(*) FROM {table_name}")
|
||||
return cur.fetchone()[0]
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the connection."""
|
||||
if self._conn:
|
||||
self._conn.close()
|
||||
self._conn = None
|
||||
self._initialized = False
|
||||
@@ -0,0 +1,309 @@
|
||||
"""
|
||||
SurrealDB Vector implementation of KnowledgeStore.
|
||||
|
||||
Requires: surrealdb
|
||||
Install: pip install surrealdb
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import KnowledgeStore, KnowledgeDocument
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SurrealDBVectorKnowledgeStore(KnowledgeStore):
|
||||
"""
|
||||
SurrealDB vector store for knowledge/RAG.
|
||||
|
||||
Uses SurrealDB's vector capabilities.
|
||||
|
||||
Example:
|
||||
store = SurrealDBVectorKnowledgeStore(
|
||||
url="ws://localhost:8000/rpc",
|
||||
namespace="praisonai",
|
||||
database="vectors"
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: str = "ws://localhost:8000/rpc",
|
||||
namespace: str = "praisonai",
|
||||
database: str = "vectors",
|
||||
username: str = "root",
|
||||
password: str = "root",
|
||||
embedding_dim: int = 1536,
|
||||
):
|
||||
"""
|
||||
Initialize SurrealDB Vector store.
|
||||
|
||||
Args:
|
||||
url: SurrealDB WebSocket URL
|
||||
namespace: SurrealDB namespace
|
||||
database: SurrealDB database
|
||||
username: Authentication username
|
||||
password: Authentication password
|
||||
embedding_dim: Embedding dimension
|
||||
"""
|
||||
self.url = url
|
||||
self.namespace = namespace
|
||||
self.database = database
|
||||
self.username = username
|
||||
self.password = password
|
||||
self.embedding_dim = embedding_dim
|
||||
|
||||
self._client = None
|
||||
self._initialized = False
|
||||
|
||||
def _init_client(self):
|
||||
"""Initialize SurrealDB client lazily."""
|
||||
if self._initialized:
|
||||
return
|
||||
|
||||
try:
|
||||
from surrealdb import Surreal
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"surrealdb is required for SurrealDB Vector support. "
|
||||
"Install with: pip install surrealdb"
|
||||
)
|
||||
|
||||
import asyncio
|
||||
|
||||
async def connect():
|
||||
client = Surreal(self.url)
|
||||
await client.connect()
|
||||
await client.signin({"user": self.username, "pass": self.password})
|
||||
await client.use(self.namespace, self.database)
|
||||
return client
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
self._client = loop.run_until_complete(connect())
|
||||
self._initialized = True
|
||||
|
||||
def _run_async(self, coro):
|
||||
"""Run async coroutine synchronously."""
|
||||
import asyncio
|
||||
return asyncio.get_event_loop().run_until_complete(coro)
|
||||
|
||||
def create_collection(
|
||||
self,
|
||||
name: str,
|
||||
dimension: int = 1536,
|
||||
distance: str = "cosine",
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""Create a collection (SurrealDB creates tables on insert)."""
|
||||
self._init_client()
|
||||
# SurrealDB creates tables automatically
|
||||
logger.info(f"Collection '{name}' will be created on first insert")
|
||||
|
||||
def delete_collection(self, name: str) -> bool:
|
||||
"""Delete a collection."""
|
||||
self._init_client()
|
||||
try:
|
||||
self._run_async(self._client.query(f"REMOVE TABLE {name}"))
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete collection {name}: {e}")
|
||||
return False
|
||||
|
||||
def collection_exists(self, name: str) -> bool:
|
||||
"""Check if collection exists."""
|
||||
self._init_client()
|
||||
try:
|
||||
result = self._run_async(self._client.query(f"INFO FOR TABLE {name}"))
|
||||
return bool(result)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def list_collections(self) -> List[str]:
|
||||
"""List all collections."""
|
||||
self._init_client()
|
||||
try:
|
||||
result = self._run_async(self._client.query("INFO FOR DB"))
|
||||
if result and len(result) > 0:
|
||||
tables = result[0].get("result", {}).get("tables", {})
|
||||
return list(tables.keys())
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to list collections: {e}")
|
||||
return []
|
||||
|
||||
def insert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert documents."""
|
||||
self._init_client()
|
||||
|
||||
ids = []
|
||||
for doc in documents:
|
||||
record = {
|
||||
"content": doc.content,
|
||||
"embedding": doc.embedding,
|
||||
"metadata": doc.metadata or {},
|
||||
"content_hash": doc.content_hash,
|
||||
"created_at": doc.created_at,
|
||||
}
|
||||
|
||||
try:
|
||||
result = self._run_async(
|
||||
self._client.create(f"{collection}:{doc.id}", record)
|
||||
)
|
||||
ids.append(doc.id)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to insert document {doc.id}: {e}")
|
||||
|
||||
return ids
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Upsert documents."""
|
||||
self._init_client()
|
||||
|
||||
ids = []
|
||||
for doc in documents:
|
||||
record = {
|
||||
"content": doc.content,
|
||||
"embedding": doc.embedding,
|
||||
"metadata": doc.metadata or {},
|
||||
"content_hash": doc.content_hash,
|
||||
"created_at": doc.created_at,
|
||||
}
|
||||
|
||||
try:
|
||||
self._run_async(
|
||||
self._client.update(f"{collection}:{doc.id}", record)
|
||||
)
|
||||
ids.append(doc.id)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to upsert document {doc.id}: {e}")
|
||||
|
||||
return ids
|
||||
|
||||
def search(
|
||||
self,
|
||||
collection: str,
|
||||
query_embedding: List[float],
|
||||
limit: int = 5,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
score_threshold: Optional[float] = None
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Search for similar documents using vector search."""
|
||||
self._init_client()
|
||||
|
||||
# SurrealDB vector search query
|
||||
embedding_str = str(query_embedding)
|
||||
query = f"""
|
||||
SELECT *, vector::similarity::cosine(embedding, {embedding_str}) AS score
|
||||
FROM {collection}
|
||||
ORDER BY score DESC
|
||||
LIMIT {limit}
|
||||
"""
|
||||
|
||||
try:
|
||||
result = self._run_async(self._client.query(query))
|
||||
|
||||
documents = []
|
||||
if result and len(result) > 0:
|
||||
for row in result[0].get("result", []):
|
||||
score = row.get("score", 0)
|
||||
if score_threshold and score < score_threshold:
|
||||
continue
|
||||
|
||||
doc_id = str(row.get("id", "")).split(":")[-1]
|
||||
documents.append(KnowledgeDocument(
|
||||
id=doc_id,
|
||||
content=row.get("content", ""),
|
||||
embedding=row.get("embedding"),
|
||||
metadata={**(row.get("metadata") or {}), "score": score},
|
||||
content_hash=row.get("content_hash"),
|
||||
created_at=row.get("created_at", 0)
|
||||
))
|
||||
|
||||
return documents
|
||||
except Exception as e:
|
||||
logger.error(f"Vector search failed: {e}")
|
||||
return []
|
||||
|
||||
def get(
|
||||
self,
|
||||
collection: str,
|
||||
ids: List[str]
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Get documents by IDs."""
|
||||
self._init_client()
|
||||
|
||||
documents = []
|
||||
for doc_id in ids:
|
||||
try:
|
||||
result = self._run_async(self._client.select(f"{collection}:{doc_id}"))
|
||||
if result:
|
||||
row = result if isinstance(result, dict) else result[0]
|
||||
documents.append(KnowledgeDocument(
|
||||
id=doc_id,
|
||||
content=row.get("content", ""),
|
||||
embedding=row.get("embedding"),
|
||||
metadata=row.get("metadata"),
|
||||
content_hash=row.get("content_hash"),
|
||||
created_at=row.get("created_at", 0)
|
||||
))
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to get document {doc_id}: {e}")
|
||||
|
||||
return documents
|
||||
|
||||
def delete(
|
||||
self,
|
||||
collection: str,
|
||||
ids: Optional[List[str]] = None,
|
||||
filters: Optional[Dict[str, Any]] = None
|
||||
) -> int:
|
||||
"""Delete documents."""
|
||||
self._init_client()
|
||||
|
||||
count = 0
|
||||
if ids:
|
||||
for doc_id in ids:
|
||||
try:
|
||||
self._run_async(self._client.delete(f"{collection}:{doc_id}"))
|
||||
count += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to delete {doc_id}: {e}")
|
||||
else:
|
||||
try:
|
||||
self._run_async(self._client.query(f"DELETE FROM {collection}"))
|
||||
count = -1 # Unknown count
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete all: {e}")
|
||||
|
||||
return count
|
||||
|
||||
def count(self, collection: str) -> int:
|
||||
"""Count documents."""
|
||||
self._init_client()
|
||||
|
||||
try:
|
||||
result = self._run_async(
|
||||
self._client.query(f"SELECT count() FROM {collection} GROUP ALL")
|
||||
)
|
||||
if result and len(result) > 0:
|
||||
return result[0].get("result", [{}])[0].get("count", 0)
|
||||
return 0
|
||||
except Exception as e:
|
||||
logger.error(f"Count failed: {e}")
|
||||
return 0
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the connection."""
|
||||
if self._client:
|
||||
self._run_async(self._client.close())
|
||||
self._client = None
|
||||
self._initialized = False
|
||||
@@ -0,0 +1,266 @@
|
||||
"""
|
||||
Upstash Vector implementation of KnowledgeStore.
|
||||
|
||||
Requires: upstash-vector
|
||||
Install: pip install upstash-vector
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import KnowledgeStore, KnowledgeDocument
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class UpstashVectorKnowledgeStore(KnowledgeStore):
|
||||
"""
|
||||
Upstash Vector store for knowledge/RAG.
|
||||
|
||||
Uses Upstash's serverless vector database.
|
||||
|
||||
Example:
|
||||
store = UpstashVectorKnowledgeStore(
|
||||
url="https://xxx.upstash.io",
|
||||
token="your-token"
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: Optional[str] = None,
|
||||
token: Optional[str] = None,
|
||||
embedding_dim: int = 1536,
|
||||
):
|
||||
"""
|
||||
Initialize Upstash Vector store.
|
||||
|
||||
Args:
|
||||
url: Upstash Vector REST URL (or UPSTASH_VECTOR_REST_URL env)
|
||||
token: Upstash Vector REST token (or UPSTASH_VECTOR_REST_TOKEN env)
|
||||
embedding_dim: Embedding dimension
|
||||
"""
|
||||
self.url = url or os.getenv("UPSTASH_VECTOR_REST_URL")
|
||||
self.token = token or os.getenv("UPSTASH_VECTOR_REST_TOKEN")
|
||||
self.embedding_dim = embedding_dim
|
||||
|
||||
self._index = None
|
||||
self._initialized = False
|
||||
|
||||
def _init_client(self):
|
||||
"""Initialize Upstash client lazily."""
|
||||
if self._initialized:
|
||||
return
|
||||
|
||||
if not self.url or not self.token:
|
||||
raise ValueError(
|
||||
"Upstash Vector requires url and token. "
|
||||
"Set UPSTASH_VECTOR_REST_URL and UPSTASH_VECTOR_REST_TOKEN env vars."
|
||||
)
|
||||
|
||||
try:
|
||||
from upstash_vector import Index
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"upstash-vector is required for Upstash Vector support. "
|
||||
"Install with: pip install upstash-vector"
|
||||
)
|
||||
|
||||
self._index = Index(url=self.url, token=self.token)
|
||||
self._initialized = True
|
||||
|
||||
def create_collection(
|
||||
self,
|
||||
name: str,
|
||||
dimension: int = 1536,
|
||||
distance: str = "cosine",
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""Create collection (Upstash uses namespaces)."""
|
||||
self._init_client()
|
||||
# Upstash Vector uses a single index with namespaces
|
||||
logger.info(f"Using namespace '{name}' in Upstash Vector index")
|
||||
|
||||
def delete_collection(self, name: str) -> bool:
|
||||
"""Delete all vectors in namespace."""
|
||||
self._init_client()
|
||||
try:
|
||||
self._index.delete_namespace(name)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete namespace {name}: {e}")
|
||||
return False
|
||||
|
||||
def collection_exists(self, name: str) -> bool:
|
||||
"""Check if namespace has vectors."""
|
||||
self._init_client()
|
||||
try:
|
||||
info = self._index.info()
|
||||
namespaces = info.namespaces if hasattr(info, 'namespaces') else {}
|
||||
return name in namespaces
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def list_collections(self) -> List[str]:
|
||||
"""List all namespaces."""
|
||||
self._init_client()
|
||||
try:
|
||||
info = self._index.info()
|
||||
namespaces = info.namespaces if hasattr(info, 'namespaces') else {}
|
||||
return list(namespaces.keys())
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to list namespaces: {e}")
|
||||
return []
|
||||
|
||||
def insert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert documents."""
|
||||
self._init_client()
|
||||
|
||||
vectors = []
|
||||
for doc in documents:
|
||||
if not doc.embedding:
|
||||
logger.warning(f"Document {doc.id} has no embedding, skipping")
|
||||
continue
|
||||
|
||||
vectors.append({
|
||||
"id": doc.id,
|
||||
"vector": doc.embedding,
|
||||
"metadata": {
|
||||
"content": doc.content,
|
||||
**(doc.metadata or {}),
|
||||
"content_hash": doc.content_hash,
|
||||
"created_at": doc.created_at,
|
||||
}
|
||||
})
|
||||
|
||||
if vectors:
|
||||
self._index.upsert(vectors, namespace=collection)
|
||||
|
||||
return [v["id"] for v in vectors]
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Upsert documents (same as insert for Upstash)."""
|
||||
return self.insert(collection, documents)
|
||||
|
||||
def search(
|
||||
self,
|
||||
collection: str,
|
||||
query_embedding: List[float],
|
||||
limit: int = 5,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
score_threshold: Optional[float] = None
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Search for similar documents."""
|
||||
self._init_client()
|
||||
|
||||
try:
|
||||
results = self._index.query(
|
||||
vector=query_embedding,
|
||||
top_k=limit,
|
||||
include_metadata=True,
|
||||
namespace=collection,
|
||||
filter=filters
|
||||
)
|
||||
|
||||
documents = []
|
||||
for result in results:
|
||||
score = result.score if hasattr(result, 'score') else 0
|
||||
if score_threshold and score < score_threshold:
|
||||
continue
|
||||
|
||||
metadata = result.metadata if hasattr(result, 'metadata') else {}
|
||||
content = metadata.pop("content", "") if metadata else ""
|
||||
|
||||
documents.append(KnowledgeDocument(
|
||||
id=result.id,
|
||||
content=content,
|
||||
metadata={**metadata, "score": score},
|
||||
content_hash=metadata.get("content_hash") if metadata else None,
|
||||
created_at=metadata.get("created_at", 0) if metadata else 0
|
||||
))
|
||||
|
||||
return documents
|
||||
except Exception as e:
|
||||
logger.error(f"Search failed: {e}")
|
||||
return []
|
||||
|
||||
def get(
|
||||
self,
|
||||
collection: str,
|
||||
ids: List[str]
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Get documents by IDs."""
|
||||
self._init_client()
|
||||
|
||||
documents = []
|
||||
try:
|
||||
results = self._index.fetch(ids, include_metadata=True, namespace=collection)
|
||||
|
||||
for result in results:
|
||||
if result:
|
||||
metadata = result.metadata if hasattr(result, 'metadata') else {}
|
||||
content = metadata.pop("content", "") if metadata else ""
|
||||
|
||||
documents.append(KnowledgeDocument(
|
||||
id=result.id,
|
||||
content=content,
|
||||
embedding=result.vector if hasattr(result, 'vector') else None,
|
||||
metadata=metadata,
|
||||
content_hash=metadata.get("content_hash") if metadata else None,
|
||||
created_at=metadata.get("created_at", 0) if metadata else 0
|
||||
))
|
||||
except Exception as e:
|
||||
logger.error(f"Fetch failed: {e}")
|
||||
|
||||
return documents
|
||||
|
||||
def delete(
|
||||
self,
|
||||
collection: str,
|
||||
ids: Optional[List[str]] = None,
|
||||
filters: Optional[Dict[str, Any]] = None
|
||||
) -> int:
|
||||
"""Delete documents."""
|
||||
self._init_client()
|
||||
|
||||
try:
|
||||
if ids:
|
||||
self._index.delete(ids, namespace=collection)
|
||||
return len(ids)
|
||||
elif filters:
|
||||
# Upstash doesn't support filter-based delete directly
|
||||
logger.warning("Filter-based delete not supported, use IDs")
|
||||
return 0
|
||||
else:
|
||||
self._index.delete_namespace(collection)
|
||||
return -1 # Unknown count
|
||||
except Exception as e:
|
||||
logger.error(f"Delete failed: {e}")
|
||||
return 0
|
||||
|
||||
def count(self, collection: str) -> int:
|
||||
"""Count documents in namespace."""
|
||||
self._init_client()
|
||||
|
||||
try:
|
||||
info = self._index.info()
|
||||
namespaces = info.namespaces if hasattr(info, 'namespaces') else {}
|
||||
ns_info = namespaces.get(collection, {})
|
||||
return ns_info.get("vector_count", 0) if isinstance(ns_info, dict) else 0
|
||||
except Exception as e:
|
||||
logger.error(f"Count failed: {e}")
|
||||
return 0
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the client."""
|
||||
self._index = None
|
||||
self._initialized = False
|
||||
@@ -0,0 +1,223 @@
|
||||
"""
|
||||
Weaviate implementation of KnowledgeStore.
|
||||
|
||||
Requires: weaviate-client
|
||||
Install: pip install weaviate-client
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import KnowledgeStore, KnowledgeDocument
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WeaviateKnowledgeStore(KnowledgeStore):
|
||||
"""
|
||||
Weaviate-based knowledge store for vector search.
|
||||
|
||||
Example:
|
||||
store = WeaviateKnowledgeStore(
|
||||
url="http://localhost:8080"
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: Optional[str] = None,
|
||||
api_key: Optional[str] = None,
|
||||
additional_headers: Optional[Dict[str, str]] = None,
|
||||
):
|
||||
try:
|
||||
import weaviate
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"weaviate-client is required for Weaviate support. "
|
||||
"Install with: pip install weaviate-client"
|
||||
)
|
||||
|
||||
url = url or os.getenv("WEAVIATE_URL", "http://localhost:8080")
|
||||
api_key = api_key or os.getenv("WEAVIATE_API_KEY")
|
||||
|
||||
auth = weaviate.auth.AuthApiKey(api_key) if api_key else None
|
||||
self._client = weaviate.connect_to_custom(
|
||||
http_host=url.replace("http://", "").replace("https://", "").split(":")[0],
|
||||
http_port=int(url.split(":")[-1]) if ":" in url.split("/")[-1] else 8080,
|
||||
http_secure=url.startswith("https"),
|
||||
auth_credentials=auth,
|
||||
additional_headers=additional_headers,
|
||||
)
|
||||
logger.info(f"Connected to Weaviate at {url}")
|
||||
|
||||
def create_collection(
|
||||
self,
|
||||
name: str,
|
||||
dimension: int,
|
||||
distance: str = "cosine",
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""Create a new collection (class in Weaviate)."""
|
||||
from weaviate.classes.config import Configure, Property, DataType
|
||||
|
||||
distance_map = {"cosine": "cosine", "euclidean": "l2-squared", "dot": "dot"}
|
||||
|
||||
self._client.collections.create(
|
||||
name=name,
|
||||
vectorizer_config=Configure.Vectorizer.none(),
|
||||
vector_index_config=Configure.VectorIndex.hnsw(
|
||||
distance_metric=distance_map.get(distance, "cosine")
|
||||
),
|
||||
properties=[
|
||||
Property(name="content", data_type=DataType.TEXT),
|
||||
Property(name="content_hash", data_type=DataType.TEXT),
|
||||
Property(name="created_at", data_type=DataType.NUMBER),
|
||||
]
|
||||
)
|
||||
logger.info(f"Created Weaviate collection: {name}")
|
||||
|
||||
def delete_collection(self, name: str) -> bool:
|
||||
"""Delete a collection."""
|
||||
try:
|
||||
self._client.collections.delete(name)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to delete collection {name}: {e}")
|
||||
return False
|
||||
|
||||
def collection_exists(self, name: str) -> bool:
|
||||
"""Check if a collection exists."""
|
||||
return self._client.collections.exists(name)
|
||||
|
||||
def list_collections(self) -> List[str]:
|
||||
"""List all collections."""
|
||||
return [c.name for c in self._client.collections.list_all().values()]
|
||||
|
||||
def insert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert documents."""
|
||||
col = self._client.collections.get(collection)
|
||||
ids = []
|
||||
|
||||
with col.batch.dynamic() as batch:
|
||||
for doc in documents:
|
||||
if doc.embedding is None:
|
||||
raise ValueError(f"Document {doc.id} has no embedding")
|
||||
|
||||
properties = {
|
||||
"content": doc.content,
|
||||
"content_hash": doc.content_hash or "",
|
||||
"created_at": doc.created_at,
|
||||
**(doc.metadata or {})
|
||||
}
|
||||
|
||||
batch.add_object(
|
||||
properties=properties,
|
||||
vector=doc.embedding,
|
||||
uuid=doc.id,
|
||||
)
|
||||
ids.append(doc.id)
|
||||
|
||||
return ids
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
collection: str,
|
||||
documents: List[KnowledgeDocument]
|
||||
) -> List[str]:
|
||||
"""Insert or update documents."""
|
||||
return self.insert(collection, documents)
|
||||
|
||||
def search(
|
||||
self,
|
||||
collection: str,
|
||||
query_embedding: List[float],
|
||||
limit: int = 5,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
score_threshold: Optional[float] = None
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Search for similar documents."""
|
||||
col = self._client.collections.get(collection)
|
||||
|
||||
query = col.query.near_vector(
|
||||
near_vector=query_embedding,
|
||||
limit=limit,
|
||||
return_metadata=["distance"],
|
||||
)
|
||||
|
||||
documents = []
|
||||
for obj in query.objects:
|
||||
if score_threshold:
|
||||
score = 1 - (obj.metadata.distance or 0)
|
||||
if score < score_threshold:
|
||||
continue
|
||||
|
||||
props = obj.properties
|
||||
doc = KnowledgeDocument(
|
||||
id=str(obj.uuid),
|
||||
content=props.get("content", ""),
|
||||
embedding=None,
|
||||
metadata={k: v for k, v in props.items() if k not in ("content", "content_hash", "created_at")},
|
||||
content_hash=props.get("content_hash"),
|
||||
created_at=props.get("created_at", 0),
|
||||
)
|
||||
documents.append(doc)
|
||||
|
||||
return documents
|
||||
|
||||
def get(
|
||||
self,
|
||||
collection: str,
|
||||
ids: List[str]
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""Get documents by IDs."""
|
||||
col = self._client.collections.get(collection)
|
||||
documents = []
|
||||
|
||||
for doc_id in ids:
|
||||
try:
|
||||
obj = col.query.fetch_object_by_id(doc_id, include_vector=True)
|
||||
if obj:
|
||||
props = obj.properties
|
||||
doc = KnowledgeDocument(
|
||||
id=str(obj.uuid),
|
||||
content=props.get("content", ""),
|
||||
embedding=list(obj.vector) if obj.vector else None,
|
||||
metadata={k: v for k, v in props.items() if k not in ("content", "content_hash", "created_at")},
|
||||
content_hash=props.get("content_hash"),
|
||||
created_at=props.get("created_at", 0),
|
||||
)
|
||||
documents.append(doc)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return documents
|
||||
|
||||
def delete(
|
||||
self,
|
||||
collection: str,
|
||||
ids: Optional[List[str]] = None,
|
||||
filters: Optional[Dict[str, Any]] = None
|
||||
) -> int:
|
||||
"""Delete documents."""
|
||||
col = self._client.collections.get(collection)
|
||||
|
||||
if ids:
|
||||
for doc_id in ids:
|
||||
col.data.delete_by_id(doc_id)
|
||||
return len(ids)
|
||||
return 0
|
||||
|
||||
def count(self, collection: str) -> int:
|
||||
"""Count documents."""
|
||||
col = self._client.collections.get(collection)
|
||||
return col.aggregate.over_all(total_count=True).total_count or 0
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the store."""
|
||||
if self._client:
|
||||
self._client.close()
|
||||
@@ -0,0 +1,10 @@
|
||||
"""
|
||||
Schema versioning and migrations for PraisonAI persistence.
|
||||
|
||||
This module provides tools for managing database schema versions
|
||||
and applying migrations when upgrading.
|
||||
"""
|
||||
|
||||
from .manager import MigrationManager, SchemaVersion
|
||||
|
||||
__all__ = ["MigrationManager", "SchemaVersion"]
|
||||
@@ -0,0 +1,251 @@
|
||||
"""
|
||||
Migration manager for PraisonAI persistence schema versioning.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
from packaging import version as pkg_version
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchemaVersion:
|
||||
"""Represents a schema version."""
|
||||
version: str
|
||||
description: str = ""
|
||||
applied_at: Optional[float] = None
|
||||
|
||||
def __lt__(self, other: "SchemaVersion") -> bool:
|
||||
return pkg_version.parse(self.version) < pkg_version.parse(other.version)
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
if not isinstance(other, SchemaVersion):
|
||||
return False
|
||||
return self.version == other.version
|
||||
|
||||
|
||||
@dataclass
|
||||
class Migration:
|
||||
"""Represents a single migration."""
|
||||
version: str
|
||||
description: str
|
||||
up: Callable[[Any], None]
|
||||
down: Optional[Callable[[Any], None]] = None
|
||||
|
||||
|
||||
class MigrationManager:
|
||||
"""
|
||||
Manages database schema versions and migrations.
|
||||
|
||||
Example:
|
||||
manager = MigrationManager(db_adapter)
|
||||
manager.register_migration(
|
||||
version="1.0.0",
|
||||
description="Initial schema",
|
||||
up=lambda db: db.execute("CREATE TABLE ..."),
|
||||
down=lambda db: db.execute("DROP TABLE ...")
|
||||
)
|
||||
manager.migrate_up() # Apply all pending migrations
|
||||
"""
|
||||
|
||||
# Current schema version
|
||||
CURRENT_VERSION = "1.0.0"
|
||||
|
||||
def __init__(self, db_adapter: Any = None):
|
||||
"""
|
||||
Initialize migration manager.
|
||||
|
||||
Args:
|
||||
db_adapter: Database adapter (PraisonDB or similar)
|
||||
"""
|
||||
self._db = db_adapter
|
||||
self._migrations: List[Migration] = []
|
||||
self._versions_table = "praison_schema_versions"
|
||||
|
||||
# Register built-in migrations
|
||||
self._register_builtin_migrations()
|
||||
|
||||
def _register_builtin_migrations(self):
|
||||
"""Register built-in migrations."""
|
||||
self.register_migration(
|
||||
version="1.0.0",
|
||||
description="Initial schema with sessions, messages, runs",
|
||||
up=self._migrate_1_0_0_up,
|
||||
down=self._migrate_1_0_0_down
|
||||
)
|
||||
|
||||
def register_migration(
|
||||
self,
|
||||
version: str,
|
||||
description: str,
|
||||
up: Callable[[Any], None],
|
||||
down: Optional[Callable[[Any], None]] = None
|
||||
):
|
||||
"""
|
||||
Register a new migration.
|
||||
|
||||
Args:
|
||||
version: Version string (e.g., "1.0.0")
|
||||
description: Human-readable description
|
||||
up: Function to apply migration
|
||||
down: Optional function to rollback migration
|
||||
"""
|
||||
migration = Migration(
|
||||
version=version,
|
||||
description=description,
|
||||
up=up,
|
||||
down=down
|
||||
)
|
||||
self._migrations.append(migration)
|
||||
self._migrations.sort(key=lambda m: pkg_version.parse(m.version))
|
||||
|
||||
def get_current_version(self) -> Optional[str]:
|
||||
"""Get the current schema version from database."""
|
||||
if not self._db:
|
||||
return None
|
||||
|
||||
try:
|
||||
# Try to get version from state store
|
||||
if hasattr(self._db, '_state_store') and self._db._state_store:
|
||||
self._db._init_stores()
|
||||
version_data = self._db._state_store.get(f"{self._versions_table}:current")
|
||||
if version_data:
|
||||
return version_data.get("version")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to get schema version: {e}")
|
||||
|
||||
return None
|
||||
|
||||
def set_current_version(self, version: str):
|
||||
"""Set the current schema version in database."""
|
||||
if not self._db:
|
||||
return
|
||||
|
||||
try:
|
||||
if hasattr(self._db, '_state_store') and self._db._state_store:
|
||||
self._db._init_stores()
|
||||
import time
|
||||
self._db._state_store.set(f"{self._versions_table}:current", {
|
||||
"version": version,
|
||||
"applied_at": time.time()
|
||||
})
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to set schema version: {e}")
|
||||
|
||||
def get_pending_migrations(self) -> List[Migration]:
|
||||
"""Get list of migrations that haven't been applied."""
|
||||
current = self.get_current_version()
|
||||
if not current:
|
||||
return self._migrations.copy()
|
||||
|
||||
current_ver = pkg_version.parse(current)
|
||||
return [
|
||||
m for m in self._migrations
|
||||
if pkg_version.parse(m.version) > current_ver
|
||||
]
|
||||
|
||||
def migrate_up(self, target_version: Optional[str] = None) -> List[str]:
|
||||
"""
|
||||
Apply pending migrations up to target version.
|
||||
|
||||
Args:
|
||||
target_version: Optional target version (default: latest)
|
||||
|
||||
Returns:
|
||||
List of applied migration versions
|
||||
"""
|
||||
pending = self.get_pending_migrations()
|
||||
if not pending:
|
||||
logger.info("No pending migrations")
|
||||
return []
|
||||
|
||||
target = pkg_version.parse(target_version) if target_version else None
|
||||
applied = []
|
||||
|
||||
for migration in pending:
|
||||
if target and pkg_version.parse(migration.version) > target:
|
||||
break
|
||||
|
||||
logger.info(f"Applying migration {migration.version}: {migration.description}")
|
||||
try:
|
||||
migration.up(self._db)
|
||||
self.set_current_version(migration.version)
|
||||
applied.append(migration.version)
|
||||
logger.info(f"Successfully applied migration {migration.version}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to apply migration {migration.version}: {e}")
|
||||
raise
|
||||
|
||||
return applied
|
||||
|
||||
def migrate_down(self, target_version: str) -> List[str]:
|
||||
"""
|
||||
Rollback migrations down to target version.
|
||||
|
||||
Args:
|
||||
target_version: Target version to rollback to
|
||||
|
||||
Returns:
|
||||
List of rolled back migration versions
|
||||
"""
|
||||
current = self.get_current_version()
|
||||
if not current:
|
||||
logger.info("No migrations to rollback")
|
||||
return []
|
||||
|
||||
current_ver = pkg_version.parse(current)
|
||||
target_ver = pkg_version.parse(target_version)
|
||||
|
||||
if target_ver >= current_ver:
|
||||
logger.info("Target version is not lower than current")
|
||||
return []
|
||||
|
||||
# Get migrations to rollback (in reverse order)
|
||||
to_rollback = [
|
||||
m for m in reversed(self._migrations)
|
||||
if pkg_version.parse(m.version) > target_ver and pkg_version.parse(m.version) <= current_ver
|
||||
]
|
||||
|
||||
rolled_back = []
|
||||
for migration in to_rollback:
|
||||
if not migration.down:
|
||||
logger.warning(f"Migration {migration.version} has no down function, skipping")
|
||||
continue
|
||||
|
||||
logger.info(f"Rolling back migration {migration.version}")
|
||||
try:
|
||||
migration.down(self._db)
|
||||
rolled_back.append(migration.version)
|
||||
logger.info(f"Successfully rolled back migration {migration.version}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to rollback migration {migration.version}: {e}")
|
||||
raise
|
||||
|
||||
self.set_current_version(target_version)
|
||||
return rolled_back
|
||||
|
||||
def get_migration_status(self) -> Dict[str, Any]:
|
||||
"""Get current migration status."""
|
||||
current = self.get_current_version()
|
||||
pending = self.get_pending_migrations()
|
||||
|
||||
return {
|
||||
"current_version": current,
|
||||
"latest_version": self._migrations[-1].version if self._migrations else None,
|
||||
"pending_count": len(pending),
|
||||
"pending_versions": [m.version for m in pending],
|
||||
"all_versions": [m.version for m in self._migrations]
|
||||
}
|
||||
|
||||
# --- Built-in Migrations ---
|
||||
|
||||
def _migrate_1_0_0_up(self, db: Any):
|
||||
"""Initial schema migration."""
|
||||
# This is a no-op since tables are created on first use
|
||||
logger.info("Initial schema (1.0.0) - tables created on first use")
|
||||
|
||||
def _migrate_1_0_0_down(self, db: Any):
|
||||
"""Rollback initial schema."""
|
||||
logger.info("Rolling back initial schema (1.0.0)")
|
||||
@@ -0,0 +1,406 @@
|
||||
"""
|
||||
PersistenceOrchestrator - Central coordinator for all persistence operations.
|
||||
|
||||
Hooks into agent lifecycle to provide automatic conversation persistence,
|
||||
knowledge retrieval, and state management.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional, TYPE_CHECKING
|
||||
|
||||
from .conversation.base import ConversationStore, ConversationSession, ConversationMessage
|
||||
from .knowledge.base import KnowledgeStore, KnowledgeDocument
|
||||
from .state.base import StateStore
|
||||
from .config import PersistenceConfig
|
||||
from .factory import create_stores_from_config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PersistenceOrchestrator:
|
||||
"""
|
||||
Central coordinator for all persistence operations.
|
||||
|
||||
Lives in wrapper layer, hooks into agent lifecycle to provide:
|
||||
- Automatic session loading/creation at agent start
|
||||
- Message persistence during conversation
|
||||
- Session metadata updates at agent end
|
||||
- Knowledge retrieval for RAG
|
||||
- State caching
|
||||
|
||||
Example:
|
||||
orchestrator = PersistenceOrchestrator(
|
||||
conversation_store=PostgresConversationStore(...),
|
||||
knowledge_store=QdrantKnowledgeStore(...),
|
||||
state_store=RedisStateStore(...),
|
||||
)
|
||||
|
||||
# Hook into agent lifecycle
|
||||
history = orchestrator.on_agent_start(agent, session_id="sess-123")
|
||||
# ... agent runs ...
|
||||
orchestrator.on_message(session_id, user_message)
|
||||
orchestrator.on_message(session_id, assistant_message)
|
||||
orchestrator.on_agent_end(agent, session_id)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
conversation_store: Optional[ConversationStore] = None,
|
||||
knowledge_store: Optional[KnowledgeStore] = None,
|
||||
state_store: Optional[StateStore] = None,
|
||||
config: Optional[PersistenceConfig] = None,
|
||||
):
|
||||
"""
|
||||
Initialize the orchestrator.
|
||||
|
||||
Args:
|
||||
conversation_store: Store for session/message persistence
|
||||
knowledge_store: Store for vector embeddings
|
||||
state_store: Store for key-value state
|
||||
config: Configuration (alternative to passing stores directly)
|
||||
"""
|
||||
if config:
|
||||
stores = create_stores_from_config(config)
|
||||
self.conversation = stores["conversation"] or conversation_store
|
||||
self.knowledge = stores["knowledge"] or knowledge_store
|
||||
self.state = stores["state"] or state_store
|
||||
self._config = config
|
||||
else:
|
||||
self.conversation = conversation_store
|
||||
self.knowledge = knowledge_store
|
||||
self.state = state_store
|
||||
self._config = None
|
||||
|
||||
self._current_session: Optional[ConversationSession] = None
|
||||
self._session_cache: Dict[str, ConversationSession] = {}
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: PersistenceConfig) -> "PersistenceOrchestrator":
|
||||
"""Create orchestrator from configuration."""
|
||||
return cls(config=config)
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "PersistenceOrchestrator":
|
||||
"""Create orchestrator from environment variables."""
|
||||
config = PersistenceConfig.from_env()
|
||||
return cls(config=config)
|
||||
|
||||
# =========================================================================
|
||||
# Agent Lifecycle Hooks
|
||||
# =========================================================================
|
||||
|
||||
def on_agent_start(
|
||||
self,
|
||||
agent: Any,
|
||||
session_id: Optional[str] = None,
|
||||
user_id: Optional[str] = None,
|
||||
resume: bool = True,
|
||||
) -> List[ConversationMessage]:
|
||||
"""
|
||||
Called before agent run. Loads or creates session.
|
||||
|
||||
Args:
|
||||
agent: The agent instance
|
||||
session_id: Session ID (generated if not provided)
|
||||
user_id: User ID for session
|
||||
resume: Whether to load existing session history
|
||||
|
||||
Returns:
|
||||
List of previous messages if resuming, empty list otherwise
|
||||
"""
|
||||
if not session_id:
|
||||
session_id = str(uuid.uuid4())
|
||||
|
||||
if not self.conversation:
|
||||
logger.debug("No conversation store configured, skipping session load")
|
||||
return []
|
||||
|
||||
# Try to load existing session
|
||||
session = None
|
||||
if resume:
|
||||
session = self.conversation.get_session(session_id)
|
||||
|
||||
if session:
|
||||
logger.info(f"Resuming session: {session_id}")
|
||||
self._current_session = session
|
||||
self._session_cache[session_id] = session
|
||||
|
||||
# Load previous messages
|
||||
messages = self.conversation.get_messages(session_id)
|
||||
return messages
|
||||
else:
|
||||
# Create new session
|
||||
agent_id = getattr(agent, "name", None) or getattr(agent, "agent_id", None)
|
||||
session = ConversationSession(
|
||||
session_id=session_id,
|
||||
user_id=user_id,
|
||||
agent_id=agent_id,
|
||||
name=f"Session {session_id[:8]}",
|
||||
metadata={"agent_type": type(agent).__name__},
|
||||
)
|
||||
self.conversation.create_session(session)
|
||||
logger.info(f"Created new session: {session_id}")
|
||||
self._current_session = session
|
||||
self._session_cache[session_id] = session
|
||||
return []
|
||||
|
||||
def on_message(
|
||||
self,
|
||||
session_id: str,
|
||||
role: str,
|
||||
content: str,
|
||||
tool_calls: Optional[List[Dict]] = None,
|
||||
tool_call_id: Optional[str] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
) -> Optional[ConversationMessage]:
|
||||
"""
|
||||
Called after each message (user or assistant).
|
||||
|
||||
Args:
|
||||
session_id: Session ID
|
||||
role: Message role (user, assistant, system, tool)
|
||||
content: Message content
|
||||
tool_calls: Tool calls (for assistant messages)
|
||||
tool_call_id: Tool call ID (for tool response messages)
|
||||
metadata: Additional metadata
|
||||
|
||||
Returns:
|
||||
The persisted message, or None if no store configured
|
||||
"""
|
||||
if not self.conversation:
|
||||
return None
|
||||
|
||||
message = ConversationMessage(
|
||||
id=str(uuid.uuid4()),
|
||||
session_id=session_id,
|
||||
role=role,
|
||||
content=content,
|
||||
tool_calls=tool_calls,
|
||||
tool_call_id=tool_call_id,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
self.conversation.add_message(session_id, message)
|
||||
logger.debug(f"Persisted {role} message to session {session_id}")
|
||||
return message
|
||||
|
||||
def on_agent_end(
|
||||
self,
|
||||
agent: Any,
|
||||
session_id: str,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Called after agent run. Updates session metadata.
|
||||
|
||||
Args:
|
||||
agent: The agent instance
|
||||
session_id: Session ID
|
||||
metadata: Additional metadata to store
|
||||
"""
|
||||
if not self.conversation:
|
||||
return
|
||||
|
||||
session = self._session_cache.get(session_id) or self.conversation.get_session(session_id)
|
||||
if session:
|
||||
session.updated_at = time.time()
|
||||
if metadata:
|
||||
session.metadata = {**(session.metadata or {}), **metadata}
|
||||
self.conversation.update_session(session)
|
||||
logger.debug(f"Updated session metadata: {session_id}")
|
||||
|
||||
# =========================================================================
|
||||
# Knowledge Retrieval
|
||||
# =========================================================================
|
||||
|
||||
def retrieve_knowledge(
|
||||
self,
|
||||
query_embedding: List[float],
|
||||
collection: str = "default",
|
||||
limit: int = 5,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
) -> List[KnowledgeDocument]:
|
||||
"""
|
||||
Retrieve relevant documents for RAG.
|
||||
|
||||
Args:
|
||||
query_embedding: Query vector embedding
|
||||
collection: Collection name
|
||||
limit: Max documents to return
|
||||
filters: Metadata filters
|
||||
|
||||
Returns:
|
||||
List of relevant documents
|
||||
"""
|
||||
if not self.knowledge:
|
||||
logger.debug("No knowledge store configured")
|
||||
return []
|
||||
|
||||
return self.knowledge.search(
|
||||
collection=collection,
|
||||
query_embedding=query_embedding,
|
||||
limit=limit,
|
||||
filters=filters,
|
||||
)
|
||||
|
||||
def add_knowledge(
|
||||
self,
|
||||
documents: List[KnowledgeDocument],
|
||||
collection: str = "default",
|
||||
) -> List[str]:
|
||||
"""
|
||||
Add documents to knowledge store.
|
||||
|
||||
Args:
|
||||
documents: Documents with embeddings
|
||||
collection: Collection name
|
||||
|
||||
Returns:
|
||||
List of document IDs
|
||||
"""
|
||||
if not self.knowledge:
|
||||
raise ValueError("No knowledge store configured")
|
||||
|
||||
return self.knowledge.upsert(collection, documents)
|
||||
|
||||
# =========================================================================
|
||||
# State Management
|
||||
# =========================================================================
|
||||
|
||||
def get_state(self, key: str) -> Optional[Any]:
|
||||
"""Get state value."""
|
||||
if not self.state:
|
||||
return None
|
||||
return self.state.get(key)
|
||||
|
||||
def set_state(self, key: str, value: Any, ttl: Optional[int] = None) -> None:
|
||||
"""Set state value with optional TTL."""
|
||||
if not self.state:
|
||||
return
|
||||
self.state.set(key, value, ttl)
|
||||
|
||||
def delete_state(self, key: str) -> bool:
|
||||
"""Delete state value."""
|
||||
if not self.state:
|
||||
return False
|
||||
return self.state.delete(key)
|
||||
|
||||
# =========================================================================
|
||||
# Session Management
|
||||
# =========================================================================
|
||||
|
||||
def get_session(self, session_id: str) -> Optional[ConversationSession]:
|
||||
"""Get a session by ID."""
|
||||
if not self.conversation:
|
||||
return None
|
||||
return self.conversation.get_session(session_id)
|
||||
|
||||
def list_sessions(
|
||||
self,
|
||||
user_id: Optional[str] = None,
|
||||
limit: int = 100,
|
||||
) -> List[ConversationSession]:
|
||||
"""List sessions for a user."""
|
||||
if not self.conversation:
|
||||
return []
|
||||
return self.conversation.list_sessions(user_id=user_id, limit=limit)
|
||||
|
||||
def delete_session(self, session_id: str) -> bool:
|
||||
"""Delete a session and all its messages."""
|
||||
if not self.conversation:
|
||||
return False
|
||||
|
||||
if session_id in self._session_cache:
|
||||
del self._session_cache[session_id]
|
||||
|
||||
return self.conversation.delete_session(session_id)
|
||||
|
||||
def get_messages(
|
||||
self,
|
||||
session_id: str,
|
||||
limit: Optional[int] = None,
|
||||
) -> List[ConversationMessage]:
|
||||
"""Get messages from a session."""
|
||||
if not self.conversation:
|
||||
return []
|
||||
return self.conversation.get_messages(session_id, limit=limit)
|
||||
|
||||
# =========================================================================
|
||||
# Context Building
|
||||
# =========================================================================
|
||||
|
||||
def build_context(
|
||||
self,
|
||||
session_id: str,
|
||||
query_embedding: Optional[List[float]] = None,
|
||||
history_limit: int = 20,
|
||||
knowledge_limit: int = 5,
|
||||
knowledge_collection: str = "default",
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Build context for agent including history and knowledge.
|
||||
|
||||
Args:
|
||||
session_id: Session ID
|
||||
query_embedding: Query embedding for knowledge retrieval
|
||||
history_limit: Max history messages
|
||||
knowledge_limit: Max knowledge documents
|
||||
knowledge_collection: Knowledge collection name
|
||||
|
||||
Returns:
|
||||
Dict with 'history' and 'knowledge' keys
|
||||
"""
|
||||
context = {
|
||||
"history": [],
|
||||
"knowledge": [],
|
||||
}
|
||||
|
||||
# Get conversation history
|
||||
if self.conversation:
|
||||
messages = self.conversation.get_messages(session_id, limit=history_limit)
|
||||
context["history"] = [
|
||||
{"role": m.role, "content": m.content}
|
||||
for m in messages
|
||||
]
|
||||
|
||||
# Get relevant knowledge
|
||||
if self.knowledge and query_embedding:
|
||||
docs = self.knowledge.search(
|
||||
collection=knowledge_collection,
|
||||
query_embedding=query_embedding,
|
||||
limit=knowledge_limit,
|
||||
)
|
||||
context["knowledge"] = [
|
||||
{"content": d.content, "metadata": d.metadata}
|
||||
for d in docs
|
||||
]
|
||||
|
||||
return context
|
||||
|
||||
# =========================================================================
|
||||
# Cleanup
|
||||
# =========================================================================
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close all stores and release resources."""
|
||||
if self.conversation:
|
||||
self.conversation.close()
|
||||
if self.knowledge:
|
||||
self.knowledge.close()
|
||||
if self.state:
|
||||
self.state.close()
|
||||
|
||||
self._session_cache.clear()
|
||||
logger.info("Persistence orchestrator closed")
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.close()
|
||||
return False
|
||||
@@ -0,0 +1,21 @@
|
||||
"""
|
||||
StateStore implementations for fast key-value state and caching.
|
||||
|
||||
Supported backends:
|
||||
- Redis (redis)
|
||||
- DynamoDB (dynamodb)
|
||||
- Firestore (firestore)
|
||||
- MongoDB (mongodb)
|
||||
- Upstash (upstash)
|
||||
- In-memory / JSON file (memory)
|
||||
"""
|
||||
|
||||
__all__ = [
|
||||
"StateStore",
|
||||
]
|
||||
|
||||
def __getattr__(name: str):
|
||||
if name == "StateStore":
|
||||
from .base import StateStore
|
||||
return StateStore
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
@@ -0,0 +1,200 @@
|
||||
"""
|
||||
Async MongoDB implementation of StateStore.
|
||||
|
||||
Requires: motor
|
||||
Install: pip install motor
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import StateStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AsyncMongoDBStateStore(StateStore):
|
||||
"""
|
||||
Async MongoDB state store using motor.
|
||||
|
||||
Provides high-performance async database operations.
|
||||
|
||||
Example:
|
||||
store = AsyncMongoDBStateStore(
|
||||
url="mongodb://localhost:27017",
|
||||
database="praisonai"
|
||||
)
|
||||
await store.init()
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: str = "mongodb://localhost:27017",
|
||||
database: str = "praisonai",
|
||||
collection: str = "state",
|
||||
):
|
||||
"""
|
||||
Initialize async MongoDB store.
|
||||
|
||||
Args:
|
||||
url: MongoDB connection URL
|
||||
database: Database name
|
||||
collection: Collection name
|
||||
"""
|
||||
self.url = url
|
||||
self.database_name = database
|
||||
self.collection_name = collection
|
||||
self._client = None
|
||||
self._db = None
|
||||
self._collection = None
|
||||
self._initialized = False
|
||||
|
||||
async def init(self):
|
||||
"""Initialize connection."""
|
||||
if self._initialized:
|
||||
return
|
||||
|
||||
try:
|
||||
from motor.motor_asyncio import AsyncIOMotorClient
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"motor is required for async MongoDB support. "
|
||||
"Install with: pip install motor"
|
||||
)
|
||||
|
||||
self._client = AsyncIOMotorClient(self.url)
|
||||
self._db = self._client[self.database_name]
|
||||
self._collection = self._db[self.collection_name]
|
||||
|
||||
# Create TTL index for expiration
|
||||
await self._collection.create_index("expires_at", expireAfterSeconds=0)
|
||||
|
||||
self._initialized = True
|
||||
|
||||
async def async_get(self, key: str) -> Optional[Dict[str, Any]]:
|
||||
"""Get state by key asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
doc = await self._collection.find_one({"_id": key})
|
||||
if doc:
|
||||
# Check TTL manually (in case index hasn't cleaned up yet)
|
||||
if "expires_at" in doc and doc["expires_at"]:
|
||||
if time.time() > doc["expires_at"]:
|
||||
await self.async_delete(key)
|
||||
return None
|
||||
|
||||
# Remove internal fields
|
||||
result = dict(doc)
|
||||
result.pop("_id", None)
|
||||
result.pop("expires_at", None)
|
||||
return result.get("value", result)
|
||||
return None
|
||||
|
||||
def get(self, key: str) -> Optional[Dict[str, Any]]:
|
||||
"""Sync wrapper for get."""
|
||||
return asyncio.get_event_loop().run_until_complete(self.async_get(key))
|
||||
|
||||
async def async_set(self, key: str, value: Dict[str, Any], ttl: Optional[int] = None) -> bool:
|
||||
"""Set state by key with optional TTL asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
doc = {
|
||||
"_id": key,
|
||||
"value": value,
|
||||
"updated_at": time.time(),
|
||||
}
|
||||
|
||||
if ttl:
|
||||
doc["expires_at"] = time.time() + ttl
|
||||
|
||||
try:
|
||||
await self._collection.replace_one(
|
||||
{"_id": key},
|
||||
doc,
|
||||
upsert=True
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Error setting state {key}: {e}")
|
||||
return False
|
||||
|
||||
def set(self, key: str, value: Dict[str, Any], ttl: Optional[int] = None) -> bool:
|
||||
"""Sync wrapper for set."""
|
||||
return asyncio.get_event_loop().run_until_complete(self.async_set(key, value, ttl))
|
||||
|
||||
async def async_delete(self, key: str) -> bool:
|
||||
"""Delete state by key asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
result = await self._collection.delete_one({"_id": key})
|
||||
return result.deleted_count > 0
|
||||
|
||||
def delete(self, key: str) -> bool:
|
||||
"""Sync wrapper for delete."""
|
||||
return asyncio.get_event_loop().run_until_complete(self.async_delete(key))
|
||||
|
||||
async def async_exists(self, key: str) -> bool:
|
||||
"""Check if key exists asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
count = await self._collection.count_documents({"_id": key}, limit=1)
|
||||
return count > 0
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
"""Sync wrapper for exists."""
|
||||
return asyncio.get_event_loop().run_until_complete(self.async_exists(key))
|
||||
|
||||
async def async_list_keys(self, prefix: Optional[str] = None) -> List[str]:
|
||||
"""List all keys with optional prefix filter asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
query = {}
|
||||
if prefix:
|
||||
query["_id"] = {"$regex": f"^{prefix}"}
|
||||
|
||||
cursor = self._collection.find(query, {"_id": 1})
|
||||
keys = []
|
||||
async for doc in cursor:
|
||||
keys.append(doc["_id"])
|
||||
return keys
|
||||
|
||||
def list_keys(self, prefix: Optional[str] = None) -> List[str]:
|
||||
"""Sync wrapper for list_keys."""
|
||||
return asyncio.get_event_loop().run_until_complete(self.async_list_keys(prefix))
|
||||
|
||||
async def async_clear(self, prefix: Optional[str] = None) -> int:
|
||||
"""Clear all keys with optional prefix filter asynchronously."""
|
||||
if not self._initialized:
|
||||
await self.init()
|
||||
|
||||
query = {}
|
||||
if prefix:
|
||||
query["_id"] = {"$regex": f"^{prefix}"}
|
||||
|
||||
result = await self._collection.delete_many(query)
|
||||
return result.deleted_count
|
||||
|
||||
def clear(self, prefix: Optional[str] = None) -> int:
|
||||
"""Sync wrapper for clear."""
|
||||
return asyncio.get_event_loop().run_until_complete(self.async_clear(prefix))
|
||||
|
||||
async def async_close(self) -> None:
|
||||
"""Close the connection."""
|
||||
if self._client:
|
||||
self._client.close()
|
||||
self._client = None
|
||||
self._db = None
|
||||
self._collection = None
|
||||
self._initialized = False
|
||||
|
||||
def close(self) -> None:
|
||||
"""Sync wrapper for close."""
|
||||
if self._client:
|
||||
asyncio.get_event_loop().run_until_complete(self.async_close())
|
||||
@@ -0,0 +1,107 @@
|
||||
"""
|
||||
Base interfaces for StateStore.
|
||||
|
||||
StateStore handles fast key-value state and caching.
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Dict, List, Optional
|
||||
import json
|
||||
|
||||
|
||||
class StateStore(ABC):
|
||||
"""
|
||||
Abstract base class for key-value state persistence.
|
||||
|
||||
Implementations handle fast state/cache storage for different backends:
|
||||
- Redis (dedicated KV store)
|
||||
- DynamoDB, Firestore (serverless document stores)
|
||||
- MongoDB (document store)
|
||||
- Upstash (serverless Redis)
|
||||
- In-memory / JSON file (zero-dependency fallback)
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get(self, key: str) -> Optional[Any]:
|
||||
"""Get a value by key. Returns None if not found."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def set(
|
||||
self,
|
||||
key: str,
|
||||
value: Any,
|
||||
ttl: Optional[int] = None
|
||||
) -> None:
|
||||
"""Set a value. TTL is in seconds."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def delete(self, key: str) -> bool:
|
||||
"""Delete a key. Returns True if key existed."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def exists(self, key: str) -> bool:
|
||||
"""Check if a key exists."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def keys(self, pattern: str = "*") -> List[str]:
|
||||
"""List keys matching pattern."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def ttl(self, key: str) -> Optional[int]:
|
||||
"""Get remaining TTL in seconds. Returns None if no TTL or key doesn't exist."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def expire(self, key: str, ttl: int) -> bool:
|
||||
"""Set TTL on existing key. Returns True if key exists."""
|
||||
raise NotImplementedError
|
||||
|
||||
def get_json(self, key: str) -> Optional[Any]:
|
||||
"""Get and deserialize JSON value."""
|
||||
value = self.get(key)
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, str):
|
||||
return json.loads(value)
|
||||
return value
|
||||
|
||||
def set_json(self, key: str, value: Any, ttl: Optional[int] = None) -> None:
|
||||
"""Serialize and set JSON value."""
|
||||
self.set(key, json.dumps(value), ttl)
|
||||
|
||||
@abstractmethod
|
||||
def hget(self, key: str, field: str) -> Optional[Any]:
|
||||
"""Get a field from a hash."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def hset(self, key: str, field: str, value: Any) -> None:
|
||||
"""Set a field in a hash."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def hgetall(self, key: str) -> Dict[str, Any]:
|
||||
"""Get all fields from a hash."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def hdel(self, key: str, *fields: str) -> int:
|
||||
"""Delete fields from a hash. Returns count deleted."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def close(self) -> None:
|
||||
"""Close the store and release resources."""
|
||||
raise NotImplementedError
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.close()
|
||||
return False
|
||||
@@ -0,0 +1,226 @@
|
||||
"""
|
||||
DynamoDB implementation of StateStore.
|
||||
|
||||
Requires: boto3
|
||||
Install: pip install boto3
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import StateStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DynamoDBStateStore(StateStore):
|
||||
"""
|
||||
DynamoDB-based state store.
|
||||
|
||||
Example:
|
||||
store = DynamoDBStateStore(
|
||||
table_name="praisonai_state",
|
||||
region="us-east-1"
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
table_name: str = "praisonai_state",
|
||||
region: Optional[str] = None,
|
||||
endpoint_url: Optional[str] = None,
|
||||
auto_create_table: bool = True,
|
||||
):
|
||||
try:
|
||||
import boto3
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"boto3 is required for DynamoDB support. "
|
||||
"Install with: pip install boto3"
|
||||
)
|
||||
|
||||
region = region or os.getenv("AWS_REGION", "us-east-1")
|
||||
|
||||
self._dynamodb = boto3.resource(
|
||||
"dynamodb",
|
||||
region_name=region,
|
||||
endpoint_url=endpoint_url,
|
||||
)
|
||||
self._client = boto3.client(
|
||||
"dynamodb",
|
||||
region_name=region,
|
||||
endpoint_url=endpoint_url,
|
||||
)
|
||||
self.table_name = table_name
|
||||
|
||||
if auto_create_table:
|
||||
self._create_table()
|
||||
|
||||
self._table = self._dynamodb.Table(table_name)
|
||||
logger.info(f"Connected to DynamoDB table: {table_name}")
|
||||
|
||||
def _create_table(self) -> None:
|
||||
"""Create table if not exists."""
|
||||
try:
|
||||
self._client.describe_table(TableName=self.table_name)
|
||||
except self._client.exceptions.ResourceNotFoundException:
|
||||
self._client.create_table(
|
||||
TableName=self.table_name,
|
||||
KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}],
|
||||
AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}],
|
||||
BillingMode="PAY_PER_REQUEST",
|
||||
)
|
||||
waiter = self._client.get_waiter("table_exists")
|
||||
waiter.wait(TableName=self.table_name)
|
||||
|
||||
# Enable TTL
|
||||
self._client.update_time_to_live(
|
||||
TableName=self.table_name,
|
||||
TimeToLiveSpecification={"Enabled": True, "AttributeName": "ttl"}
|
||||
)
|
||||
logger.info(f"Created DynamoDB table: {self.table_name}")
|
||||
|
||||
def get(self, key: str) -> Optional[Any]:
|
||||
"""Get a value by key."""
|
||||
response = self._table.get_item(Key={"pk": key})
|
||||
item = response.get("Item")
|
||||
|
||||
if not item:
|
||||
return None
|
||||
|
||||
# Check TTL
|
||||
if item.get("ttl") and item["ttl"] <= int(time.time()):
|
||||
return None
|
||||
|
||||
value = item.get("value")
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
return value
|
||||
return value
|
||||
|
||||
def set(
|
||||
self,
|
||||
key: str,
|
||||
value: Any,
|
||||
ttl: Optional[int] = None
|
||||
) -> None:
|
||||
"""Set a value with optional TTL."""
|
||||
item = {
|
||||
"pk": key,
|
||||
"value": json.dumps(value) if not isinstance(value, str) else value,
|
||||
"updated_at": int(time.time()),
|
||||
}
|
||||
|
||||
if ttl:
|
||||
item["ttl"] = int(time.time()) + ttl
|
||||
|
||||
self._table.put_item(Item=item)
|
||||
|
||||
def delete(self, key: str) -> bool:
|
||||
"""Delete a key."""
|
||||
response = self._table.delete_item(
|
||||
Key={"pk": key},
|
||||
ReturnValues="ALL_OLD"
|
||||
)
|
||||
return "Attributes" in response
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
"""Check if a key exists."""
|
||||
response = self._table.get_item(
|
||||
Key={"pk": key},
|
||||
ProjectionExpression="pk,#t",
|
||||
ExpressionAttributeNames={"#t": "ttl"}
|
||||
)
|
||||
item = response.get("Item")
|
||||
if not item:
|
||||
return False
|
||||
if item.get("ttl") and item["ttl"] <= int(time.time()):
|
||||
return False
|
||||
return True
|
||||
|
||||
def keys(self, pattern: str = "*") -> List[str]:
|
||||
"""List keys matching pattern."""
|
||||
# DynamoDB scan is expensive, use with caution
|
||||
response = self._table.scan(ProjectionExpression="pk")
|
||||
keys = [item["pk"] for item in response.get("Items", [])]
|
||||
|
||||
if pattern != "*":
|
||||
import fnmatch
|
||||
keys = [k for k in keys if fnmatch.fnmatch(k, pattern)]
|
||||
|
||||
return keys
|
||||
|
||||
def ttl(self, key: str) -> Optional[int]:
|
||||
"""Get remaining TTL in seconds."""
|
||||
response = self._table.get_item(
|
||||
Key={"pk": key},
|
||||
ProjectionExpression="#t",
|
||||
ExpressionAttributeNames={"#t": "ttl"}
|
||||
)
|
||||
item = response.get("Item")
|
||||
if not item or "ttl" not in item:
|
||||
return None
|
||||
|
||||
remaining = item["ttl"] - int(time.time())
|
||||
if remaining <= 0:
|
||||
return None
|
||||
return remaining
|
||||
|
||||
def expire(self, key: str, ttl: int) -> bool:
|
||||
"""Set TTL on existing key."""
|
||||
try:
|
||||
self._table.update_item(
|
||||
Key={"pk": key},
|
||||
UpdateExpression="SET #t = :ttl",
|
||||
ExpressionAttributeNames={"#t": "ttl"},
|
||||
ExpressionAttributeValues={":ttl": int(time.time()) + ttl},
|
||||
ConditionExpression="attribute_exists(pk)"
|
||||
)
|
||||
return True
|
||||
except self._client.exceptions.ConditionalCheckFailedException:
|
||||
return False
|
||||
|
||||
def hget(self, key: str, field: str) -> Optional[Any]:
|
||||
"""Get a field from a hash."""
|
||||
value = self.get(key)
|
||||
if not isinstance(value, dict):
|
||||
return None
|
||||
return value.get(field)
|
||||
|
||||
def hset(self, key: str, field: str, value: Any) -> None:
|
||||
"""Set a field in a hash."""
|
||||
current = self.get(key)
|
||||
if not isinstance(current, dict):
|
||||
current = {}
|
||||
current[field] = value
|
||||
self.set(key, current)
|
||||
|
||||
def hgetall(self, key: str) -> Dict[str, Any]:
|
||||
"""Get all fields from a hash."""
|
||||
value = self.get(key)
|
||||
if not isinstance(value, dict):
|
||||
return {}
|
||||
return value
|
||||
|
||||
def hdel(self, key: str, *fields: str) -> int:
|
||||
"""Delete fields from a hash."""
|
||||
current = self.get(key)
|
||||
if not isinstance(current, dict):
|
||||
return 0
|
||||
count = 0
|
||||
for field in fields:
|
||||
if field in current:
|
||||
del current[field]
|
||||
count += 1
|
||||
if count > 0:
|
||||
self.set(key, current)
|
||||
return count
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the store."""
|
||||
pass # boto3 handles cleanup
|
||||
@@ -0,0 +1,175 @@
|
||||
"""
|
||||
Firestore implementation of StateStore.
|
||||
|
||||
Requires: google-cloud-firestore
|
||||
Install: pip install google-cloud-firestore
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import StateStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FirestoreStateStore(StateStore):
|
||||
"""
|
||||
Firestore-based state store.
|
||||
|
||||
Example:
|
||||
store = FirestoreStateStore(
|
||||
project="my-project",
|
||||
collection="praisonai_state"
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
project: Optional[str] = None,
|
||||
collection: str = "praisonai_state",
|
||||
credentials_path: Optional[str] = None,
|
||||
):
|
||||
try:
|
||||
from google.cloud import firestore
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"google-cloud-firestore is required for Firestore support. "
|
||||
"Install with: pip install google-cloud-firestore"
|
||||
)
|
||||
|
||||
if credentials_path:
|
||||
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = credentials_path
|
||||
|
||||
project = project or os.getenv("FIRESTORE_PROJECT")
|
||||
self._client = firestore.Client(project=project)
|
||||
self._collection = self._client.collection(collection)
|
||||
|
||||
logger.info(f"Connected to Firestore collection: {collection}")
|
||||
|
||||
def get(self, key: str) -> Optional[Any]:
|
||||
"""Get a value by key."""
|
||||
doc = self._collection.document(key).get()
|
||||
if not doc.exists:
|
||||
return None
|
||||
|
||||
data = doc.to_dict()
|
||||
|
||||
# Check TTL
|
||||
if data.get("expires_at") and data["expires_at"] <= time.time():
|
||||
self._collection.document(key).delete()
|
||||
return None
|
||||
|
||||
return data.get("value")
|
||||
|
||||
def set(
|
||||
self,
|
||||
key: str,
|
||||
value: Any,
|
||||
ttl: Optional[int] = None
|
||||
) -> None:
|
||||
"""Set a value with optional TTL."""
|
||||
data = {"value": value, "updated_at": time.time()}
|
||||
|
||||
if ttl:
|
||||
data["expires_at"] = time.time() + ttl
|
||||
|
||||
self._collection.document(key).set(data)
|
||||
|
||||
def delete(self, key: str) -> bool:
|
||||
"""Delete a key."""
|
||||
doc = self._collection.document(key)
|
||||
if doc.get().exists:
|
||||
doc.delete()
|
||||
return True
|
||||
return False
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
"""Check if a key exists."""
|
||||
doc = self._collection.document(key).get()
|
||||
if not doc.exists:
|
||||
return False
|
||||
|
||||
data = doc.to_dict()
|
||||
if data.get("expires_at") and data["expires_at"] <= time.time():
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def keys(self, pattern: str = "*") -> List[str]:
|
||||
"""List keys matching pattern."""
|
||||
docs = self._collection.stream()
|
||||
keys = [doc.id for doc in docs]
|
||||
|
||||
if pattern != "*":
|
||||
import fnmatch
|
||||
keys = [k for k in keys if fnmatch.fnmatch(k, pattern)]
|
||||
|
||||
return keys
|
||||
|
||||
def ttl(self, key: str) -> Optional[int]:
|
||||
"""Get remaining TTL in seconds."""
|
||||
doc = self._collection.document(key).get()
|
||||
if not doc.exists:
|
||||
return None
|
||||
|
||||
data = doc.to_dict()
|
||||
if "expires_at" not in data:
|
||||
return None
|
||||
|
||||
remaining = data["expires_at"] - time.time()
|
||||
if remaining <= 0:
|
||||
return None
|
||||
return int(remaining)
|
||||
|
||||
def expire(self, key: str, ttl: int) -> bool:
|
||||
"""Set TTL on existing key."""
|
||||
doc = self._collection.document(key)
|
||||
if not doc.get().exists:
|
||||
return False
|
||||
|
||||
doc.update({"expires_at": time.time() + ttl})
|
||||
return True
|
||||
|
||||
def hget(self, key: str, field: str) -> Optional[Any]:
|
||||
"""Get a field from a hash."""
|
||||
value = self.get(key)
|
||||
if not isinstance(value, dict):
|
||||
return None
|
||||
return value.get(field)
|
||||
|
||||
def hset(self, key: str, field: str, value: Any) -> None:
|
||||
"""Set a field in a hash."""
|
||||
current = self.get(key)
|
||||
if not isinstance(current, dict):
|
||||
current = {}
|
||||
current[field] = value
|
||||
self.set(key, current)
|
||||
|
||||
def hgetall(self, key: str) -> Dict[str, Any]:
|
||||
"""Get all fields from a hash."""
|
||||
value = self.get(key)
|
||||
if not isinstance(value, dict):
|
||||
return {}
|
||||
return value
|
||||
|
||||
def hdel(self, key: str, *fields: str) -> int:
|
||||
"""Delete fields from a hash."""
|
||||
current = self.get(key)
|
||||
if not isinstance(current, dict):
|
||||
return 0
|
||||
count = 0
|
||||
for field in fields:
|
||||
if field in current:
|
||||
del current[field]
|
||||
count += 1
|
||||
if count > 0:
|
||||
self.set(key, current)
|
||||
return count
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the store."""
|
||||
pass # Firestore client handles cleanup
|
||||
@@ -0,0 +1,155 @@
|
||||
"""
|
||||
Google Cloud Storage implementation of StateStore.
|
||||
|
||||
Requires: google-cloud-storage
|
||||
Install: pip install google-cloud-storage
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import StateStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GCSStateStore(StateStore):
|
||||
"""
|
||||
Google Cloud Storage-based state store.
|
||||
|
||||
Stores state as JSON objects in GCS buckets.
|
||||
|
||||
Example:
|
||||
store = GCSStateStore(
|
||||
bucket_name="my-praisonai-state",
|
||||
prefix="state/"
|
||||
)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
bucket_name: str,
|
||||
prefix: str = "praisonai_state/",
|
||||
project: Optional[str] = None,
|
||||
credentials_path: Optional[str] = None,
|
||||
):
|
||||
"""
|
||||
Initialize GCS state store.
|
||||
|
||||
Args:
|
||||
bucket_name: GCS bucket name
|
||||
prefix: Object prefix for state keys
|
||||
project: GCP project ID (optional)
|
||||
credentials_path: Path to service account JSON (optional)
|
||||
"""
|
||||
try:
|
||||
from google.cloud import storage
|
||||
if credentials_path:
|
||||
self._client = storage.Client.from_service_account_json(
|
||||
credentials_path, project=project
|
||||
)
|
||||
else:
|
||||
self._client = storage.Client(project=project)
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"google-cloud-storage is required for GCS support. "
|
||||
"Install with: pip install google-cloud-storage"
|
||||
)
|
||||
|
||||
self.bucket_name = bucket_name
|
||||
self.prefix = prefix
|
||||
self._bucket = self._client.bucket(bucket_name)
|
||||
|
||||
def _key_to_path(self, key: str) -> str:
|
||||
"""Convert key to GCS object path."""
|
||||
return f"{self.prefix}{key}.json"
|
||||
|
||||
def get(self, key: str) -> Optional[Dict[str, Any]]:
|
||||
"""Get state by key."""
|
||||
try:
|
||||
blob = self._bucket.blob(self._key_to_path(key))
|
||||
if blob.exists():
|
||||
data = json.loads(blob.download_as_text())
|
||||
# Check TTL
|
||||
if "_ttl_expires" in data:
|
||||
if time.time() > data["_ttl_expires"]:
|
||||
self.delete(key)
|
||||
return None
|
||||
del data["_ttl_expires"]
|
||||
return data
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting state {key}: {e}")
|
||||
return None
|
||||
|
||||
def set(self, key: str, value: Dict[str, Any], ttl: Optional[int] = None) -> bool:
|
||||
"""Set state by key with optional TTL in seconds."""
|
||||
try:
|
||||
data = value.copy()
|
||||
if ttl:
|
||||
data["_ttl_expires"] = time.time() + ttl
|
||||
|
||||
blob = self._bucket.blob(self._key_to_path(key))
|
||||
blob.upload_from_string(
|
||||
json.dumps(data, default=str),
|
||||
content_type="application/json"
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Error setting state {key}: {e}")
|
||||
return False
|
||||
|
||||
def delete(self, key: str) -> bool:
|
||||
"""Delete state by key."""
|
||||
try:
|
||||
blob = self._bucket.blob(self._key_to_path(key))
|
||||
if blob.exists():
|
||||
blob.delete()
|
||||
return True
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Error deleting state {key}: {e}")
|
||||
return False
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
"""Check if key exists."""
|
||||
try:
|
||||
blob = self._bucket.blob(self._key_to_path(key))
|
||||
return blob.exists()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def list_keys(self, prefix: Optional[str] = None) -> List[str]:
|
||||
"""List all keys with optional prefix filter."""
|
||||
try:
|
||||
search_prefix = self.prefix
|
||||
if prefix:
|
||||
search_prefix = f"{self.prefix}{prefix}"
|
||||
|
||||
blobs = self._client.list_blobs(self.bucket_name, prefix=search_prefix)
|
||||
keys = []
|
||||
for blob in blobs:
|
||||
# Remove prefix and .json suffix
|
||||
key = blob.name[len(self.prefix):]
|
||||
if key.endswith(".json"):
|
||||
key = key[:-5]
|
||||
keys.append(key)
|
||||
return keys
|
||||
except Exception as e:
|
||||
logger.error(f"Error listing keys: {e}")
|
||||
return []
|
||||
|
||||
def clear(self, prefix: Optional[str] = None) -> int:
|
||||
"""Clear all keys with optional prefix filter."""
|
||||
keys = self.list_keys(prefix)
|
||||
count = 0
|
||||
for key in keys:
|
||||
if self.delete(key):
|
||||
count += 1
|
||||
return count
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the store."""
|
||||
self._client.close()
|
||||
@@ -0,0 +1,245 @@
|
||||
"""
|
||||
In-memory / JSON file implementation of StateStore.
|
||||
|
||||
Zero external dependencies - uses Python built-ins.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .base import StateStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MemoryStateStore(StateStore):
|
||||
"""
|
||||
In-memory state store with optional JSON file persistence.
|
||||
|
||||
Zero external dependencies - ideal for development and testing.
|
||||
|
||||
Example:
|
||||
# In-memory only
|
||||
store = MemoryStateStore()
|
||||
|
||||
# With file persistence
|
||||
store = MemoryStateStore(path="./state.json")
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
path: Optional[str] = None,
|
||||
auto_save: bool = True,
|
||||
save_interval: int = 60,
|
||||
):
|
||||
"""
|
||||
Initialize memory state store.
|
||||
|
||||
Args:
|
||||
path: Path to JSON file for persistence (None = in-memory only)
|
||||
auto_save: Auto-save to file periodically
|
||||
save_interval: Save interval in seconds
|
||||
"""
|
||||
self.path = path
|
||||
self.auto_save = auto_save
|
||||
self.save_interval = save_interval
|
||||
|
||||
self._data: Dict[str, Any] = {}
|
||||
self._ttls: Dict[str, float] = {} # key -> expiry timestamp
|
||||
self._lock = threading.RLock()
|
||||
self._last_save = time.time()
|
||||
|
||||
# Load from file if exists
|
||||
if path and os.path.exists(path):
|
||||
self._load()
|
||||
|
||||
logger.info(f"MemoryStateStore initialized (path={path})")
|
||||
|
||||
def _load(self) -> None:
|
||||
"""Load state from JSON file."""
|
||||
if not self.path:
|
||||
return
|
||||
|
||||
try:
|
||||
with open(self.path, "r") as f:
|
||||
saved = json.load(f)
|
||||
self._data = saved.get("data", {})
|
||||
self._ttls = saved.get("ttls", {})
|
||||
|
||||
# Clean expired keys
|
||||
now = time.time()
|
||||
expired = [k for k, exp in self._ttls.items() if exp <= now]
|
||||
for k in expired:
|
||||
self._data.pop(k, None)
|
||||
self._ttls.pop(k, None)
|
||||
|
||||
logger.debug(f"Loaded {len(self._data)} keys from {self.path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load state from {self.path}: {e}")
|
||||
|
||||
def _save(self) -> None:
|
||||
"""Save state to JSON file."""
|
||||
if not self.path:
|
||||
return
|
||||
|
||||
try:
|
||||
os.makedirs(os.path.dirname(self.path) or ".", exist_ok=True)
|
||||
with open(self.path, "w") as f:
|
||||
json.dump({"data": self._data, "ttls": self._ttls}, f)
|
||||
self._last_save = time.time()
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to save state to {self.path}: {e}")
|
||||
|
||||
def _maybe_save(self) -> None:
|
||||
"""Save if auto_save is enabled and interval has passed."""
|
||||
if self.auto_save and self.path:
|
||||
if time.time() - self._last_save >= self.save_interval:
|
||||
self._save()
|
||||
|
||||
def _check_ttl(self, key: str) -> bool:
|
||||
"""Check if key is expired. Returns True if valid, False if expired."""
|
||||
if key not in self._ttls:
|
||||
return True
|
||||
if self._ttls[key] <= time.time():
|
||||
self._data.pop(key, None)
|
||||
self._ttls.pop(key, None)
|
||||
return False
|
||||
return True
|
||||
|
||||
def get(self, key: str) -> Optional[Any]:
|
||||
"""Get a value by key."""
|
||||
with self._lock:
|
||||
if not self._check_ttl(key):
|
||||
return None
|
||||
return self._data.get(key)
|
||||
|
||||
def set(
|
||||
self,
|
||||
key: str,
|
||||
value: Any,
|
||||
ttl: Optional[int] = None
|
||||
) -> None:
|
||||
"""Set a value with optional TTL."""
|
||||
with self._lock:
|
||||
self._data[key] = value
|
||||
if ttl:
|
||||
self._ttls[key] = time.time() + ttl
|
||||
elif key in self._ttls:
|
||||
del self._ttls[key]
|
||||
self._maybe_save()
|
||||
|
||||
def delete(self, key: str) -> bool:
|
||||
"""Delete a key."""
|
||||
with self._lock:
|
||||
existed = key in self._data
|
||||
self._data.pop(key, None)
|
||||
self._ttls.pop(key, None)
|
||||
self._maybe_save()
|
||||
return existed
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
"""Check if a key exists."""
|
||||
with self._lock:
|
||||
if not self._check_ttl(key):
|
||||
return False
|
||||
return key in self._data
|
||||
|
||||
def keys(self, pattern: str = "*") -> List[str]:
|
||||
"""List keys matching pattern."""
|
||||
import fnmatch
|
||||
|
||||
with self._lock:
|
||||
# Clean expired keys first
|
||||
now = time.time()
|
||||
expired = [k for k, exp in self._ttls.items() if exp <= now]
|
||||
for k in expired:
|
||||
self._data.pop(k, None)
|
||||
self._ttls.pop(k, None)
|
||||
|
||||
if pattern == "*":
|
||||
return list(self._data.keys())
|
||||
return [k for k in self._data.keys() if fnmatch.fnmatch(k, pattern)]
|
||||
|
||||
def ttl(self, key: str) -> Optional[int]:
|
||||
"""Get remaining TTL in seconds."""
|
||||
with self._lock:
|
||||
if key not in self._ttls:
|
||||
return None
|
||||
remaining = self._ttls[key] - time.time()
|
||||
if remaining <= 0:
|
||||
self._data.pop(key, None)
|
||||
self._ttls.pop(key, None)
|
||||
return None
|
||||
return int(remaining)
|
||||
|
||||
def expire(self, key: str, ttl: int) -> bool:
|
||||
"""Set TTL on existing key."""
|
||||
with self._lock:
|
||||
if key not in self._data:
|
||||
return False
|
||||
self._ttls[key] = time.time() + ttl
|
||||
self._maybe_save()
|
||||
return True
|
||||
|
||||
def hget(self, key: str, field: str) -> Optional[Any]:
|
||||
"""Get a field from a hash."""
|
||||
with self._lock:
|
||||
if not self._check_ttl(key):
|
||||
return None
|
||||
data = self._data.get(key)
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
return data.get(field)
|
||||
|
||||
def hset(self, key: str, field: str, value: Any) -> None:
|
||||
"""Set a field in a hash."""
|
||||
with self._lock:
|
||||
if key not in self._data or not isinstance(self._data[key], dict):
|
||||
self._data[key] = {}
|
||||
self._data[key][field] = value
|
||||
self._maybe_save()
|
||||
|
||||
def hgetall(self, key: str) -> Dict[str, Any]:
|
||||
"""Get all fields from a hash."""
|
||||
with self._lock:
|
||||
if not self._check_ttl(key):
|
||||
return {}
|
||||
data = self._data.get(key)
|
||||
if not isinstance(data, dict):
|
||||
return {}
|
||||
return dict(data)
|
||||
|
||||
def hdel(self, key: str, *fields: str) -> int:
|
||||
"""Delete fields from a hash."""
|
||||
with self._lock:
|
||||
if key not in self._data or not isinstance(self._data[key], dict):
|
||||
return 0
|
||||
count = 0
|
||||
for field in fields:
|
||||
if field in self._data[key]:
|
||||
del self._data[key][field]
|
||||
count += 1
|
||||
self._maybe_save()
|
||||
return count
|
||||
|
||||
def flush(self) -> None:
|
||||
"""Force save to file."""
|
||||
with self._lock:
|
||||
self._save()
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear all data."""
|
||||
with self._lock:
|
||||
self._data.clear()
|
||||
self._ttls.clear()
|
||||
self._maybe_save()
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the store and save."""
|
||||
with self._lock:
|
||||
if self.path:
|
||||
self._save()
|
||||
Loaded 100 of 108 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user