Compare commits

...
1 Commits
Author SHA1 Message Date
Mervin Praison d00ea3e3c3 Release v2.3.11 2025-12-16 10:45:36 +00:00
52 changed files with 8277 additions and 16 deletions

No files matched your search

+67
View File
@@ -1098,6 +1098,73 @@ agent = Agent(
| Prompt Caching | OpenAI (auto), Anthropic, Bedrock, Deepseek |
## MCP (Model Context Protocol)
PraisonAI supports MCP Protocol Revision 2025-11-25 with multiple transports.
### MCP Client (Consume MCP Servers)
```python
from praisonaiagents import Agent, MCP
# stdio - Local NPX/Python servers
agent = Agent(tools=MCP("npx @modelcontextprotocol/server-memory"))
# Streamable HTTP - Production servers
agent = Agent(tools=MCP("https://api.example.com/mcp"))
# WebSocket - Real-time bidirectional
agent = Agent(tools=MCP("wss://api.example.com/mcp", auth_token="token"))
# SSE (Legacy) - Backward compatibility
agent = Agent(tools=MCP("http://localhost:8080/sse"))
# With environment variables
agent = Agent(
tools=MCP(
command="npx",
args=["-y", "@modelcontextprotocol/server-brave-search"],
env={"BRAVE_API_KEY": "your-key"}
)
)
```
### MCP Server (Expose Tools as MCP Server)
Expose your Python functions as MCP tools for Claude Desktop, Cursor, and other MCP clients:
```python
from praisonaiagents.mcp import ToolsMCPServer
def search_web(query: str, max_results: int = 5) -> dict:
"""Search the web for information."""
return {"results": [f"Result for {query}"]}
def calculate(expression: str) -> dict:
"""Evaluate a mathematical expression."""
return {"result": eval(expression)}
# Create and run MCP server
server = ToolsMCPServer(name="my-tools")
server.register_tools([search_web, calculate])
server.run() # stdio for Claude Desktop
# server.run_sse(host="0.0.0.0", port=8080) # SSE for web clients
```
### MCP Features
| Feature | Description |
|---------|-------------|
| **Session Management** | Automatic `Mcp-Session-Id` handling |
| **Protocol Versioning** | `Mcp-Protocol-Version` header |
| **Resumability** | SSE stream recovery via `Last-Event-ID` |
| **Security** | Origin validation, DNS rebinding prevention |
| **WebSocket** | Auto-reconnect with exponential backoff |
## Development:
Below is used for development only.
+1 -1
View File
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
# Install Python packages (using latest versions)
RUN pip install --no-cache-dir \
flask \
"praisonai>=2.3.10" \
"praisonai>=2.3.11" \
"praisonai[api]" \
gunicorn \
markdown
+1 -1
View File
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
# Install Python packages (using latest versions)
RUN pip install --no-cache-dir \
praisonai_tools \
"praisonai>=2.3.10" \
"praisonai>=2.3.11" \
"praisonai[chat]" \
"embedchain[github,youtube]"
+1 -1
View File
@@ -20,7 +20,7 @@ RUN mkdir -p /root/.praison
# Install Python packages (using latest versions)
RUN pip install --no-cache-dir \
praisonai_tools \
"praisonai>=2.3.10" \
"praisonai>=2.3.11" \
"praisonai[ui]" \
"praisonai[chat]" \
"praisonai[realtime]" \
+1 -1
View File
@@ -16,7 +16,7 @@ RUN mkdir -p /root/.praison
# Install Python packages (using latest versions)
RUN pip install --no-cache-dir \
praisonai_tools \
"praisonai>=2.3.10" \
"praisonai>=2.3.11" \
"praisonai[ui]" \
"praisonai[crewai]"
@@ -0,0 +1,89 @@
"""
Fast Context with Agent - Basic Example
This is the simplest way to use Fast Context with an Agent.
Just set fast_context=True and the agent will automatically
have access to rapid code search capabilities.
Fast Context provides:
- 10-20x faster code search than traditional methods
- Parallel execution (8 concurrent searches)
- Automatic caching for repeated queries
- Seamless integration with Agent workflows
"""
from praisonaiagents import Agent
def main():
print("=" * 60)
print("Agent with Fast Context - Basic Example")
print("=" * 60)
# Create an agent with Fast Context enabled
# This is the simplest configuration
agent = Agent(
name="CodeAssistant",
instructions="You are a helpful code assistant.",
fast_context=True, # Enable Fast Context
fast_context_path=".", # Search in current directory
verbose=False
)
print("\n✓ Agent created with Fast Context enabled")
print(f" - fast_context_enabled: {agent.fast_context_enabled}")
print(f" - fast_context_path: {agent.fast_context_path}")
print(f" - fast_context_model: {agent.fast_context_model}")
# Use delegate_to_fast_context to search code
print("\n" + "-" * 60)
print("Searching for 'class Agent' in the codebase...")
print("-" * 60)
context = agent.delegate_to_fast_context("class Agent")
if context:
print(f"\n✓ Found relevant code!")
print(f" Context length: {len(context)} characters")
print("\nPreview of found context:")
print("-" * 40)
# Show first 500 characters
preview = context[:500]
print(preview)
if len(context) > 500:
print("...")
else:
print("\n✗ No relevant code found")
# Another search example
print("\n" + "-" * 60)
print("Searching for 'def chat' in the codebase...")
print("-" * 60)
context = agent.delegate_to_fast_context("def chat")
if context:
print(f"\n✓ Found relevant code!")
print(f" Context length: {len(context)} characters")
else:
print("\n✗ No relevant code found")
# Access FastContext directly for more control
print("\n" + "-" * 60)
print("Direct FastContext access for advanced usage...")
print("-" * 60)
fc = agent.fast_context
if fc:
result = fc.search("import")
print(f"\n✓ Direct search for 'import':")
print(f" Files found: {result.total_files}")
print(f" Search time: {result.search_time_ms}ms")
print(f" From cache: {result.from_cache}")
print("\n" + "=" * 60)
print("Fast Context makes code search instant for your agents!")
print("=" * 60)
if __name__ == "__main__":
main()
+84
View File
@@ -0,0 +1,84 @@
"""
Example 1: Basic Fast Context Usage
This example demonstrates the basic usage of Fast Context for rapid code search.
Fast Context is 10-20x faster than traditional sequential search because it:
- Executes up to 8 tool calls in parallel
- Limits search to 4 turns maximum
- Uses a lightweight model optimized for search
"""
from praisonaiagents.context.fast import FastContext
import time
def main():
print("=" * 70)
print("Fast Context - Basic Usage Example")
print("=" * 70)
# Create FastContext instance pointing to the praisonai-agents codebase
fc = FastContext(
workspace_path="/Users/praison/praisonai-package/src/praisonai-agents"
)
# Example 1: Simple pattern search
print("\n1. Simple Pattern Search")
print("-" * 40)
start = time.perf_counter()
result = fc.search("def chat")
elapsed = (time.perf_counter() - start) * 1000
print(f" Query: 'def chat'")
print(f" Files found: {result.total_files}")
print(f" Search time: {result.search_time_ms}ms (total: {elapsed:.0f}ms)")
print(f" Turns used: {result.turns_used}")
print(f" Tool calls: {result.total_tool_calls}")
if result.files:
print("\n Top matches:")
for f in result.files[:3]:
print(f" - {f.path}")
for lr in f.line_ranges[:2]:
print(f" Lines {lr.start}-{lr.end}")
# Example 2: Search for class definitions
print("\n2. Search for Class Definitions")
print("-" * 40)
result = fc.search("class Agent")
print(f" Query: 'class Agent'")
print(f" Files found: {result.total_files}")
print(f" Search time: {result.search_time_ms}ms")
if result.files:
print("\n Files containing 'class Agent':")
for f in result.files[:5]:
print(f" - {f.path} (relevance: {f.relevance_score:.2f})")
# Example 3: Search for imports
print("\n3. Search for Specific Imports")
print("-" * 40)
result = fc.search("from typing import")
print(f" Query: 'from typing import'")
print(f" Files found: {result.total_files}")
print(f" Search time: {result.search_time_ms}ms")
# Example 4: Get formatted context string
print("\n4. Get Formatted Context for Agent")
print("-" * 40)
context = fc.get_context_for_agent("authentication", max_files=3)
print(f" Context length: {len(context)} characters")
print(f" Preview:")
preview = context[:300].replace('\n', '\n ')
print(f" {preview}...")
print("\n" + "=" * 70)
print("Fast Context provides rapid, parallel code search!")
print("=" * 70)
if __name__ == "__main__":
main()
@@ -0,0 +1,113 @@
"""
Example 2: Parallel vs Sequential Search Comparison
This example demonstrates the speed advantage of Fast Context's parallel
execution compared to traditional sequential search.
Key insight: Fast Context executes up to 8 searches simultaneously,
making it 5-10x faster for complex queries.
"""
import time
import os
from praisonaiagents.context.fast import FastContext
from praisonaiagents.context.fast.search_tools import grep_search, glob_search
WORKSPACE = "/Users/praison/praisonai-package/src/praisonai-agents"
def sequential_search(queries: list) -> dict:
"""Traditional sequential search - one query at a time."""
results = {}
for query in queries:
results[query] = grep_search(
search_path=WORKSPACE,
pattern=query,
max_results=20
)
return results
def parallel_search_with_fast_context(queries: list) -> dict:
"""Fast Context parallel search - multiple queries at once."""
fc = FastContext(workspace_path=WORKSPACE)
results = {}
for query in queries:
result = fc.search(query)
results[query] = result
return results
def main():
print("=" * 70)
print("Parallel vs Sequential Search Comparison")
print("=" * 70)
# Define multiple search queries
queries = [
"def __init__",
"class Agent",
"async def",
"import logging",
"from typing",
]
print(f"\nSearching for {len(queries)} patterns in the codebase...")
print(f"Workspace: {WORKSPACE}")
# Sequential search
print("\n1. Sequential Search (Traditional)")
print("-" * 40)
start = time.perf_counter()
seq_results = sequential_search(queries)
seq_time = (time.perf_counter() - start) * 1000
total_seq_matches = sum(len(r) for r in seq_results.values())
print(f" Total matches: {total_seq_matches}")
print(f" Time: {seq_time:.0f}ms")
# Fast Context search (with caching disabled for fair comparison)
print("\n2. Fast Context Search (Parallel)")
print("-" * 40)
fc = FastContext(workspace_path=WORKSPACE, cache_enabled=False)
start = time.perf_counter()
fc_results = {}
for query in queries:
fc_results[query] = fc.search(query)
fc_time = (time.perf_counter() - start) * 1000
total_fc_matches = sum(r.total_files for r in fc_results.values())
print(f" Total files found: {total_fc_matches}")
print(f" Time: {fc_time:.0f}ms")
# Comparison
print("\n3. Comparison")
print("-" * 40)
speedup = seq_time / fc_time if fc_time > 0 else float('inf')
print(f" Sequential: {seq_time:.0f}ms")
print(f" Fast Context: {fc_time:.0f}ms")
print(f" Speedup: {speedup:.1f}x faster")
# Detailed results
print("\n4. Results by Query")
print("-" * 40)
for query in queries:
seq_count = len(seq_results[query])
fc_count = fc_results[query].total_files
fc_ms = fc_results[query].search_time_ms
print(f" '{query}':")
print(f" Sequential: {seq_count} matches")
print(f" Fast Context: {fc_count} files in {fc_ms}ms")
print("\n" + "=" * 70)
print(f"Fast Context is {speedup:.1f}x faster due to parallel execution!")
print("=" * 70)
if __name__ == "__main__":
main()
+132
View File
@@ -0,0 +1,132 @@
"""
Example 3: Caching Performance Demonstration
This example shows how Fast Context's caching dramatically improves
performance for repeated queries.
Key benefits:
- First query: Full search execution
- Subsequent queries: Instant cache hit (0ms search time)
- Configurable TTL (time-to-live)
"""
import time
from praisonaiagents.context.fast import FastContext
WORKSPACE = "/Users/praison/praisonai-package/src/praisonai-agents"
def main():
print("=" * 70)
print("Fast Context Caching Performance")
print("=" * 70)
# Create FastContext with caching enabled
fc = FastContext(
workspace_path=WORKSPACE,
cache_enabled=True,
cache_ttl=300 # 5 minutes
)
query = "def execute"
print(f"\nQuery: '{query}'")
print(f"Cache TTL: 300 seconds")
# First search - cache miss
print("\n1. First Search (Cache Miss)")
print("-" * 40)
start = time.perf_counter()
result1 = fc.search(query)
elapsed1 = (time.perf_counter() - start) * 1000
print(f" Files found: {result1.total_files}")
print(f" From cache: {result1.from_cache}")
print(f" Search time: {result1.search_time_ms}ms")
print(f" Total time: {elapsed1:.1f}ms")
# Second search - cache hit
print("\n2. Second Search (Cache Hit)")
print("-" * 40)
start = time.perf_counter()
result2 = fc.search(query)
elapsed2 = (time.perf_counter() - start) * 1000
print(f" Files found: {result2.total_files}")
print(f" From cache: {result2.from_cache}")
print(f" Search time: {result2.search_time_ms}ms")
print(f" Total time: {elapsed2:.1f}ms")
# Third search - still cached
print("\n3. Third Search (Still Cached)")
print("-" * 40)
start = time.perf_counter()
result3 = fc.search(query)
elapsed3 = (time.perf_counter() - start) * 1000
print(f" Files found: {result3.total_files}")
print(f" From cache: {result3.from_cache}")
print(f" Total time: {elapsed3:.1f}ms")
# Clear cache and search again
print("\n4. After Cache Clear")
print("-" * 40)
fc.clear_cache()
print(" Cache cleared!")
start = time.perf_counter()
result4 = fc.search(query)
elapsed4 = (time.perf_counter() - start) * 1000
print(f" Files found: {result4.total_files}")
print(f" From cache: {result4.from_cache}")
print(f" Search time: {result4.search_time_ms}ms")
print(f" Total time: {elapsed4:.1f}ms")
# Performance comparison
print("\n5. Performance Summary")
print("-" * 40)
cache_speedup = elapsed1 / elapsed2 if elapsed2 > 0 else float('inf')
print(f" First search (no cache): {elapsed1:.1f}ms")
print(f" Cached searches: ~{elapsed2:.1f}ms")
print(f" Cache speedup: {cache_speedup:.0f}x faster")
# Multiple different queries
print("\n6. Multiple Query Caching")
print("-" * 40)
queries = ["class Agent", "async def", "import os", "def __init__"]
# First pass - populate cache
print(" First pass (populating cache):")
start = time.perf_counter()
for q in queries:
fc.search(q)
first_pass = (time.perf_counter() - start) * 1000
print(f" Time: {first_pass:.0f}ms")
# Second pass - all cached
print(" Second pass (all cached):")
start = time.perf_counter()
for q in queries:
r = fc.search(q)
assert r.from_cache, f"Expected cache hit for '{q}'"
second_pass = (time.perf_counter() - start) * 1000
print(f" Time: {second_pass:.1f}ms")
multi_speedup = first_pass / second_pass if second_pass > 0 else float('inf')
print(f" Speedup: {multi_speedup:.0f}x faster")
print("\n" + "=" * 70)
print("Caching provides instant results for repeated queries!")
print("=" * 70)
if __name__ == "__main__":
main()
+112
View File
@@ -0,0 +1,112 @@
"""
Example 4: Agent Integration with Fast Context
This example shows how to use Fast Context integrated with the Agent class
for automatic code search delegation.
Benefits:
- Seamless integration with existing Agent workflows
- Automatic context injection for code-related queries
- Configurable via Agent parameters or environment variables
"""
from praisonaiagents import Agent
WORKSPACE = "/Users/praison/praisonai-package/src/praisonai-agents"
def main():
print("=" * 70)
print("Agent Integration with Fast Context")
print("=" * 70)
# Create an agent with Fast Context enabled
print("\n1. Creating Agent with Fast Context")
print("-" * 40)
agent = Agent(
name="CodeAssistant",
instructions="You are a helpful code assistant that can search and analyze code.",
fast_context=True,
fast_context_path=WORKSPACE,
fast_context_model="gpt-4o-mini",
fast_context_max_turns=4,
fast_context_parallelism=8,
fast_context_timeout=30.0,
verbose=False
)
print(f" Agent name: {agent.name}")
print(f" Fast Context enabled: {agent.fast_context_enabled}")
print(f" Fast Context path: {agent.fast_context_path}")
print(f" Fast Context model: {agent.fast_context_model}")
print(f" Max turns: {agent.fast_context_max_turns}")
print(f" Parallelism: {agent.fast_context_parallelism}")
# Use delegate_to_fast_context for code search
print("\n2. Delegating Code Search to Fast Context")
print("-" * 40)
queries = [
"find the Agent class definition",
"search for authentication handlers",
"locate memory management code",
]
for query in queries:
print(f"\n Query: '{query}'")
context = agent.delegate_to_fast_context(query)
if context:
# Show preview of context
lines = context.split('\n')
preview_lines = lines[:5]
print(f" Found context ({len(context)} chars):")
for line in preview_lines:
print(f" {line[:60]}{'...' if len(line) > 60 else ''}")
if len(lines) > 5:
print(f" ... and {len(lines) - 5} more lines")
else:
print(" No relevant code found")
# Access the FastContext instance directly
print("\n3. Direct FastContext Access")
print("-" * 40)
fc = agent.fast_context
if fc:
result = fc.search("def chat")
print(f" Direct search for 'def chat':")
print(f" Files found: {result.total_files}")
print(f" Search time: {result.search_time_ms}ms")
for f in result.files[:3]:
print(f" - {f.path}")
# Show configuration options
print("\n4. Configuration Options")
print("-" * 40)
print(" Agent parameters:")
print(" - fast_context=True # Enable Fast Context")
print(" - fast_context_path='.' # Workspace path")
print(" - fast_context_model='gpt-4o-mini' # Search model")
print(" - fast_context_max_turns=4 # Max search turns")
print(" - fast_context_parallelism=8 # Parallel calls")
print(" - fast_context_timeout=30.0 # Timeout (seconds)")
print("\n Environment variables:")
print(" - FAST_CONTEXT_MODEL")
print(" - FAST_CONTEXT_MAX_TURNS")
print(" - FAST_CONTEXT_PARALLELISM")
print(" - FAST_CONTEXT_TIMEOUT")
print(" - FAST_CONTEXT_CACHE")
print(" - FAST_CONTEXT_CACHE_TTL")
print("\n" + "=" * 70)
print("Fast Context integrates seamlessly with Agent for code search!")
print("=" * 70)
if __name__ == "__main__":
main()
@@ -0,0 +1,121 @@
"""
Example 5: File and Symbol Indexing
This example demonstrates the file and symbol indexing capabilities
that power Fast Context's rapid search.
Features:
- FileIndexer: Index file metadata for fast lookups
- SymbolIndexer: Extract code symbols (functions, classes, imports)
- Support for Python, JavaScript, TypeScript, Go, Rust, Java
"""
import time
from praisonaiagents.context.fast.indexer import (
FileIndexer,
SymbolIndexer,
SymbolType
)
WORKSPACE = "/Users/praison/praisonai-package/src/praisonai-agents"
def main():
print("=" * 70)
print("File and Symbol Indexing")
print("=" * 70)
# File Indexer
print("\n1. File Indexer")
print("-" * 40)
start = time.perf_counter()
file_indexer = FileIndexer(workspace_path=WORKSPACE)
file_count = file_indexer.index()
file_time = (time.perf_counter() - start) * 1000
print(f" Indexed {file_count} files in {file_time:.0f}ms")
stats = file_indexer.get_stats()
print(f" Total size: {stats['total_size_mb']} MB")
print(f" Extensions: {list(stats['extensions'].keys())[:5]}")
# Find files by pattern
print("\n Finding files by pattern:")
py_files = file_indexer.find_by_pattern("**/*.py")
print(f" - Python files (*.py): {len(py_files)}")
agent_files = file_indexer.find_by_pattern("**/agent*.py")
print(f" - Agent files (agent*.py): {len(agent_files)}")
# Find files by name
print("\n Finding files by name:")
init_files = file_indexer.find_by_name("__init__.py", exact=True)
print(f" - __init__.py files: {len(init_files)}")
agent_matches = file_indexer.find_by_name("agent")
print(f" - Files containing 'agent': {len(agent_matches)}")
# Symbol Indexer
print("\n2. Symbol Indexer")
print("-" * 40)
start = time.perf_counter()
symbol_indexer = SymbolIndexer(workspace_path=WORKSPACE)
symbol_count = symbol_indexer.index()
symbol_time = (time.perf_counter() - start) * 1000
print(f" Indexed {symbol_count} symbols in {symbol_time:.0f}ms")
stats = symbol_indexer.get_stats()
print(f" Files indexed: {stats['total_files']}")
print(f" Symbols by type:")
for sym_type, count in stats['by_type'].items():
print(f" - {sym_type}: {count}")
# Find symbols by name
print("\n Finding symbols by name:")
agent_symbols = symbol_indexer.find_by_name("Agent", exact=True)
print(f" - 'Agent' (exact): {len(agent_symbols)} symbols")
for sym in agent_symbols[:3]:
print(f" {sym.symbol_type.value}: {sym.name} in {sym.file_path}:{sym.line_number}")
chat_symbols = symbol_indexer.find_by_name("chat")
print(f" - 'chat' (partial): {len(chat_symbols)} symbols")
for sym in chat_symbols[:3]:
print(f" {sym.symbol_type.value}: {sym.name} in {sym.file_path}:{sym.line_number}")
# Find symbols by type
print("\n Finding symbols by type:")
classes = symbol_indexer.find_by_type(SymbolType.CLASS)
print(f" - Classes: {len(classes)}")
for cls in classes[:5]:
print(f" class {cls.name} in {cls.file_path}:{cls.line_number}")
functions = symbol_indexer.find_by_type(SymbolType.FUNCTION)
print(f" - Functions: {len(functions)}")
methods = symbol_indexer.find_by_type(SymbolType.METHOD)
print(f" - Methods: {len(methods)}")
imports = symbol_indexer.find_by_type(SymbolType.IMPORT)
print(f" - Imports: {len(imports)}")
# Performance comparison
print("\n3. Performance Summary")
print("-" * 40)
print(f" File indexing: {file_time:.0f}ms for {file_count} files")
print(f" Symbol indexing: {symbol_time:.0f}ms for {symbol_count} symbols")
print(f" Files per second: {file_count / (file_time / 1000):.0f}")
print(f" Symbols per second: {symbol_count / (symbol_time / 1000):.0f}")
print("\n" + "=" * 70)
print("Indexing enables instant lookups for files and symbols!")
print("=" * 70)
if __name__ == "__main__":
main()
+163
View File
@@ -0,0 +1,163 @@
"""
Example 6: Context Injection for Agents
This example demonstrates how Fast Context results can be formatted
and injected into agent prompts for better code understanding.
Features:
- Multiple format styles (markdown, xml, plain)
- Token budget management
- Precision over recall (avoid context pollution)
"""
from praisonaiagents.context.fast import FastContext
from praisonaiagents.context.fast.context_injector import (
ContextInjector,
InjectionConfig,
inject_fast_context
)
WORKSPACE = "/Users/praison/praisonai-package/src/praisonai-agents"
def main():
print("=" * 70)
print("Context Injection for Agents")
print("=" * 70)
# Search for code
fc = FastContext(workspace_path=WORKSPACE)
result = fc.search("class Agent")
print(f"\nSearch results: {result.total_files} files found")
# Markdown format (default)
print("\n1. Markdown Format")
print("-" * 40)
config_md = InjectionConfig(
max_tokens=2000,
max_files=3,
max_lines_per_file=20,
format_style="markdown",
include_line_numbers=True,
include_file_content=True,
prioritize_precision=True
)
injector_md = ContextInjector(config_md)
md_context = injector_md.format_context(result)
print(f" Format: Markdown")
print(f" Length: {len(md_context)} chars (~{len(md_context)//4} tokens)")
print(f" Preview:")
for line in md_context.split('\n')[:10]:
print(f" {line[:70]}")
print(" ...")
# XML format
print("\n2. XML Format")
print("-" * 40)
config_xml = InjectionConfig(
max_tokens=2000,
max_files=3,
format_style="xml",
prioritize_precision=True
)
injector_xml = ContextInjector(config_xml)
xml_context = injector_xml.format_context(result)
print(f" Format: XML")
print(f" Length: {len(xml_context)} chars")
print(f" Preview:")
for line in xml_context.split('\n')[:10]:
print(f" {line[:70]}")
print(" ...")
# Plain text format
print("\n3. Plain Text Format")
print("-" * 40)
config_plain = InjectionConfig(
max_tokens=2000,
max_files=3,
format_style="plain"
)
injector_plain = ContextInjector(config_plain)
plain_context = injector_plain.format_context(result)
print(f" Format: Plain")
print(f" Length: {len(plain_context)} chars")
print(f" Preview:")
for line in plain_context.split('\n')[:8]:
print(f" {line[:70]}")
print(" ...")
# Inject into prompts
print("\n4. Injecting into Prompts")
print("-" * 40)
system_prompt = "You are a helpful code assistant."
user_message = "Explain how the Agent class works."
injected = inject_fast_context(
result=result,
system_prompt=system_prompt,
user_message=user_message,
max_tokens=1500,
format_style="markdown"
)
print(f" Original system prompt: {len(system_prompt)} chars")
print(f" Injected system prompt: {len(injected['system_prompt'])} chars")
print(f" User message unchanged: {injected['user_message'] == user_message}")
# Token budget management
print("\n5. Token Budget Management")
print("-" * 40)
budgets = [500, 1000, 2000, 4000]
for budget in budgets:
config = InjectionConfig(max_tokens=budget, max_files=10)
injector = ContextInjector(config)
context = injector.format_context(result)
estimated_tokens = len(context) // 4
print(f" Budget: {budget} tokens -> {len(context)} chars (~{estimated_tokens} tokens)")
# Precision mode
print("\n6. Precision vs Recall")
print("-" * 40)
# High precision (fewer, more relevant results)
config_precision = InjectionConfig(
max_tokens=2000,
max_files=5,
prioritize_precision=True
)
injector_precision = ContextInjector(config_precision)
precision_context = injector_precision.format_context(result)
# Lower precision (more results)
config_recall = InjectionConfig(
max_tokens=2000,
max_files=10,
prioritize_precision=False
)
injector_recall = ContextInjector(config_recall)
recall_context = injector_recall.format_context(result)
print(f" High precision mode: {len(precision_context)} chars")
print(f" High recall mode: {len(recall_context)} chars")
print(f" Precision mode is more focused for better agent understanding")
print("\n" + "=" * 70)
print("Context injection formats search results for optimal agent use!")
print("=" * 70)
if __name__ == "__main__":
main()
+213
View File
@@ -0,0 +1,213 @@
"""
Example 7: Search Tools Deep Dive
This example demonstrates the individual search tools that power Fast Context:
- grep_search: Pattern-based search with regex support
- glob_search: File pattern matching
- read_file: Read file contents with line ranges
- list_directory: Directory listing with filtering
"""
import time
from praisonaiagents.context.fast.search_tools import (
grep_search,
glob_search,
read_file,
list_directory
)
WORKSPACE = "/Users/praison/praisonai-package/src/praisonai-agents"
def main():
print("=" * 70)
print("Search Tools Deep Dive")
print("=" * 70)
# grep_search
print("\n1. grep_search - Pattern Search")
print("-" * 40)
# Simple pattern
start = time.perf_counter()
results = grep_search(
search_path=WORKSPACE,
pattern="def __init__",
max_results=10
)
elapsed = (time.perf_counter() - start) * 1000
print(f" Pattern: 'def __init__'")
print(f" Matches: {len(results)}")
print(f" Time: {elapsed:.0f}ms")
if results:
print(f" First match: {results[0]['path']}:{results[0]['line_number']}")
# Regex pattern
results = grep_search(
search_path=WORKSPACE,
pattern=r"class \w+Agent",
is_regex=True,
max_results=10
)
print(f"\n Regex: 'class \\w+Agent'")
print(f" Matches: {len(results)}")
for r in results[:3]:
print(f" - {r['path']}:{r['line_number']}: {r['content'][:50]}...")
# Case sensitive
results_sensitive = grep_search(
search_path=WORKSPACE,
pattern="Agent",
case_sensitive=True,
max_results=50
)
results_insensitive = grep_search(
search_path=WORKSPACE,
pattern="agent",
case_sensitive=False,
max_results=50
)
print(f"\n Case sensitive 'Agent': {len(results_sensitive)} matches")
print(f" Case insensitive 'agent': {len(results_insensitive)} matches")
# With context lines
results = grep_search(
search_path=WORKSPACE,
pattern="class Agent:",
context_lines=2,
max_results=3
)
print(f"\n With 2 context lines:")
if results:
print(f" {results[0]['path']}:")
if results[0].get('context'):
for line in results[0]['context'].split('\n')[:5]:
print(f" {line[:60]}")
# glob_search
print("\n2. glob_search - File Pattern Matching")
print("-" * 40)
# Find Python files
start = time.perf_counter()
results = glob_search(
search_path=WORKSPACE,
pattern="**/*.py",
max_results=100
)
elapsed = (time.perf_counter() - start) * 1000
print(f" Pattern: '**/*.py'")
print(f" Files found: {len(results)}")
print(f" Time: {elapsed:.0f}ms")
# Find specific files
results = glob_search(
search_path=WORKSPACE,
pattern="**/agent*.py",
max_results=20
)
print(f"\n Pattern: '**/agent*.py'")
print(f" Files found: {len(results)}")
for r in results[:5]:
print(f" - {r['path']} ({r['size']} bytes)")
# Find test files
results = glob_search(
search_path=WORKSPACE,
pattern="**/test_*.py",
max_results=50
)
print(f"\n Pattern: '**/test_*.py'")
print(f" Test files: {len(results)}")
# read_file
print("\n3. read_file - Read File Contents")
print("-" * 40)
# Read entire file
result = read_file(
filepath=f"{WORKSPACE}/praisonaiagents/__init__.py",
max_lines=20
)
print(f" File: praisonaiagents/__init__.py")
print(f" Success: {result['success']}")
print(f" Lines: {result.get('total_lines', 'N/A')}")
if result['success']:
lines = result['content'].split('\n')[:5]
print(f" Preview:")
for line in lines:
print(f" {line[:60]}")
# Read specific line range
result = read_file(
filepath=f"{WORKSPACE}/praisonaiagents/agent/agent.py",
start_line=1,
end_line=20
)
print(f"\n File: agent/agent.py (lines 1-20)")
print(f" Success: {result['success']}")
if result['success']:
lines = result['content'].split('\n')[:5]
print(f" Preview:")
for line in lines:
print(f" {line[:60]}")
# Read with context
result = read_file(
filepath=f"{WORKSPACE}/praisonaiagents/agent/agent.py",
start_line=100,
end_line=105,
context_lines=3
)
print(f"\n File: agent/agent.py (lines 100-105 with 3 context lines)")
print(f" Success: {result['success']}")
# list_directory
print("\n4. list_directory - Directory Listing")
print("-" * 40)
# List root
result = list_directory(
dir_path=WORKSPACE,
recursive=False,
max_entries=20
)
print(f" Directory: {WORKSPACE}")
print(f" Success: {result['success']}")
print(f" Entries: {len(result.get('entries', []))}")
dirs = [e for e in result.get('entries', []) if e.get('is_dir')]
files = [e for e in result.get('entries', []) if not e.get('is_dir')]
print(f" Directories: {len(dirs)}")
print(f" Files: {len(files)}")
# List recursive with depth limit
result = list_directory(
dir_path=f"{WORKSPACE}/praisonaiagents",
recursive=True,
max_depth=2,
max_entries=50
)
print(f"\n Directory: praisonaiagents (recursive, depth=2)")
print(f" Entries: {len(result.get('entries', []))}")
# Show directory structure
print(f" Structure:")
for entry in result.get('entries', [])[:10]:
prefix = "📁" if entry.get('is_dir') else "📄"
print(f" {prefix} {entry['name']}")
print("\n" + "=" * 70)
print("These search tools power Fast Context's rapid code search!")
print("=" * 70)
if __name__ == "__main__":
main()
@@ -0,0 +1,227 @@
"""
Example 8: Comprehensive Benchmark
This example provides a comprehensive benchmark comparing Fast Context
performance against traditional search approaches.
Metrics measured:
- Search latency
- Throughput (queries per second)
- Cache effectiveness
- Parallel execution speedup
"""
import time
import statistics
from praisonaiagents.context.fast import FastContext
from praisonaiagents.context.fast.search_tools import grep_search
from praisonaiagents.context.fast.indexer import FileIndexer, SymbolIndexer
WORKSPACE = "/Users/praison/praisonai-package/src/praisonai-agents"
def benchmark_search_latency():
"""Benchmark search latency."""
print("\n1. Search Latency Benchmark")
print("-" * 50)
fc = FastContext(workspace_path=WORKSPACE, cache_enabled=False)
queries = [
"def __init__",
"class Agent",
"import logging",
"async def",
"from typing import",
]
# Warm up
fc.search("warmup")
# Benchmark each query
latencies = []
for query in queries:
times = []
for _ in range(3): # 3 runs per query
start = time.perf_counter()
fc.search(query)
elapsed = (time.perf_counter() - start) * 1000
times.append(elapsed)
avg = statistics.mean(times)
latencies.append(avg)
print(f" '{query}': {avg:.1f}ms (min: {min(times):.1f}, max: {max(times):.1f})")
print(f"\n Average latency: {statistics.mean(latencies):.1f}ms")
print(f" Median latency: {statistics.median(latencies):.1f}ms")
print(f" P95 latency: {sorted(latencies)[int(len(latencies)*0.95)]:.1f}ms")
def benchmark_throughput():
"""Benchmark queries per second."""
print("\n2. Throughput Benchmark")
print("-" * 50)
fc = FastContext(workspace_path=WORKSPACE, cache_enabled=True)
queries = ["def", "class", "import", "return", "if", "for", "while", "try"]
# Cold run (no cache)
fc.clear_cache()
start = time.perf_counter()
for query in queries:
fc.search(query)
cold_time = time.perf_counter() - start
cold_qps = len(queries) / cold_time
print(f" Cold (no cache):")
print(f" Queries: {len(queries)}")
print(f" Time: {cold_time*1000:.0f}ms")
print(f" Throughput: {cold_qps:.1f} queries/second")
# Warm run (cached)
start = time.perf_counter()
for _ in range(10): # 10 iterations
for query in queries:
fc.search(query)
warm_time = time.perf_counter() - start
warm_qps = (len(queries) * 10) / warm_time
print(f"\n Warm (cached):")
print(f" Queries: {len(queries) * 10}")
print(f" Time: {warm_time*1000:.0f}ms")
print(f" Throughput: {warm_qps:.0f} queries/second")
print(f"\n Cache speedup: {warm_qps/cold_qps:.0f}x")
def benchmark_cache_effectiveness():
"""Benchmark cache hit rate and speedup."""
print("\n3. Cache Effectiveness Benchmark")
print("-" * 50)
fc = FastContext(workspace_path=WORKSPACE, cache_enabled=True)
fc.clear_cache()
queries = ["Agent", "chat", "memory", "tool", "execute"]
# First pass - cache misses
miss_times = []
for query in queries:
start = time.perf_counter()
result = fc.search(query)
elapsed = (time.perf_counter() - start) * 1000
miss_times.append(elapsed)
assert not result.from_cache
# Second pass - cache hits
hit_times = []
for query in queries:
start = time.perf_counter()
result = fc.search(query)
elapsed = (time.perf_counter() - start) * 1000
hit_times.append(elapsed)
assert result.from_cache
avg_miss = statistics.mean(miss_times)
avg_hit = statistics.mean(hit_times)
print(f" Cache miss (first query): {avg_miss:.1f}ms avg")
print(f" Cache hit (repeat query): {avg_hit:.2f}ms avg")
print(f" Cache speedup: {avg_miss/avg_hit:.0f}x faster")
print(f" Time saved per cached query: {avg_miss - avg_hit:.1f}ms")
def benchmark_parallel_execution():
"""Benchmark parallel vs sequential execution."""
print("\n4. Parallel Execution Benchmark")
print("-" * 50)
queries = ["def", "class", "import", "return", "async"]
# Sequential (one at a time)
start = time.perf_counter()
for query in queries:
grep_search(WORKSPACE, query, max_results=20)
seq_time = (time.perf_counter() - start) * 1000
# Parallel (Fast Context)
fc = FastContext(workspace_path=WORKSPACE, cache_enabled=False)
start = time.perf_counter()
for query in queries:
fc.search(query)
par_time = (time.perf_counter() - start) * 1000
print(f" Sequential: {seq_time:.0f}ms")
print(f" Parallel (Fast Context): {par_time:.0f}ms")
print(f" Speedup: {seq_time/par_time:.1f}x")
def benchmark_indexing():
"""Benchmark indexing performance."""
print("\n5. Indexing Performance Benchmark")
print("-" * 50)
# File indexing
start = time.perf_counter()
file_indexer = FileIndexer(workspace_path=WORKSPACE)
file_count = file_indexer.index()
file_time = (time.perf_counter() - start) * 1000
print(f" File Indexer:")
print(f" Files: {file_count}")
print(f" Time: {file_time:.0f}ms")
print(f" Rate: {file_count/(file_time/1000):.0f} files/second")
# Symbol indexing
start = time.perf_counter()
symbol_indexer = SymbolIndexer(workspace_path=WORKSPACE)
symbol_count = symbol_indexer.index()
symbol_time = (time.perf_counter() - start) * 1000
print(f"\n Symbol Indexer:")
print(f" Symbols: {symbol_count}")
print(f" Time: {symbol_time:.0f}ms")
print(f" Rate: {symbol_count/(symbol_time/1000):.0f} symbols/second")
# Lookup performance
start = time.perf_counter()
for _ in range(100):
file_indexer.find_by_pattern("**/*.py")
lookup_time = (time.perf_counter() - start) * 1000
print(f"\n Lookup Performance:")
print(f" 100 pattern lookups: {lookup_time:.1f}ms")
print(f" Per lookup: {lookup_time/100:.2f}ms")
def main():
print("=" * 70)
print("Fast Context Comprehensive Benchmark")
print("=" * 70)
print(f"\nWorkspace: {WORKSPACE}")
benchmark_search_latency()
benchmark_throughput()
benchmark_cache_effectiveness()
benchmark_parallel_execution()
benchmark_indexing()
print("\n" + "=" * 70)
print("Summary")
print("=" * 70)
print("""
Fast Context provides:
✓ Sub-100ms search latency for most queries
✓ 100+ queries/second with caching
✓ 100x+ speedup for cached queries
✓ 2-5x speedup from parallel execution
✓ 1000+ files/second indexing rate
These benchmarks demonstrate why Fast Context is ideal for
rapid code search in AI agent workflows.
""")
if __name__ == "__main__":
main()
@@ -0,0 +1,82 @@
"""MCP Schema Converter Example
Demonstrates how to convert Python functions to MCP-compatible schemas.
Usage:
python mcp-schema-converter.py
"""
from typing import List, Dict, Any, Optional
from praisonaiagents.mcp import function_to_mcp_schema, get_tool_metadata
# Example functions with various type annotations
def simple_search(query: str) -> str:
"""Search for information."""
return f"Results for {query}"
def advanced_search(
query: str,
max_results: int = 10,
include_images: bool = False,
filters: Optional[Dict[str, str]] = None
) -> Dict[str, Any]:
"""Advanced search with multiple options.
Args:
query: The search query
max_results: Maximum results to return
include_images: Whether to include images
filters: Optional filters to apply
"""
return {}
def process_items(items: List[str], batch_size: int = 5) -> List[Dict[str, Any]]:
"""Process a list of items.
Args:
items: List of items to process
batch_size: Size of each batch
"""
return []
async def async_fetch(url: str, timeout: float = 30.0) -> Dict[str, Any]:
"""Fetch data from URL asynchronously.
Args:
url: The URL to fetch
timeout: Request timeout in seconds
"""
return {}
def main():
functions = [simple_search, advanced_search, process_items, async_fetch]
print("=" * 60)
print("MCP Schema Converter Demo")
print("=" * 60)
for func in functions:
print(f"\n### Function: {func.__name__}")
# Get metadata
metadata = get_tool_metadata(func)
print(f"Name: {metadata['name']}")
print(f"Description: {metadata['description']}")
# Get full schema
schema = function_to_mcp_schema(func)
print("\nMCP Schema:")
import json
print(json.dumps(schema, indent=2))
print("-" * 40)
if __name__ == "__main__":
main()
@@ -0,0 +1,79 @@
"""MCP Tools Server with Async Functions
Example showing how to expose async Python functions as MCP tools.
Usage:
python mcp-tools-server-async.py
"""
import asyncio
from praisonaiagents.mcp import ToolsMCPServer
async def async_search(query: str, delay: float = 0.1) -> dict:
"""Async search function.
Args:
query: The search query
delay: Simulated delay in seconds
"""
await asyncio.sleep(delay) # Simulate async operation
return {
"query": query,
"results": [f"Async result for {query}"]
}
async def async_fetch_url(url: str) -> dict:
"""Fetch content from a URL asynchronously.
Args:
url: The URL to fetch
"""
# Mock implementation - use aiohttp in production
await asyncio.sleep(0.1)
return {
"url": url,
"status": 200,
"content": f"Content from {url}"
}
async def async_process_data(data: str, iterations: int = 3) -> dict:
"""Process data asynchronously with multiple iterations.
Args:
data: The data to process
iterations: Number of processing iterations
"""
results = []
for i in range(iterations):
await asyncio.sleep(0.05)
results.append(f"Iteration {i+1}: processed {data}")
return {
"input": data,
"iterations": iterations,
"results": results
}
# Sync function can also be mixed with async
def sync_calculate(a: int, b: int) -> dict:
"""Synchronous calculation (can be mixed with async tools)."""
return {"sum": a + b, "product": a * b}
if __name__ == "__main__":
server = ToolsMCPServer(name="async-tools-server")
# Register both async and sync tools
server.register_tools([
async_search,
async_fetch_url,
async_process_data,
sync_calculate
])
print(f"Registered tools: {server.get_tool_names()}")
server.run()
@@ -0,0 +1,27 @@
"""MCP Server with Built-in PraisonAI Tools
Expose PraisonAI's built-in tools as an MCP server.
Usage:
# Set required API keys first
export TAVILY_API_KEY=your_key
python mcp-tools-server-builtin.py
"""
from praisonaiagents.mcp import launch_tools_mcp_server
if __name__ == "__main__":
# Expose built-in PraisonAI tools as MCP server
# This allows external MCP clients to use these tools
launch_tools_mcp_server(
tool_names=[
"tavily_search", # Web search via Tavily
"duckduckgo_search", # Web search via DuckDuckGo
# Add more built-in tools as needed
],
name="praisonai-builtin-tools",
transport="stdio" # Use "sse" for web clients
)
@@ -0,0 +1,25 @@
"""Simple MCP Tools Server Example
Minimal example of exposing Python functions as MCP tools.
Usage:
python mcp-tools-server-simple.py
"""
from praisonaiagents.mcp import ToolsMCPServer
def search(query: str) -> dict:
"""Search for information."""
return {"query": query, "results": [f"Result for {query}"]}
def add(a: int, b: int) -> dict:
"""Add two numbers."""
return {"result": a + b}
if __name__ == "__main__":
server = ToolsMCPServer(name="simple-tools")
server.register_tools([search, add])
server.run()
@@ -0,0 +1,65 @@
"""MCP Tools Server with SSE Transport
Expose Python functions as MCP tools via Server-Sent Events (SSE).
This is useful for web-based clients and remote access.
Usage:
python mcp-tools-server-sse.py
# Then connect from another agent:
# agent = Agent(tools=MCP("http://localhost:8080/sse"))
"""
from praisonaiagents.mcp import ToolsMCPServer
def search_web(query: str, max_results: int = 5) -> dict:
"""Search the web for information.
Args:
query: The search query
max_results: Maximum results to return
"""
return {
"query": query,
"results": [
{"title": f"Result {i+1}", "url": f"https://example.com/{i+1}"}
for i in range(max_results)
]
}
def calculate(expression: str) -> dict:
"""Evaluate a mathematical expression.
Args:
expression: Math expression (e.g., "2 + 2 * 3")
"""
try:
allowed = set("0123456789+-*/.() ")
if not all(c in allowed for c in expression):
return {"error": "Invalid characters"}
return {"expression": expression, "result": eval(expression)}
except Exception as e:
return {"error": str(e)}
def get_time() -> dict:
"""Get the current time."""
from datetime import datetime
now = datetime.now()
return {
"time": now.strftime("%H:%M:%S"),
"date": now.strftime("%Y-%m-%d"),
"timestamp": now.isoformat()
}
if __name__ == "__main__":
server = ToolsMCPServer(name="sse-tools-server")
server.register_tools([search_web, calculate, get_time])
print("Starting SSE MCP Server...")
print("Connect with: MCP('http://localhost:8080/sse')")
server.run_sse(host="0.0.0.0", port=8080)
+140
View File
@@ -0,0 +1,140 @@
"""MCP Tools Server Example
Expose Python functions as MCP tools for external clients like Claude Desktop, Cursor, etc.
Usage:
# Run as stdio server (for Claude Desktop)
python mcp-tools-server.py
# Run as SSE server (for web clients)
python mcp-tools-server.py --sse --port 8080
"""
import argparse
from typing import List, Dict, Any
# Define your custom tools as regular Python functions
# Type hints and docstrings are used to generate MCP schemas
def search_web(query: str, max_results: int = 5) -> Dict[str, Any]:
"""Search the web for information.
Args:
query: The search query string
max_results: Maximum number of results to return (default: 5)
Returns:
Search results with titles and URLs
"""
# Mock implementation - replace with actual search logic
return {
"query": query,
"results": [
{"title": f"Result {i+1} for '{query}'", "url": f"https://example.com/{i+1}"}
for i in range(max_results)
]
}
def calculate(expression: str) -> Dict[str, Any]:
"""Evaluate a mathematical expression.
Args:
expression: A mathematical expression to evaluate (e.g., "2 + 2 * 3")
Returns:
The result of the calculation
"""
try:
# Safe evaluation of mathematical expressions
allowed_chars = set("0123456789+-*/.() ")
if not all(c in allowed_chars for c in expression):
return {"error": "Invalid characters in expression"}
result = eval(expression) # Use safer alternative in production
return {"expression": expression, "result": result}
except Exception as e:
return {"error": str(e)}
def get_weather(city: str, units: str = "celsius") -> Dict[str, Any]:
"""Get current weather for a city.
Args:
city: Name of the city
units: Temperature units - "celsius" or "fahrenheit" (default: celsius)
Returns:
Weather information for the city
"""
# Mock implementation - replace with actual weather API
return {
"city": city,
"temperature": 22 if units == "celsius" else 72,
"units": units,
"condition": "Sunny",
"humidity": 45
}
def list_files(directory: str = ".", pattern: str = "*") -> List[str]:
"""List files in a directory.
Args:
directory: Directory path to list (default: current directory)
pattern: Glob pattern to filter files (default: *)
Returns:
List of file names matching the pattern
"""
import glob
import os
try:
full_pattern = os.path.join(directory, pattern)
files = glob.glob(full_pattern)
return [os.path.basename(f) for f in files]
except Exception as e:
return [f"Error: {str(e)}"]
def main():
parser = argparse.ArgumentParser(description="Run PraisonAI Tools as MCP Server")
parser.add_argument("--sse", action="store_true", help="Use SSE transport instead of stdio")
parser.add_argument("--port", type=int, default=8080, help="Port for SSE server (default: 8080)")
parser.add_argument("--host", type=str, default="0.0.0.0", help="Host for SSE server")
parser.add_argument("--debug", action="store_true", help="Enable debug logging")
args = parser.parse_args()
from praisonaiagents.mcp import ToolsMCPServer
# Create server with custom name
server = ToolsMCPServer(
name="praisonai-custom-tools",
debug=args.debug
)
# Register all tools
server.register_tools([
search_web,
calculate,
get_weather,
list_files
])
# Print registered tools
print(f"📦 Registered {len(server.tools)} tools:")
for name in server.get_tool_names():
print(f" - {name}")
print()
# Run the server
if args.sse:
server.run_sse(host=args.host, port=args.port)
else:
server.run_stdio()
if __name__ == "__main__":
main()
@@ -0,0 +1,287 @@
"""
Fast Context Usage Examples.
This file demonstrates how to use the Fast Context feature for rapid code search.
Fast Context is inspired by Windsurf's SWE-grep approach:
- Parallel tool execution (up to 8 concurrent calls)
- Limited turns (max 4) for fast response
- Lightweight model for search operations
- Result caching for repeated queries
"""
# Example 1: Basic FastContext Usage
# ==================================
def example_basic_search():
"""Basic search using FastContext."""
from praisonaiagents.context.fast import FastContext
# Create FastContext instance
fc = FastContext(workspace_path=".")
# Search for a pattern
result = fc.search("def authenticate")
print(f"Found {result.total_files} files")
print(f"Search time: {result.search_time_ms}ms")
# Print results
for file_match in result.files[:5]:
print(f" - {file_match.path}")
for lr in file_match.line_ranges:
print(f" Lines {lr.start}-{lr.end}")
# Example 2: Search with Patterns
# ===============================
def example_search_with_patterns():
"""Search with include/exclude patterns."""
from praisonaiagents.context.fast import FastContext
fc = FastContext(workspace_path=".")
# Search only in Python files
result = fc.search(
"class Agent",
include_patterns=["**/*.py"],
exclude_patterns=["**/tests/**", "**/test_*.py"]
)
print(f"Found {result.total_files} files matching 'class Agent'")
for f in result.files:
print(f" - {f.path}")
# Example 3: Get Context for Agent
# ================================
def example_get_context_for_agent():
"""Get formatted context string for an agent."""
from praisonaiagents.context.fast import FastContext
fc = FastContext(workspace_path=".")
# Get context formatted for injection into an agent
context = fc.get_context_for_agent(
"authentication handlers",
max_files=5,
max_lines_per_file=50
)
print("Context for agent:")
print(context[:500] + "..." if len(context) > 500 else context)
# Example 4: Agent with FastContext Integration
# =============================================
def example_agent_with_fast_context():
"""Use FastContext with an Agent."""
from praisonaiagents import Agent
# Create agent with FastContext enabled
agent = Agent(
name="CodeAssistant",
instructions="You are a helpful code assistant.",
fast_context=True,
fast_context_path=".", # Current directory
fast_context_model="gpt-4o-mini",
fast_context_max_turns=4,
fast_context_parallelism=8
)
# Use delegate_to_fast_context for code search
context = agent.delegate_to_fast_context("find authentication code")
if context:
print("Found relevant code context:")
print(context[:300] + "...")
else:
print("No relevant code found")
# Example 5: File Indexer
# =======================
def example_file_indexer():
"""Use FileIndexer for fast file lookups."""
from praisonaiagents.context.fast.indexer import FileIndexer
# Create and run indexer
indexer = FileIndexer(workspace_path=".")
count = indexer.index()
print(f"Indexed {count} files")
# Find files by pattern
py_files = indexer.find_by_pattern("**/*.py")
print(f"Found {len(py_files)} Python files")
# Find files by extension
js_files = indexer.find_by_extension(".js")
print(f"Found {len(js_files)} JavaScript files")
# Get stats
stats = indexer.get_stats()
print(f"Total size: {stats['total_size_mb']} MB")
# Example 6: Symbol Indexer
# =========================
def example_symbol_indexer():
"""Use SymbolIndexer to find code symbols."""
from praisonaiagents.context.fast.indexer import SymbolIndexer, SymbolType
# Create and run indexer
indexer = SymbolIndexer(workspace_path=".")
count = indexer.index()
print(f"Indexed {count} symbols")
# Find symbols by name
agent_symbols = indexer.find_by_name("Agent")
print(f"Found {len(agent_symbols)} symbols containing 'Agent'")
# Find all classes
classes = indexer.find_by_type(SymbolType.CLASS)
print(f"Found {len(classes)} classes")
# Get stats
stats = indexer.get_stats()
print(f"Symbols by type: {stats['by_type']}")
# Example 7: Context Injector
# ===========================
def example_context_injector():
"""Use ContextInjector to format results for agents."""
from praisonaiagents.context.fast import FastContext
from praisonaiagents.context.fast.context_injector import (
ContextInjector,
InjectionConfig
)
# Search for code
fc = FastContext(workspace_path=".")
result = fc.search("def main")
# Configure injection
config = InjectionConfig(
max_tokens=2000,
max_files=5,
format_style="markdown", # or "xml", "plain"
prioritize_precision=True
)
# Inject into prompts
injector = ContextInjector(config)
formatted = injector.format_context(result)
print("Formatted context:")
print(formatted[:500] + "..." if len(formatted) > 500 else formatted)
# Example 8: Environment Variable Configuration
# =============================================
def example_env_config():
"""Configure FastContext via environment variables."""
import os
# Set environment variables (usually done in shell or .env file)
os.environ["FAST_CONTEXT_MODEL"] = "gpt-4o-mini"
os.environ["FAST_CONTEXT_MAX_TURNS"] = "4"
os.environ["FAST_CONTEXT_PARALLELISM"] = "8"
os.environ["FAST_CONTEXT_TIMEOUT"] = "30.0"
os.environ["FAST_CONTEXT_CACHE"] = "true"
os.environ["FAST_CONTEXT_CACHE_TTL"] = "300"
from praisonaiagents.context.fast import FastContext
# FastContext will use environment variables as defaults
fc = FastContext(workspace_path=".")
print(f"Model: {fc.model}")
print(f"Max turns: {fc.max_turns}")
print(f"Max parallel: {fc.max_parallel}")
print(f"Cache enabled: {fc.cache_enabled}")
# Example 9: Caching
# ==================
def example_caching():
"""Demonstrate result caching."""
from praisonaiagents.context.fast import FastContext
fc = FastContext(
workspace_path=".",
cache_enabled=True,
cache_ttl=300 # 5 minutes
)
# First search - not cached
result1 = fc.search("def main")
print(f"First search: from_cache={result1.from_cache}, time={result1.search_time_ms}ms")
# Second search - cached
result2 = fc.search("def main")
print(f"Second search: from_cache={result2.from_cache}, time={result2.search_time_ms}ms")
# Clear cache
fc.clear_cache()
# Third search - not cached
result3 = fc.search("def main")
print(f"Third search: from_cache={result3.from_cache}, time={result3.search_time_ms}ms")
# Example 10: Context Manager
# ===========================
def example_context_manager():
"""Use FastContext as a context manager."""
from praisonaiagents.context.fast import FastContext
with FastContext(workspace_path=".") as fc:
result = fc.search("import")
print(f"Found {result.total_files} files with imports")
# Resources are automatically cleaned up
if __name__ == "__main__":
print("=" * 60)
print("Fast Context Examples")
print("=" * 60)
print("\n1. Basic Search")
print("-" * 40)
example_basic_search()
print("\n2. Search with Patterns")
print("-" * 40)
example_search_with_patterns()
print("\n3. Get Context for Agent")
print("-" * 40)
example_get_context_for_agent()
print("\n5. File Indexer")
print("-" * 40)
example_file_indexer()
print("\n9. Caching")
print("-" * 40)
example_caching()
print("\n10. Context Manager")
print("-" * 40)
example_context_manager()
print("\n" + "=" * 60)
print("All examples completed!")
@@ -56,6 +56,23 @@ from .session import Session
from .memory.memory import Memory
from .guardrails import GuardrailResult, LLMGuardrail
# Fast Context support (lazy loaded to avoid performance impact)
def __getattr__(name):
"""Lazy load FastContext to avoid impacting package load time."""
if name == "FastContext":
from praisonaiagents.context.fast import FastContext
return FastContext
elif name == "FastContextResult":
from praisonaiagents.context.fast import FastContextResult
return FastContextResult
elif name == "FileMatch":
from praisonaiagents.context.fast import FileMatch
return FileMatch
elif name == "LineRange":
from praisonaiagents.context.fast import LineRange
return LineRange
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
# Planning mode support
try:
from .planning import (
@@ -273,3 +290,11 @@ if _mcp_available:
if FlowDisplay is not None:
__all__.extend(['FlowDisplay', 'track_workflow'])
# Add FastContext exports (lazy loaded)
__all__.extend([
'FastContext',
'FastContextResult',
'FileMatch',
'LineRange'
])
@@ -242,7 +242,13 @@ class Agent:
plan_mode: bool = False,
planning: bool = False,
planning_tools: Optional[List[Any]] = None,
planning_reasoning: bool = False
planning_reasoning: bool = False,
fast_context: bool = False,
fast_context_path: Optional[str] = None,
fast_context_model: str = "gpt-4o-mini",
fast_context_max_turns: int = 4,
fast_context_parallelism: int = 8,
fast_context_timeout: float = 30.0
):
"""Initialize an Agent instance.
@@ -585,6 +591,15 @@ Your Goal: {self.goal}
self._knowledge_config = knowledge_config
self.knowledge = None # Will be initialized on first use
# Fast Context configuration (lazy loaded)
self.fast_context_enabled = fast_context
self.fast_context_path = fast_context_path or os.getcwd()
self.fast_context_model = fast_context_model
self.fast_context_max_turns = fast_context_max_turns
self.fast_context_parallelism = fast_context_parallelism
self.fast_context_timeout = fast_context_timeout
self._fast_context_instance = None # Lazy loaded
@property
def console(self):
"""Lazily initialize Rich Console only when needed."""
@@ -713,6 +728,57 @@ Your Goal: {self.goal}
return supports_prompt_caching(model_name)
@property
def fast_context(self):
"""Lazily initialize FastContext instance when needed.
Returns:
FastContext instance or None if not enabled
"""
if not self.fast_context_enabled:
return None
if self._fast_context_instance is None:
try:
from ..context.fast import FastContext
self._fast_context_instance = FastContext(
workspace_path=self.fast_context_path,
model=self.fast_context_model,
max_turns=self.fast_context_max_turns,
max_parallel=self.fast_context_parallelism,
timeout=self.fast_context_timeout,
verbose=self.verbose
)
except ImportError:
logging.warning("FastContext not available")
return None
return self._fast_context_instance
def delegate_to_fast_context(self, query: str) -> Optional[str]:
"""Delegate a code search query to FastContext subagent.
This method uses the FastContext subagent to rapidly search
the codebase and return relevant context.
Args:
query: Natural language search query
Returns:
Formatted context string or None if FastContext not available
"""
if not self.fast_context_enabled or self.fast_context is None:
return None
try:
result = self.fast_context.search(query)
if result.total_files > 0:
return self.fast_context.get_context_for_agent(query)
return None
except Exception as e:
logging.warning(f"FastContext search failed: {e}")
return None
def _init_rules_manager(self):
"""
Initialize RulesManager for persistent rules/instructions.
@@ -0,0 +1,30 @@
"""
Context module for PraisonAI Agents.
This module provides context retrieval capabilities including:
- Fast Context: Rapid parallel code search subagent
"""
# Lazy imports to avoid impacting package load time
def __getattr__(name):
"""Lazy import to avoid impacting package performance."""
if name == "FastContext":
from praisonaiagents.context.fast.fast_context import FastContext
return FastContext
elif name == "FastContextResult":
from praisonaiagents.context.fast.result import FastContextResult
return FastContextResult
elif name == "FileMatch":
from praisonaiagents.context.fast.result import FileMatch
return FileMatch
elif name == "LineRange":
from praisonaiagents.context.fast.result import LineRange
return LineRange
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
__all__ = [
"FastContext",
"FastContextResult",
"FileMatch",
"LineRange",
]
@@ -0,0 +1,40 @@
"""
Fast Context module for rapid parallel code search.
This module implements a specialized subagent that retrieves relevant code
from a codebase using parallel tool calls, similar to Windsurf's Fast Context.
Key features:
- Parallel tool execution (up to 8 concurrent calls)
- Limited serial turns (max 4) for fast response
- Restricted tool set (grep, glob, read) for safety
- Returns files + line ranges (no summarization)
"""
# Lazy imports to avoid impacting package load time
def __getattr__(name):
"""Lazy import to avoid impacting package performance."""
if name == "FastContext":
from praisonaiagents.context.fast.fast_context import FastContext
return FastContext
elif name == "FastContextResult":
from praisonaiagents.context.fast.result import FastContextResult
return FastContextResult
elif name == "FileMatch":
from praisonaiagents.context.fast.result import FileMatch
return FileMatch
elif name == "LineRange":
from praisonaiagents.context.fast.result import LineRange
return LineRange
elif name == "FastContextAgent":
from praisonaiagents.context.fast.fast_context_agent import FastContextAgent
return FastContextAgent
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
__all__ = [
"FastContext",
"FastContextResult",
"FileMatch",
"LineRange",
"FastContextAgent",
]
@@ -0,0 +1,383 @@
"""
Context Injector for Fast Context.
Injects FastContext results into the main agent with:
- Token budget management
- Context formatting for different models
- Precision over recall (avoid context pollution)
"""
import logging
from typing import Optional, List, Dict, Any
from dataclasses import dataclass
from praisonaiagents.context.fast.result import FastContextResult, FileMatch
logger = logging.getLogger(__name__)
@dataclass
class InjectionConfig:
"""Configuration for context injection.
Attributes:
max_tokens: Maximum tokens for injected context
max_files: Maximum files to include
max_lines_per_file: Maximum lines per file
include_line_numbers: Whether to include line numbers
include_file_content: Whether to include file content
format_style: Format style ('markdown', 'xml', 'plain')
prioritize_precision: If True, include fewer but more relevant results
"""
max_tokens: int = 4000
max_files: int = 10
max_lines_per_file: int = 100
include_line_numbers: bool = True
include_file_content: bool = True
format_style: str = "markdown"
prioritize_precision: bool = True
# Approximate tokens per character (conservative estimate)
chars_per_token: float = 4.0
class ContextInjector:
"""Injects FastContext results into agent context.
Handles:
- Token budget management
- Result formatting
- Context pollution prevention
"""
def __init__(self, config: Optional[InjectionConfig] = None):
"""Initialize context injector.
Args:
config: Injection configuration
"""
self.config = config or InjectionConfig()
def estimate_tokens(self, text: str) -> int:
"""Estimate token count for text.
Args:
text: Text to estimate
Returns:
Estimated token count
"""
return int(len(text) / self.config.chars_per_token)
def inject(
self,
result: FastContextResult,
system_prompt: Optional[str] = None,
user_message: Optional[str] = None
) -> Dict[str, str]:
"""Inject FastContext results into agent context.
Args:
result: FastContext search result
system_prompt: Original system prompt
user_message: Original user message
Returns:
Dict with 'system_prompt' and 'user_message' keys
"""
# Format the context
context_str = self.format_context(result)
# Inject into appropriate location
if system_prompt:
# Add to system prompt
injected_system = self._inject_into_system_prompt(system_prompt, context_str)
return {
"system_prompt": injected_system,
"user_message": user_message or ""
}
elif user_message:
# Add to user message
injected_user = self._inject_into_user_message(user_message, context_str)
return {
"system_prompt": "",
"user_message": injected_user
}
else:
return {
"system_prompt": context_str,
"user_message": ""
}
def format_context(self, result: FastContextResult) -> str:
"""Format FastContext result as context string.
Args:
result: FastContext search result
Returns:
Formatted context string
"""
if self.config.format_style == "markdown":
return self._format_markdown(result)
elif self.config.format_style == "xml":
return self._format_xml(result)
else:
return self._format_plain(result)
def _format_markdown(self, result: FastContextResult) -> str:
"""Format result as markdown.
Args:
result: FastContext search result
Returns:
Markdown formatted string
"""
if not result.files:
return ""
lines = ["# Relevant Code Context"]
lines.append(f"*Found {result.total_files} file(s) in {result.search_time_ms}ms*\n")
token_budget = self.config.max_tokens
files_included = 0
# Sort by relevance
sorted_files = sorted(
result.files,
key=lambda f: f.relevance_score,
reverse=True
)
# Apply precision filter if enabled
if self.config.prioritize_precision:
sorted_files = [f for f in sorted_files if f.relevance_score >= 0.5]
for file_match in sorted_files:
if files_included >= self.config.max_files:
break
file_section = self._format_file_markdown(file_match)
section_tokens = self.estimate_tokens(file_section)
if section_tokens > token_budget:
# Try truncated version
file_section = self._format_file_markdown(file_match, truncate=True)
section_tokens = self.estimate_tokens(file_section)
if section_tokens > token_budget:
continue
lines.append(file_section)
token_budget -= section_tokens
files_included += 1
if files_included < result.total_files:
lines.append(f"\n*... and {result.total_files - files_included} more files*")
return "\n".join(lines)
def _format_file_markdown(
self,
file_match: FileMatch,
truncate: bool = False
) -> str:
"""Format a single file match as markdown.
Args:
file_match: File match to format
truncate: Whether to truncate content
Returns:
Markdown formatted string
"""
lines = [f"## `{file_match.path}`"]
if not self.config.include_file_content:
if file_match.line_ranges:
ranges = ", ".join(
f"L{lr.start}-{lr.end}" for lr in file_match.line_ranges[:3]
)
lines.append(f"Relevant lines: {ranges}")
return "\n".join(lines)
max_lines = self.config.max_lines_per_file
if truncate:
max_lines = max_lines // 2
lines_shown = 0
for lr in file_match.line_ranges:
if lines_shown >= max_lines:
break
if self.config.include_line_numbers:
lines.append(f"Lines {lr.start}-{lr.end}:")
if lr.content:
content = lr.content
content_lines = content.split("\n")
if len(content_lines) > max_lines - lines_shown:
content_lines = content_lines[:max_lines - lines_shown]
content = "\n".join(content_lines) + "\n..."
lines.append(f"```\n{content}\n```")
lines_shown += len(content_lines)
return "\n".join(lines)
def _format_xml(self, result: FastContextResult) -> str:
"""Format result as XML.
Args:
result: FastContext search result
Returns:
XML formatted string
"""
if not result.files:
return ""
lines = ["<code_context>"]
lines.append(f" <summary files=\"{result.total_files}\" search_time_ms=\"{result.search_time_ms}\"/>")
token_budget = self.config.max_tokens
files_included = 0
sorted_files = sorted(
result.files,
key=lambda f: f.relevance_score,
reverse=True
)
for file_match in sorted_files[:self.config.max_files]:
if files_included >= self.config.max_files:
break
file_xml = self._format_file_xml(file_match)
section_tokens = self.estimate_tokens(file_xml)
if section_tokens > token_budget:
continue
lines.append(file_xml)
token_budget -= section_tokens
files_included += 1
lines.append("</code_context>")
return "\n".join(lines)
def _format_file_xml(self, file_match: FileMatch) -> str:
"""Format a single file match as XML.
Args:
file_match: File match to format
Returns:
XML formatted string
"""
lines = [f' <file path="{file_match.path}" relevance="{file_match.relevance_score:.2f}">']
for lr in file_match.line_ranges[:3]:
lines.append(f' <lines start="{lr.start}" end="{lr.end}">')
if lr.content and self.config.include_file_content:
# Escape XML special characters
content = lr.content.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
lines.append(f" <![CDATA[{content}]]>")
lines.append(" </lines>")
lines.append(" </file>")
return "\n".join(lines)
def _format_plain(self, result: FastContextResult) -> str:
"""Format result as plain text.
Args:
result: FastContext search result
Returns:
Plain text formatted string
"""
if not result.files:
return ""
lines = [f"Code Context ({result.total_files} files):"]
for file_match in result.files[:self.config.max_files]:
lines.append(f"\nFile: {file_match.path}")
for lr in file_match.line_ranges[:3]:
lines.append(f" Lines {lr.start}-{lr.end}")
if lr.content and self.config.include_file_content:
for line in lr.content.split("\n")[:10]:
lines.append(f" {line}")
return "\n".join(lines)
def _inject_into_system_prompt(
self,
system_prompt: str,
context: str
) -> str:
"""Inject context into system prompt.
Args:
system_prompt: Original system prompt
context: Context to inject
Returns:
Modified system prompt
"""
if not context:
return system_prompt
# Add context section
return f"{system_prompt}\n\n{context}"
def _inject_into_user_message(
self,
user_message: str,
context: str
) -> str:
"""Inject context into user message.
Args:
user_message: Original user message
context: Context to inject
Returns:
Modified user message
"""
if not context:
return user_message
# Prepend context
return f"{context}\n\n---\n\n{user_message}"
def inject_fast_context(
result: FastContextResult,
system_prompt: Optional[str] = None,
user_message: Optional[str] = None,
max_tokens: int = 4000,
format_style: str = "markdown"
) -> Dict[str, str]:
"""Convenience function to inject FastContext results.
Args:
result: FastContext search result
system_prompt: Original system prompt
user_message: Original user message
max_tokens: Maximum tokens for context
format_style: Format style ('markdown', 'xml', 'plain')
Returns:
Dict with 'system_prompt' and 'user_message' keys
"""
config = InjectionConfig(
max_tokens=max_tokens,
format_style=format_style
)
injector = ContextInjector(config)
return injector.inject(result, system_prompt, user_message)
@@ -0,0 +1,384 @@
"""
FastContext - High-level API for fast parallel code search.
This module provides the main FastContext class that can be used
standalone or integrated with the main Agent class.
Example usage:
# Standalone usage
from praisonaiagents.context.fast import FastContext
fc = FastContext(workspace_path="/path/to/project")
result = fc.search("find authentication handlers")
print(result.to_context_string())
# With Agent integration
from praisonaiagents import Agent
agent = Agent(
name="CodeAssistant",
fast_context=True,
fast_context_path="/path/to/project"
)
"""
import os
import hashlib
import json
import logging
from typing import Optional, List, Dict, Any
from praisonaiagents.context.fast.result import FastContextResult, FileMatch, LineRange
from praisonaiagents.context.fast.fast_context_agent import FastContextAgent
logger = logging.getLogger(__name__)
class FastContext:
"""High-level API for fast parallel code search.
FastContext provides rapid code search capabilities using:
- Parallel tool execution (up to 8 concurrent calls)
- Limited turns (max 4) for fast response
- Optional LLM-powered intelligent search
- Result caching for repeated queries
Attributes:
workspace_path: Root directory for searches
model: LLM model for intelligent search
max_turns: Maximum search turns
max_parallel: Maximum parallel tool calls
cache_enabled: Whether to cache results
"""
# Environment variable names for configuration
ENV_MODEL = "FAST_CONTEXT_MODEL"
ENV_MAX_TURNS = "FAST_CONTEXT_MAX_TURNS"
ENV_MAX_PARALLEL = "FAST_CONTEXT_PARALLELISM"
ENV_TIMEOUT = "FAST_CONTEXT_TIMEOUT"
ENV_CACHE_ENABLED = "FAST_CONTEXT_CACHE"
ENV_CACHE_TTL = "FAST_CONTEXT_CACHE_TTL"
def __init__(
self,
workspace_path: Optional[str] = None,
model: Optional[str] = None,
max_turns: Optional[int] = None,
max_parallel: Optional[int] = None,
timeout: Optional[float] = None,
cache_enabled: Optional[bool] = None,
cache_ttl: Optional[int] = None,
verbose: bool = False
):
"""Initialize FastContext.
Configuration can be set via parameters or environment variables.
Parameters take precedence over environment variables.
Environment variables:
FAST_CONTEXT_MODEL: LLM model name (default: gpt-4o-mini)
FAST_CONTEXT_MAX_TURNS: Maximum search turns (default: 4)
FAST_CONTEXT_PARALLELISM: Max parallel calls (default: 8)
FAST_CONTEXT_TIMEOUT: Timeout in seconds (default: 30.0)
FAST_CONTEXT_CACHE: Enable caching (default: true)
FAST_CONTEXT_CACHE_TTL: Cache TTL in seconds (default: 300)
Args:
workspace_path: Root directory for searches (defaults to cwd)
model: LLM model for intelligent search
max_turns: Maximum search turns
max_parallel: Maximum parallel tool calls per turn
timeout: Timeout per tool call in seconds
cache_enabled: Whether to cache search results
cache_ttl: Cache time-to-live in seconds
verbose: If True, print debug information
"""
self.workspace_path = os.path.abspath(workspace_path or os.getcwd())
# Apply environment variable overrides with defaults
self.model = model or os.environ.get(self.ENV_MODEL, "gpt-4o-mini")
self.max_turns = max_turns if max_turns is not None else int(os.environ.get(self.ENV_MAX_TURNS, "4"))
self.max_parallel = max_parallel if max_parallel is not None else int(os.environ.get(self.ENV_MAX_PARALLEL, "8"))
self.timeout = timeout if timeout is not None else float(os.environ.get(self.ENV_TIMEOUT, "30.0"))
# Cache settings
cache_env = os.environ.get(self.ENV_CACHE_ENABLED, "true").lower()
self.cache_enabled = cache_enabled if cache_enabled is not None else cache_env in ("true", "1", "yes")
self.cache_ttl = cache_ttl if cache_ttl is not None else int(os.environ.get(self.ENV_CACHE_TTL, "300"))
self.verbose = verbose
# Cache storage
self._cache: Dict[str, Dict[str, Any]] = {}
# Agent instance (lazy loaded)
self._agent: Optional[FastContextAgent] = None
def _get_agent(self) -> FastContextAgent:
"""Get or create FastContextAgent instance."""
if self._agent is None:
self._agent = FastContextAgent(
workspace_path=self.workspace_path,
max_turns=self.max_turns,
max_parallel=self.max_parallel,
model=self.model,
timeout=self.timeout,
verbose=self.verbose
)
return self._agent
def _get_cache_key(self, query: str, **kwargs) -> str:
"""Generate cache key for a query."""
key_data = {
"query": query,
"workspace": self.workspace_path,
**kwargs
}
key_str = json.dumps(key_data, sort_keys=True)
return hashlib.md5(key_str.encode()).hexdigest()
def _get_cached(self, cache_key: str) -> Optional[FastContextResult]:
"""Get cached result if valid."""
if not self.cache_enabled:
return None
import time
cached = self._cache.get(cache_key)
if cached:
if time.time() - cached["timestamp"] < self.cache_ttl:
result = cached["result"]
result.from_cache = True
return result
else:
# Expired
del self._cache[cache_key]
return None
def _set_cached(self, cache_key: str, result: FastContextResult) -> None:
"""Cache a result."""
if not self.cache_enabled:
return
import time
self._cache[cache_key] = {
"result": result,
"timestamp": time.time()
}
def search(
self,
query: str,
include_patterns: Optional[List[str]] = None,
exclude_patterns: Optional[List[str]] = None,
use_llm: bool = False
) -> FastContextResult:
"""Search for code relevant to the query.
Args:
query: Search query (pattern or natural language)
include_patterns: Glob patterns to include (e.g., ["*.py"])
exclude_patterns: Glob patterns to exclude
use_llm: If True, use LLM for intelligent search
Returns:
FastContextResult with matching files and lines
"""
# Check cache
cache_key = self._get_cache_key(
query,
include_patterns=include_patterns,
exclude_patterns=exclude_patterns,
use_llm=use_llm
)
cached = self._get_cached(cache_key)
if cached:
if self.verbose:
logger.info(f"Cache hit for query: {query}")
return cached
# Perform search
agent = self._get_agent()
if use_llm:
result = agent.search(query, include_patterns, exclude_patterns)
else:
result = agent.search_simple(query, include_patterns, exclude_patterns)
# Cache result
self._set_cached(cache_key, result)
return result
def search_files(
self,
pattern: str,
max_results: int = 50
) -> FastContextResult:
"""Search for files matching a glob pattern.
Args:
pattern: Glob pattern (e.g., "**/*.py", "src/*.js")
max_results: Maximum results to return
Returns:
FastContextResult with matching files
"""
from praisonaiagents.context.fast.search_tools import glob_search
result = FastContextResult(query=pattern)
result.start_timer()
matches = glob_search(
self.workspace_path,
pattern,
max_results=max_results
)
for match in matches:
result.add_file(FileMatch(
path=match["path"],
relevance_score=1.0
))
result.stop_timer()
result.turns_used = 1
result.total_tool_calls = 1
return result
def read_context(
self,
filepath: str,
start_line: Optional[int] = None,
end_line: Optional[int] = None,
context_lines: int = 5
) -> Optional[str]:
"""Read file content with context.
Args:
filepath: Path to file (relative to workspace)
start_line: Starting line (1-indexed)
end_line: Ending line (1-indexed)
context_lines: Additional context lines
Returns:
File content string or None if error
"""
from praisonaiagents.context.fast.search_tools import read_file
full_path = filepath
if not os.path.isabs(filepath):
full_path = os.path.join(self.workspace_path, filepath)
result = read_file(
full_path,
start_line=start_line,
end_line=end_line,
context_lines=context_lines
)
if result["success"]:
return result["content"]
return None
def get_context_for_agent(
self,
query: str,
max_files: int = 10,
max_lines_per_file: int = 100,
include_content: bool = True
) -> str:
"""Get formatted context string for an agent.
This method searches for relevant code and formats it
as a context string that can be injected into an agent's
system prompt or message.
Args:
query: Search query
max_files: Maximum files to include
max_lines_per_file: Maximum lines per file
include_content: Whether to include file content
Returns:
Formatted context string
"""
result = self.search(query)
if not result.files:
return f"No relevant code found for: {query}"
lines = [f"# Relevant Code Context for: {query}"]
lines.append(f"Found {result.total_files} relevant file(s) in {result.search_time_ms}ms\n")
for i, file_match in enumerate(result.files[:max_files]):
lines.append(f"## {file_match.path}")
if include_content and file_match.line_ranges:
for lr in file_match.line_ranges:
# Limit lines
if lr.line_count > max_lines_per_file:
lr = LineRange(
start=lr.start,
end=lr.start + max_lines_per_file - 1,
content=lr.content,
relevance_score=lr.relevance_score
)
lines.append(f"Lines {lr.start}-{lr.end}:")
# Read content if not already present
if lr.content:
lines.append(f"```\n{lr.content}\n```")
else:
content = self.read_context(
file_match.path,
start_line=lr.start,
end_line=lr.end
)
if content:
lines.append(f"```\n{content}\n```")
lines.append("")
if len(result.files) > max_files:
lines.append(f"... and {len(result.files) - max_files} more files")
return "\n".join(lines)
def clear_cache(self) -> None:
"""Clear the result cache."""
self._cache.clear()
def close(self) -> None:
"""Close resources."""
if self._agent:
self._agent.close()
self._agent = None
self._cache.clear()
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
# Convenience function for quick searches
def fast_search(
query: str,
workspace_path: Optional[str] = None,
**kwargs
) -> FastContextResult:
"""Convenience function for quick code search.
Args:
query: Search query
workspace_path: Root directory (defaults to cwd)
**kwargs: Additional arguments for FastContext
Returns:
FastContextResult with matching files
"""
with FastContext(workspace_path=workspace_path, **kwargs) as fc:
return fc.search(query)
@@ -0,0 +1,392 @@
"""
FastContextAgent - Specialized subagent for rapid code search.
This agent is designed for fast context retrieval using:
- Parallel tool execution (up to 8 concurrent calls)
- Limited turns (max 4) for fast response
- Lightweight model (gpt-4o-mini by default)
- Restricted tool set (grep, glob, read, list_dir)
"""
import os
import json
import logging
from typing import List, Dict, Any, Optional
from praisonaiagents.context.fast.result import (
FastContextResult,
FileMatch,
LineRange
)
from praisonaiagents.context.fast.search_tools import (
grep_search,
glob_search,
read_file,
list_directory,
FAST_CONTEXT_TOOLS
)
from praisonaiagents.context.fast.parallel_executor import (
ToolCallBatch,
ParallelSearchCoordinator
)
logger = logging.getLogger(__name__)
class FastContextAgent:
"""Specialized agent for fast parallel code search.
This agent performs rapid code search using parallel tool calls
and a lightweight model optimized for retrieval tasks.
Attributes:
workspace_path: Root directory for searches
max_turns: Maximum search turns (default: 4)
max_parallel: Maximum parallel tool calls (default: 8)
model: LLM model for intelligent search (default: gpt-4o-mini)
timeout: Timeout per tool call in seconds
"""
def __init__(
self,
workspace_path: str,
max_turns: int = 4,
max_parallel: int = 8,
model: str = "gpt-4o-mini",
timeout: float = 30.0,
verbose: bool = False
):
"""Initialize FastContextAgent.
Args:
workspace_path: Root directory for searches
max_turns: Maximum search turns
max_parallel: Maximum parallel tool calls per turn
model: LLM model name for intelligent search
timeout: Timeout per tool call in seconds
verbose: If True, print debug information
"""
self.workspace_path = os.path.abspath(workspace_path)
self.max_turns = max_turns
self.max_parallel = max_parallel
self.model = model
self.timeout = timeout
self.verbose = verbose
# Initialize coordinator
self.coordinator = ParallelSearchCoordinator(
max_parallel=max_parallel,
max_turns=max_turns,
timeout=timeout
)
# LLM instance (lazy loaded)
self._llm = None
def _get_llm(self):
"""Get or create LLM instance (lazy loading)."""
if self._llm is None:
try:
from praisonaiagents.llm import LLM
self._llm = LLM(model=self.model)
except Exception as e:
logger.warning(f"Could not initialize LLM: {e}")
self._llm = None
return self._llm
def get_tools(self) -> List[Dict[str, Any]]:
"""Get tool definitions for LLM function calling.
Returns:
List of tool definition dictionaries
"""
return FAST_CONTEXT_TOOLS
def execute_tool(self, tool_name: str, **kwargs) -> Any:
"""Execute a single tool.
Args:
tool_name: Name of the tool
**kwargs: Tool arguments
Returns:
Tool result
"""
# Prepend workspace path if needed
if tool_name in ("grep_search", "glob_search"):
if "search_path" not in kwargs or kwargs["search_path"] == ".":
kwargs["search_path"] = self.workspace_path
elif not os.path.isabs(kwargs["search_path"]):
kwargs["search_path"] = os.path.join(
self.workspace_path, kwargs["search_path"]
)
elif tool_name == "list_directory":
if "dir_path" not in kwargs or kwargs["dir_path"] == ".":
kwargs["dir_path"] = self.workspace_path
elif not os.path.isabs(kwargs["dir_path"]):
kwargs["dir_path"] = os.path.join(
self.workspace_path, kwargs["dir_path"]
)
elif tool_name == "read_file":
if "filepath" in kwargs and not os.path.isabs(kwargs["filepath"]):
kwargs["filepath"] = os.path.join(
self.workspace_path, kwargs["filepath"]
)
tools = {
"grep_search": grep_search,
"glob_search": glob_search,
"read_file": read_file,
"list_directory": list_directory
}
if tool_name not in tools:
return {"error": f"Unknown tool: {tool_name}"}
try:
return tools[tool_name](**kwargs)
except Exception as e:
return {"error": str(e)}
def search_simple(
self,
query: str,
include_patterns: Optional[List[str]] = None,
exclude_patterns: Optional[List[str]] = None,
max_results: int = 50
) -> FastContextResult:
"""Perform a simple search without LLM.
This is a fast, direct search that doesn't use the LLM
for query understanding. Good for simple pattern searches.
Args:
query: Search pattern
include_patterns: Glob patterns to include
exclude_patterns: Glob patterns to exclude
max_results: Maximum results per tool
Returns:
FastContextResult with matching files and lines
"""
result = FastContextResult(query=query)
result.start_timer()
# Execute grep search
grep_results = grep_search(
search_path=self.workspace_path,
pattern=query,
include_patterns=include_patterns,
exclude_patterns=exclude_patterns,
max_results=max_results,
context_lines=2
)
# Convert to FastContextResult format
for match in grep_results:
file_match = FileMatch(
path=match["path"],
relevance_score=1.0,
match_count=1
)
file_match.add_line_range(LineRange(
start=max(1, match["line_number"] - 2),
end=match["line_number"] + 2,
content=match.get("context"),
relevance_score=1.0
))
result.add_file(file_match)
result.stop_timer()
result.turns_used = 1
result.total_tool_calls = 1
return result
def search(
self,
query: str,
include_patterns: Optional[List[str]] = None,
exclude_patterns: Optional[List[str]] = None
) -> FastContextResult:
"""Perform intelligent search using LLM.
Uses the LLM to understand the query and plan search strategy,
then executes parallel tool calls to find relevant code.
Args:
query: Natural language search query
include_patterns: Glob patterns to include
exclude_patterns: Glob patterns to exclude
Returns:
FastContextResult with matching files and lines
"""
result = FastContextResult(query=query)
result.start_timer()
llm = self._get_llm()
# If no LLM available, fall back to simple search
if llm is None:
logger.info("LLM not available, falling back to simple search")
return self.search_simple(query, include_patterns, exclude_patterns)
self.coordinator.start()
# System prompt for the search agent
system_prompt = f"""You are a fast code search agent. Your task is to find relevant code in a codebase.
Workspace: {self.workspace_path}
You have access to these tools:
- grep_search: Search for patterns in files
- glob_search: Find files by pattern
- read_file: Read file contents
- list_directory: List directory contents
Strategy:
1. Start with broad searches (glob, list_directory) to understand structure
2. Use grep_search to find specific patterns
3. Use read_file to get detailed context for relevant files
4. Execute up to {self.max_parallel} tool calls in parallel per turn
5. Complete search in {self.max_turns} turns or less
Return files and line ranges that are relevant to the query.
Be efficient - use parallel tool calls to explore multiple paths at once."""
messages = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": f"Find code relevant to: {query}"}
]
# Add file pattern constraints if provided
if include_patterns:
messages[1]["content"] += f"\nOnly search in files matching: {include_patterns}"
if exclude_patterns:
messages[1]["content"] += f"\nExclude files matching: {exclude_patterns}"
try:
# Multi-turn search loop
for turn in range(self.max_turns):
if self.verbose:
logger.info(f"Search turn {turn + 1}/{self.max_turns}")
# Get LLM response with tool calls
response = llm.client.chat.completions.create(
model=self.model,
messages=messages,
tools=[{"type": "function", "function": t} for t in self.get_tools()],
tool_choice="auto"
)
message = response.choices[0].message
# Check if we have tool calls
if not message.tool_calls:
# No more tool calls, search complete
break
# Execute tool calls in parallel
batch = ToolCallBatch(max_size=self.max_parallel)
tool_call_map = {} # Map tool call ID to batch index
for tool_call in message.tool_calls[:self.max_parallel]:
try:
args = json.loads(tool_call.function.arguments)
batch.add(tool_call.function.name, **args)
tool_call_map[tool_call.id] = len(batch.tasks) - 1
except json.JSONDecodeError:
continue
# Execute batch
if batch.tasks:
tool_results = self.coordinator.execute_turn_sync(batch)
# Process results
messages.append({"role": "assistant", "content": None, "tool_calls": [
{
"id": tc.id,
"type": "function",
"function": {
"name": tc.function.name,
"arguments": tc.function.arguments
}
}
for tc in message.tool_calls[:self.max_parallel]
]})
# Add tool results to messages
for tool_call in message.tool_calls[:self.max_parallel]:
idx = tool_call_map.get(tool_call.id)
if idx is not None and idx < len(tool_results):
tool_result = tool_results[idx]
# Process grep results into FastContextResult
if tool_call.function.name == "grep_search":
if isinstance(tool_result, list):
for match in tool_result:
file_match = FileMatch(
path=match.get("path", ""),
relevance_score=1.0,
match_count=1
)
line_num = match.get("line_number", 1)
file_match.add_line_range(LineRange(
start=max(1, line_num - 2),
end=line_num + 2,
content=match.get("context"),
relevance_score=1.0
))
result.add_file(file_match)
messages.append({
"role": "tool",
"tool_call_id": tool_call.id,
"content": json.dumps(tool_result) if not isinstance(tool_result, str) else tool_result
})
except Exception as e:
logger.error(f"Error during LLM search: {e}")
# Fall back to simple search
return self.search_simple(query, include_patterns, exclude_patterns)
finally:
self.coordinator.stop()
result.stop_timer()
result.turns_used = self.coordinator.turns_used
result.total_tool_calls = self.coordinator.total_tool_calls
result.sort_by_relevance()
return result
async def search_async(
self,
query: str,
include_patterns: Optional[List[str]] = None,
exclude_patterns: Optional[List[str]] = None
) -> FastContextResult:
"""Async version of search.
Args:
query: Natural language search query
include_patterns: Glob patterns to include
exclude_patterns: Glob patterns to exclude
Returns:
FastContextResult with matching files and lines
"""
# For now, wrap sync version
# TODO: Implement true async with async LLM calls
return self.search(query, include_patterns, exclude_patterns)
def close(self) -> None:
"""Close resources."""
self.coordinator.close()
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
@@ -0,0 +1,23 @@
"""
Indexer module for Fast Context.
Provides file and symbol indexing capabilities for fast code search.
"""
def __getattr__(name):
"""Lazy load indexer classes."""
if name == "FileIndexer":
from .file_indexer import FileIndexer
return FileIndexer
elif name == "SymbolIndexer":
from .symbol_indexer import SymbolIndexer
return SymbolIndexer
elif name == "Symbol":
from .symbol_indexer import Symbol
return Symbol
elif name == "SymbolType":
from .symbol_indexer import SymbolType
return SymbolType
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
__all__ = ["FileIndexer", "SymbolIndexer", "Symbol", "SymbolType"]
@@ -0,0 +1,445 @@
"""
File Indexer for Fast Context.
Provides file indexing capabilities for faster code search:
- Indexes file paths and metadata
- Supports .gitignore and .praisonignore patterns
- Incremental updates on file changes
- Memory-efficient storage
"""
import os
import fnmatch
import hashlib
import json
import logging
from dataclasses import dataclass
from typing import Dict, List, Set, Optional, Any
from pathlib import Path
import time
logger = logging.getLogger(__name__)
@dataclass
class FileInfo:
"""Information about an indexed file.
Attributes:
path: Relative path from workspace root
absolute_path: Absolute file path
size: File size in bytes
modified: Last modification timestamp
extension: File extension (lowercase)
hash: Content hash for change detection
"""
path: str
absolute_path: str
size: int
modified: float
extension: str
hash: Optional[str] = None
def to_dict(self) -> Dict[str, Any]:
"""Convert to dictionary."""
return {
"path": self.path,
"absolute_path": self.absolute_path,
"size": self.size,
"modified": self.modified,
"extension": self.extension,
"hash": self.hash
}
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "FileInfo":
"""Create from dictionary."""
return cls(**data)
class FileIndexer:
"""Indexes files in a workspace for fast search.
The indexer maintains an in-memory index of files that can be
used to quickly find files by pattern without filesystem traversal.
Attributes:
workspace_path: Root directory to index
files: Dictionary mapping relative paths to FileInfo
extensions: Set of indexed file extensions
"""
# Default extensions to index
DEFAULT_EXTENSIONS = {
".py", ".js", ".ts", ".jsx", ".tsx", ".java", ".go", ".rs",
".c", ".cpp", ".h", ".hpp", ".cs", ".rb", ".php", ".swift",
".kt", ".scala", ".r", ".m", ".mm", ".sql", ".sh", ".bash",
".yaml", ".yml", ".json", ".xml", ".html", ".css", ".scss",
".md", ".txt", ".rst", ".toml", ".ini", ".cfg", ".conf"
}
# Default patterns to ignore
DEFAULT_IGNORE_PATTERNS = {
"__pycache__", "*.pyc", "*.pyo", ".git", ".svn", ".hg",
"node_modules", "venv", ".venv", "env", ".env",
"dist", "build", ".cache", ".pytest_cache",
"*.egg-info", "*.egg", "*.whl", "*.so", "*.dll", "*.dylib"
}
def __init__(
self,
workspace_path: str,
extensions: Optional[Set[str]] = None,
ignore_patterns: Optional[Set[str]] = None,
respect_gitignore: bool = True,
max_file_size: int = 10 * 1024 * 1024, # 10MB
compute_hashes: bool = False
):
"""Initialize file indexer.
Args:
workspace_path: Root directory to index
extensions: File extensions to index (None = all text files)
ignore_patterns: Patterns to ignore
respect_gitignore: Whether to respect .gitignore
max_file_size: Maximum file size to index
compute_hashes: Whether to compute content hashes
"""
self.workspace_path = os.path.abspath(workspace_path)
self.extensions = extensions or self.DEFAULT_EXTENSIONS
self.ignore_patterns = ignore_patterns or self.DEFAULT_IGNORE_PATTERNS.copy()
self.respect_gitignore = respect_gitignore
self.max_file_size = max_file_size
self.compute_hashes = compute_hashes
# Index storage
self.files: Dict[str, FileInfo] = {}
self.by_extension: Dict[str, List[str]] = {}
# Gitignore patterns
self._gitignore_patterns: Set[str] = set()
# Index metadata
self.last_indexed: Optional[float] = None
self.total_files: int = 0
self.total_size: int = 0
def _load_gitignore(self) -> None:
"""Load patterns from .gitignore and .praisonignore."""
self._gitignore_patterns.clear()
for filename in [".gitignore", ".praisonignore"]:
filepath = os.path.join(self.workspace_path, filename)
if os.path.exists(filepath):
try:
with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
for line in f:
line = line.strip()
if line and not line.startswith('#'):
self._gitignore_patterns.add(line)
except Exception:
pass
def _should_ignore(self, path: str, is_dir: bool = False) -> bool:
"""Check if a path should be ignored.
Args:
path: Relative path from workspace
is_dir: Whether the path is a directory
Returns:
True if path should be ignored
"""
name = os.path.basename(path)
parts = Path(path).parts
# Check default ignore patterns
for pattern in self.ignore_patterns:
if fnmatch.fnmatch(name, pattern):
return True
if any(fnmatch.fnmatch(part, pattern) for part in parts):
return True
# Check gitignore patterns
if self.respect_gitignore:
for pattern in self._gitignore_patterns:
# Handle directory patterns
if pattern.endswith('/'):
dir_pattern = pattern.rstrip('/')
if is_dir and fnmatch.fnmatch(name, dir_pattern):
return True
if any(fnmatch.fnmatch(part, dir_pattern) for part in parts):
return True
# Handle ** patterns
elif '**' in pattern:
fnmatch_pattern = pattern.replace('**/', '*').replace('**', '*')
if fnmatch.fnmatch(path, fnmatch_pattern):
return True
# Handle simple patterns
else:
if fnmatch.fnmatch(name, pattern):
return True
if fnmatch.fnmatch(path, pattern):
return True
return False
def _compute_hash(self, filepath: str) -> Optional[str]:
"""Compute MD5 hash of file content.
Args:
filepath: Path to file
Returns:
MD5 hash string or None on error
"""
if not self.compute_hashes:
return None
try:
hasher = hashlib.md5()
with open(filepath, 'rb') as f:
for chunk in iter(lambda: f.read(8192), b''):
hasher.update(chunk)
return hasher.hexdigest()
except Exception:
return None
def index(self) -> int:
"""Index all files in the workspace.
Returns:
Number of files indexed
"""
start_time = time.perf_counter()
# Load gitignore patterns
if self.respect_gitignore:
self._load_gitignore()
# Clear existing index
self.files.clear()
self.by_extension.clear()
self.total_files = 0
self.total_size = 0
# Walk directory tree
for root, dirs, files in os.walk(self.workspace_path):
# Get relative path
rel_root = os.path.relpath(root, self.workspace_path)
if rel_root == '.':
rel_root = ''
# Filter directories in-place
dirs[:] = [d for d in dirs if not self._should_ignore(
os.path.join(rel_root, d) if rel_root else d, is_dir=True
)]
for filename in files:
rel_path = os.path.join(rel_root, filename) if rel_root else filename
abs_path = os.path.join(root, filename)
# Check if should ignore
if self._should_ignore(rel_path):
continue
# Check extension
ext = os.path.splitext(filename)[1].lower()
if self.extensions and ext not in self.extensions:
continue
try:
stat = os.stat(abs_path)
# Check file size
if stat.st_size > self.max_file_size:
continue
# Create file info
file_info = FileInfo(
path=rel_path,
absolute_path=abs_path,
size=stat.st_size,
modified=stat.st_mtime,
extension=ext,
hash=self._compute_hash(abs_path)
)
# Add to index
self.files[rel_path] = file_info
# Add to extension index
if ext not in self.by_extension:
self.by_extension[ext] = []
self.by_extension[ext].append(rel_path)
self.total_files += 1
self.total_size += stat.st_size
except Exception as e:
logger.debug(f"Error indexing {abs_path}: {e}")
continue
self.last_indexed = time.perf_counter()
elapsed = self.last_indexed - start_time
logger.info(f"Indexed {self.total_files} files ({self.total_size / 1024 / 1024:.2f} MB) in {elapsed:.2f}s")
return self.total_files
def find_by_pattern(self, pattern: str) -> List[FileInfo]:
"""Find files matching a glob pattern.
Args:
pattern: Glob pattern (e.g., "**/*.py", "src/*.js")
Returns:
List of matching FileInfo objects
"""
results = []
for rel_path, file_info in self.files.items():
if fnmatch.fnmatch(rel_path, pattern):
results.append(file_info)
elif fnmatch.fnmatch(os.path.basename(rel_path), pattern):
results.append(file_info)
return results
def find_by_extension(self, extension: str) -> List[FileInfo]:
"""Find files by extension.
Args:
extension: File extension (with or without dot)
Returns:
List of matching FileInfo objects
"""
if not extension.startswith('.'):
extension = '.' + extension
extension = extension.lower()
paths = self.by_extension.get(extension, [])
return [self.files[p] for p in paths if p in self.files]
def find_by_name(self, name: str, exact: bool = False) -> List[FileInfo]:
"""Find files by name.
Args:
name: Filename to search for
exact: If True, match exact name; otherwise, partial match
Returns:
List of matching FileInfo objects
"""
results = []
name_lower = name.lower()
for rel_path, file_info in self.files.items():
filename = os.path.basename(rel_path).lower()
if exact:
if filename == name_lower:
results.append(file_info)
else:
if name_lower in filename:
results.append(file_info)
return results
def get_file(self, path: str) -> Optional[FileInfo]:
"""Get file info by path.
Args:
path: Relative path from workspace
Returns:
FileInfo or None if not found
"""
return self.files.get(path)
def is_stale(self, max_age: float = 60.0) -> bool:
"""Check if index is stale.
Args:
max_age: Maximum age in seconds
Returns:
True if index is stale or not indexed
"""
if self.last_indexed is None:
return True
return (time.perf_counter() - self.last_indexed) > max_age
def get_stats(self) -> Dict[str, Any]:
"""Get index statistics.
Returns:
Dictionary with index stats
"""
return {
"workspace_path": self.workspace_path,
"total_files": self.total_files,
"total_size_bytes": self.total_size,
"total_size_mb": round(self.total_size / 1024 / 1024, 2),
"extensions": dict(sorted(
((ext, len(paths)) for ext, paths in self.by_extension.items()),
key=lambda x: x[1],
reverse=True
)),
"last_indexed": self.last_indexed
}
def save_index(self, filepath: str) -> None:
"""Save index to file.
Args:
filepath: Path to save index
"""
data = {
"workspace_path": self.workspace_path,
"last_indexed": self.last_indexed,
"files": {path: info.to_dict() for path, info in self.files.items()}
}
with open(filepath, 'w', encoding='utf-8') as f:
json.dump(data, f, indent=2)
def load_index(self, filepath: str) -> bool:
"""Load index from file.
Args:
filepath: Path to load index from
Returns:
True if loaded successfully
"""
try:
with open(filepath, 'r', encoding='utf-8') as f:
data = json.load(f)
# Verify workspace path matches
if data.get("workspace_path") != self.workspace_path:
return False
self.last_indexed = data.get("last_indexed")
self.files.clear()
self.by_extension.clear()
self.total_files = 0
self.total_size = 0
for path, info_dict in data.get("files", {}).items():
file_info = FileInfo.from_dict(info_dict)
self.files[path] = file_info
ext = file_info.extension
if ext not in self.by_extension:
self.by_extension[ext] = []
self.by_extension[ext].append(path)
self.total_files += 1
self.total_size += file_info.size
return True
except Exception as e:
logger.debug(f"Error loading index: {e}")
return False
@@ -0,0 +1,321 @@
"""
File Watcher for Incremental Indexing.
Monitors file system changes and triggers incremental index updates:
- Watches for file creation, modification, deletion
- Debounces rapid changes
- Supports background indexing
"""
import os
import time
import logging
import threading
from typing import Dict, Set, Callable, Optional, Any
from dataclasses import dataclass
from enum import Enum
logger = logging.getLogger(__name__)
class ChangeType(Enum):
"""Types of file system changes."""
CREATED = "created"
MODIFIED = "modified"
DELETED = "deleted"
@dataclass
class FileChange:
"""Represents a file system change.
Attributes:
path: Path to the changed file
change_type: Type of change
timestamp: When the change was detected
"""
path: str
change_type: ChangeType
timestamp: float
class FileWatcher:
"""Watches for file system changes and triggers callbacks.
Uses polling-based approach for cross-platform compatibility.
For production use, consider using watchdog library.
Attributes:
workspace_path: Root directory to watch
extensions: File extensions to watch
poll_interval: Seconds between polls
"""
def __init__(
self,
workspace_path: str,
extensions: Optional[Set[str]] = None,
poll_interval: float = 1.0,
debounce_delay: float = 0.5,
on_change: Optional[Callable[[FileChange], None]] = None
):
"""Initialize file watcher.
Args:
workspace_path: Root directory to watch
extensions: File extensions to watch (None = all)
poll_interval: Seconds between polls
debounce_delay: Delay before processing changes
on_change: Callback for file changes
"""
self.workspace_path = os.path.abspath(workspace_path)
self.extensions = extensions or {
'.py', '.js', '.ts', '.jsx', '.tsx', '.go', '.rs', '.java',
'.c', '.cpp', '.h', '.hpp', '.cs', '.rb', '.php', '.swift'
}
self.poll_interval = poll_interval
self.debounce_delay = debounce_delay
self.on_change = on_change
# State tracking
self._file_mtimes: Dict[str, float] = {}
self._pending_changes: Dict[str, FileChange] = {}
self._running = False
self._thread: Optional[threading.Thread] = None
self._lock = threading.Lock()
def _scan_files(self) -> Dict[str, float]:
"""Scan workspace and get file modification times.
Returns:
Dict mapping file paths to modification times
"""
mtimes = {}
for root, dirs, files in os.walk(self.workspace_path):
# Skip hidden directories
dirs[:] = [d for d in dirs if not d.startswith('.')]
for filename in files:
ext = os.path.splitext(filename)[1].lower()
if ext in self.extensions:
file_path = os.path.join(root, filename)
try:
mtimes[file_path] = os.path.getmtime(file_path)
except OSError:
pass
return mtimes
def _detect_changes(self, new_mtimes: Dict[str, float]) -> None:
"""Detect changes between old and new file states.
Args:
new_mtimes: New file modification times
"""
old_paths = set(self._file_mtimes.keys())
new_paths = set(new_mtimes.keys())
now = time.time()
# Detect created files
for path in new_paths - old_paths:
self._add_pending_change(FileChange(
path=path,
change_type=ChangeType.CREATED,
timestamp=now
))
# Detect deleted files
for path in old_paths - new_paths:
self._add_pending_change(FileChange(
path=path,
change_type=ChangeType.DELETED,
timestamp=now
))
# Detect modified files
for path in old_paths & new_paths:
if new_mtimes[path] > self._file_mtimes[path]:
self._add_pending_change(FileChange(
path=path,
change_type=ChangeType.MODIFIED,
timestamp=now
))
self._file_mtimes = new_mtimes
def _add_pending_change(self, change: FileChange) -> None:
"""Add a change to pending queue (with debouncing).
Args:
change: File change to add
"""
with self._lock:
self._pending_changes[change.path] = change
def _process_pending_changes(self) -> None:
"""Process pending changes after debounce delay."""
with self._lock:
now = time.time()
to_process = []
for path, change in list(self._pending_changes.items()):
if now - change.timestamp >= self.debounce_delay:
to_process.append(change)
del self._pending_changes[path]
# Process changes outside lock
for change in to_process:
if self.on_change:
try:
self.on_change(change)
except Exception as e:
logger.error(f"Error processing change {change.path}: {e}")
def _poll_loop(self) -> None:
"""Main polling loop."""
# Initial scan
self._file_mtimes = self._scan_files()
while self._running:
try:
# Scan for changes
new_mtimes = self._scan_files()
self._detect_changes(new_mtimes)
# Process pending changes
self._process_pending_changes()
except Exception as e:
logger.error(f"Error in file watcher: {e}")
time.sleep(self.poll_interval)
def start(self) -> None:
"""Start watching for file changes."""
if self._running:
return
self._running = True
self._thread = threading.Thread(target=self._poll_loop, daemon=True)
self._thread.start()
logger.info(f"File watcher started for {self.workspace_path}")
def stop(self) -> None:
"""Stop watching for file changes."""
self._running = False
if self._thread:
self._thread.join(timeout=2.0)
self._thread = None
logger.info("File watcher stopped")
def __enter__(self):
self.start()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.stop()
class IncrementalIndexer:
"""Manages incremental index updates based on file changes.
Combines FileIndexer/SymbolIndexer with FileWatcher for
automatic index updates when files change.
"""
def __init__(
self,
workspace_path: str,
file_indexer: Optional[Any] = None,
symbol_indexer: Optional[Any] = None,
poll_interval: float = 2.0
):
"""Initialize incremental indexer.
Args:
workspace_path: Root directory to index
file_indexer: FileIndexer instance (optional)
symbol_indexer: SymbolIndexer instance (optional)
poll_interval: Seconds between file system polls
"""
self.workspace_path = os.path.abspath(workspace_path)
self.file_indexer = file_indexer
self.symbol_indexer = symbol_indexer
self._watcher = FileWatcher(
workspace_path=workspace_path,
poll_interval=poll_interval,
on_change=self._handle_change
)
# Statistics
self.changes_processed = 0
self.last_change_time: Optional[float] = None
def _handle_change(self, change: FileChange) -> None:
"""Handle a file change.
Args:
change: File change to process
"""
rel_path = os.path.relpath(change.path, self.workspace_path)
logger.debug(f"Processing {change.change_type.value}: {rel_path}")
if change.change_type == ChangeType.DELETED:
# Remove from indexes
if self.file_indexer and hasattr(self.file_indexer, 'files'):
self.file_indexer.files.pop(rel_path, None)
if self.symbol_indexer and hasattr(self.symbol_indexer, 'symbols'):
self.symbol_indexer.symbols.pop(rel_path, None)
else:
# Re-index the file
if self.file_indexer and hasattr(self.file_indexer, 'index_file'):
# FileIndexer doesn't have index_file, so we skip
pass
if self.symbol_indexer and hasattr(self.symbol_indexer, 'index_file'):
self.symbol_indexer.index_file(change.path)
self.changes_processed += 1
self.last_change_time = time.time()
def start(self) -> None:
"""Start incremental indexing."""
# Initial full index
if self.file_indexer:
self.file_indexer.index()
if self.symbol_indexer:
self.symbol_indexer.index()
# Start watching
self._watcher.start()
def stop(self) -> None:
"""Stop incremental indexing."""
self._watcher.stop()
def get_stats(self) -> Dict[str, Any]:
"""Get indexer statistics.
Returns:
Dictionary with stats
"""
stats = {
"changes_processed": self.changes_processed,
"last_change_time": self.last_change_time
}
if self.file_indexer:
stats["file_indexer"] = self.file_indexer.get_stats()
if self.symbol_indexer:
stats["symbol_indexer"] = self.symbol_indexer.get_stats()
return stats
def __enter__(self):
self.start()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.stop()
@@ -0,0 +1,669 @@
"""
Symbol Indexer for Fast Context.
Extracts code symbols from source files:
- Python: functions, classes, imports
- JavaScript/TypeScript: functions, classes, imports, exports
- Go: functions, types, imports
- Rust: functions, structs, impl blocks, use statements
- Java: classes, methods, imports
"""
import os
import re
import logging
from dataclasses import dataclass, field
from typing import Dict, List, Set, Optional, Any
from pathlib import Path
from enum import Enum
logger = logging.getLogger(__name__)
class SymbolType(Enum):
"""Types of code symbols."""
FUNCTION = "function"
CLASS = "class"
METHOD = "method"
IMPORT = "import"
EXPORT = "export"
VARIABLE = "variable"
CONSTANT = "constant"
TYPE = "type"
STRUCT = "struct"
INTERFACE = "interface"
MODULE = "module"
@dataclass
class Symbol:
"""Represents a code symbol.
Attributes:
name: Symbol name
symbol_type: Type of symbol
file_path: File containing the symbol
line_number: Line number (1-indexed)
end_line: End line number (for multi-line symbols)
signature: Full signature (for functions/methods)
parent: Parent symbol name (for methods)
docstring: Documentation string if available
"""
name: str
symbol_type: SymbolType
file_path: str
line_number: int
end_line: Optional[int] = None
signature: Optional[str] = None
parent: Optional[str] = None
docstring: Optional[str] = None
def to_dict(self) -> Dict[str, Any]:
"""Convert to dictionary."""
return {
"name": self.name,
"type": self.symbol_type.value,
"file_path": self.file_path,
"line_number": self.line_number,
"end_line": self.end_line,
"signature": self.signature,
"parent": self.parent,
"docstring": self.docstring
}
class LanguageParser:
"""Base class for language-specific parsers."""
def parse(self, content: str, file_path: str) -> List[Symbol]:
"""Parse content and extract symbols.
Args:
content: File content
file_path: Path to the file
Returns:
List of extracted symbols
"""
raise NotImplementedError
class PythonParser(LanguageParser):
"""Parser for Python files."""
# Regex patterns for Python
FUNCTION_PATTERN = re.compile(
r'^(\s*)def\s+(\w+)\s*\((.*?)\)\s*(?:->.*?)?:',
re.MULTILINE
)
CLASS_PATTERN = re.compile(
r'^(\s*)class\s+(\w+)\s*(?:\((.*?)\))?:',
re.MULTILINE
)
IMPORT_PATTERN = re.compile(
r'^(?:from\s+(\S+)\s+)?import\s+(.+)$',
re.MULTILINE
)
ASYNC_FUNCTION_PATTERN = re.compile(
r'^(\s*)async\s+def\s+(\w+)\s*\((.*?)\)\s*(?:->.*?)?:',
re.MULTILINE
)
def parse(self, content: str, file_path: str) -> List[Symbol]:
symbols = []
lines = content.split('\n')
# Track class context for methods
current_class = None
class_indent = 0
# Parse classes
for match in self.CLASS_PATTERN.finditer(content):
indent = len(match.group(1))
name = match.group(2)
bases = match.group(3) or ""
line_num = content[:match.start()].count('\n') + 1
symbols.append(Symbol(
name=name,
symbol_type=SymbolType.CLASS,
file_path=file_path,
line_number=line_num,
signature=f"class {name}({bases})" if bases else f"class {name}"
))
# Parse functions (including async)
for pattern in [self.FUNCTION_PATTERN, self.ASYNC_FUNCTION_PATTERN]:
for match in pattern.finditer(content):
indent = len(match.group(1))
name = match.group(2)
params = match.group(3)
line_num = content[:match.start()].count('\n') + 1
# Determine if it's a method (inside a class)
is_method = indent > 0
parent = None
if is_method:
# Find parent class
for sym in symbols:
if sym.symbol_type == SymbolType.CLASS:
if sym.line_number < line_num:
parent = sym.name
symbol_type = SymbolType.METHOD if is_method else SymbolType.FUNCTION
is_async = "async" in match.group(0)
prefix = "async def" if is_async else "def"
symbols.append(Symbol(
name=name,
symbol_type=symbol_type,
file_path=file_path,
line_number=line_num,
signature=f"{prefix} {name}({params})",
parent=parent
))
# Parse imports
for match in self.IMPORT_PATTERN.finditer(content):
from_module = match.group(1)
imports = match.group(2)
line_num = content[:match.start()].count('\n') + 1
if from_module:
sig = f"from {from_module} import {imports}"
else:
sig = f"import {imports}"
symbols.append(Symbol(
name=imports.split(',')[0].strip().split(' ')[0],
symbol_type=SymbolType.IMPORT,
file_path=file_path,
line_number=line_num,
signature=sig
))
return symbols
class JavaScriptParser(LanguageParser):
"""Parser for JavaScript/TypeScript files."""
FUNCTION_PATTERN = re.compile(
r'(?:export\s+)?(?:async\s+)?function\s+(\w+)\s*\((.*?)\)',
re.MULTILINE
)
ARROW_FUNCTION_PATTERN = re.compile(
r'(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?\((.*?)\)\s*=>',
re.MULTILINE
)
CLASS_PATTERN = re.compile(
r'(?:export\s+)?class\s+(\w+)(?:\s+extends\s+(\w+))?',
re.MULTILINE
)
IMPORT_PATTERN = re.compile(
r'import\s+(?:{([^}]+)}|(\w+))\s+from\s+[\'"]([^\'"]+)[\'"]',
re.MULTILINE
)
EXPORT_PATTERN = re.compile(
r'export\s+(?:default\s+)?(?:const|let|var|function|class)\s+(\w+)',
re.MULTILINE
)
def parse(self, content: str, file_path: str) -> List[Symbol]:
symbols = []
# Parse functions
for match in self.FUNCTION_PATTERN.finditer(content):
name = match.group(1)
params = match.group(2)
line_num = content[:match.start()].count('\n') + 1
symbols.append(Symbol(
name=name,
symbol_type=SymbolType.FUNCTION,
file_path=file_path,
line_number=line_num,
signature=f"function {name}({params})"
))
# Parse arrow functions
for match in self.ARROW_FUNCTION_PATTERN.finditer(content):
name = match.group(1)
params = match.group(2)
line_num = content[:match.start()].count('\n') + 1
symbols.append(Symbol(
name=name,
symbol_type=SymbolType.FUNCTION,
file_path=file_path,
line_number=line_num,
signature=f"const {name} = ({params}) =>"
))
# Parse classes
for match in self.CLASS_PATTERN.finditer(content):
name = match.group(1)
extends = match.group(2)
line_num = content[:match.start()].count('\n') + 1
sig = f"class {name}"
if extends:
sig += f" extends {extends}"
symbols.append(Symbol(
name=name,
symbol_type=SymbolType.CLASS,
file_path=file_path,
line_number=line_num,
signature=sig
))
# Parse imports
for match in self.IMPORT_PATTERN.finditer(content):
named = match.group(1)
default = match.group(2)
module = match.group(3)
line_num = content[:match.start()].count('\n') + 1
name = default or (named.split(',')[0].strip() if named else module)
symbols.append(Symbol(
name=name,
symbol_type=SymbolType.IMPORT,
file_path=file_path,
line_number=line_num,
signature=f"import from '{module}'"
))
# Parse exports
for match in self.EXPORT_PATTERN.finditer(content):
name = match.group(1)
line_num = content[:match.start()].count('\n') + 1
symbols.append(Symbol(
name=name,
symbol_type=SymbolType.EXPORT,
file_path=file_path,
line_number=line_num,
signature=f"export {name}"
))
return symbols
class GoParser(LanguageParser):
"""Parser for Go files."""
FUNCTION_PATTERN = re.compile(
r'^func\s+(?:\((\w+)\s+\*?(\w+)\)\s+)?(\w+)\s*\((.*?)\)',
re.MULTILINE
)
TYPE_PATTERN = re.compile(
r'^type\s+(\w+)\s+(struct|interface)',
re.MULTILINE
)
IMPORT_PATTERN = re.compile(
r'import\s+(?:\(\s*([\s\S]*?)\s*\)|"([^"]+)")',
re.MULTILINE
)
def parse(self, content: str, file_path: str) -> List[Symbol]:
symbols = []
# Parse functions
for match in self.FUNCTION_PATTERN.finditer(content):
receiver_name = match.group(1)
receiver_type = match.group(2)
name = match.group(3)
params = match.group(4)
line_num = content[:match.start()].count('\n') + 1
if receiver_type:
# Method
symbols.append(Symbol(
name=name,
symbol_type=SymbolType.METHOD,
file_path=file_path,
line_number=line_num,
signature=f"func ({receiver_name} {receiver_type}) {name}({params})",
parent=receiver_type
))
else:
# Function
symbols.append(Symbol(
name=name,
symbol_type=SymbolType.FUNCTION,
file_path=file_path,
line_number=line_num,
signature=f"func {name}({params})"
))
# Parse types
for match in self.TYPE_PATTERN.finditer(content):
name = match.group(1)
kind = match.group(2)
line_num = content[:match.start()].count('\n') + 1
symbol_type = SymbolType.STRUCT if kind == "struct" else SymbolType.INTERFACE
symbols.append(Symbol(
name=name,
symbol_type=symbol_type,
file_path=file_path,
line_number=line_num,
signature=f"type {name} {kind}"
))
return symbols
class RustParser(LanguageParser):
"""Parser for Rust files."""
FUNCTION_PATTERN = re.compile(
r'^(\s*)(?:pub\s+)?(?:async\s+)?fn\s+(\w+)\s*(?:<[^>]*>)?\s*\((.*?)\)',
re.MULTILINE
)
STRUCT_PATTERN = re.compile(
r'^(?:pub\s+)?struct\s+(\w+)',
re.MULTILINE
)
IMPL_PATTERN = re.compile(
r'^impl(?:<[^>]*>)?\s+(?:(\w+)\s+for\s+)?(\w+)',
re.MULTILINE
)
USE_PATTERN = re.compile(
r'^use\s+(.+);',
re.MULTILINE
)
def parse(self, content: str, file_path: str) -> List[Symbol]:
symbols = []
# Parse functions
for match in self.FUNCTION_PATTERN.finditer(content):
indent = len(match.group(1))
name = match.group(2)
params = match.group(3)
line_num = content[:match.start()].count('\n') + 1
is_method = indent > 0
symbol_type = SymbolType.METHOD if is_method else SymbolType.FUNCTION
symbols.append(Symbol(
name=name,
symbol_type=symbol_type,
file_path=file_path,
line_number=line_num,
signature=f"fn {name}({params})"
))
# Parse structs
for match in self.STRUCT_PATTERN.finditer(content):
name = match.group(1)
line_num = content[:match.start()].count('\n') + 1
symbols.append(Symbol(
name=name,
symbol_type=SymbolType.STRUCT,
file_path=file_path,
line_number=line_num,
signature=f"struct {name}"
))
# Parse impl blocks
for match in self.IMPL_PATTERN.finditer(content):
trait_name = match.group(1)
type_name = match.group(2)
line_num = content[:match.start()].count('\n') + 1
if trait_name:
sig = f"impl {trait_name} for {type_name}"
else:
sig = f"impl {type_name}"
symbols.append(Symbol(
name=type_name,
symbol_type=SymbolType.TYPE,
file_path=file_path,
line_number=line_num,
signature=sig
))
return symbols
class JavaParser(LanguageParser):
"""Parser for Java files."""
CLASS_PATTERN = re.compile(
r'(?:public\s+)?(?:abstract\s+)?(?:final\s+)?class\s+(\w+)(?:\s+extends\s+(\w+))?(?:\s+implements\s+([^{]+))?',
re.MULTILINE
)
METHOD_PATTERN = re.compile(
r'(?:public|private|protected)?\s*(?:static\s+)?(?:final\s+)?(?:\w+(?:<[^>]+>)?)\s+(\w+)\s*\((.*?)\)',
re.MULTILINE
)
IMPORT_PATTERN = re.compile(
r'^import\s+(?:static\s+)?([^;]+);',
re.MULTILINE
)
def parse(self, content: str, file_path: str) -> List[Symbol]:
symbols = []
# Parse classes
for match in self.CLASS_PATTERN.finditer(content):
name = match.group(1)
extends = match.group(2)
implements = match.group(3)
line_num = content[:match.start()].count('\n') + 1
sig = f"class {name}"
if extends:
sig += f" extends {extends}"
if implements:
sig += f" implements {implements.strip()}"
symbols.append(Symbol(
name=name,
symbol_type=SymbolType.CLASS,
file_path=file_path,
line_number=line_num,
signature=sig
))
# Parse methods
for match in self.METHOD_PATTERN.finditer(content):
name = match.group(1)
params = match.group(2)
line_num = content[:match.start()].count('\n') + 1
# Skip constructors and common keywords
if name in ('if', 'while', 'for', 'switch', 'catch'):
continue
symbols.append(Symbol(
name=name,
symbol_type=SymbolType.METHOD,
file_path=file_path,
line_number=line_num,
signature=f"{name}({params})"
))
# Parse imports
for match in self.IMPORT_PATTERN.finditer(content):
import_path = match.group(1)
line_num = content[:match.start()].count('\n') + 1
# Get the last part of the import
name = import_path.split('.')[-1]
symbols.append(Symbol(
name=name,
symbol_type=SymbolType.IMPORT,
file_path=file_path,
line_number=line_num,
signature=f"import {import_path}"
))
return symbols
class SymbolIndexer:
"""Indexes code symbols from source files.
Supports multiple languages:
- Python (.py)
- JavaScript (.js, .jsx)
- TypeScript (.ts, .tsx)
- Go (.go)
- Rust (.rs)
- Java (.java)
"""
# Map file extensions to parsers
PARSERS = {
'.py': PythonParser,
'.js': JavaScriptParser,
'.jsx': JavaScriptParser,
'.ts': JavaScriptParser,
'.tsx': JavaScriptParser,
'.go': GoParser,
'.rs': RustParser,
'.java': JavaParser,
}
def __init__(self, workspace_path: str):
"""Initialize symbol indexer.
Args:
workspace_path: Root directory to index
"""
self.workspace_path = os.path.abspath(workspace_path)
self.symbols: Dict[str, List[Symbol]] = {} # file_path -> symbols
self.by_name: Dict[str, List[Symbol]] = {} # symbol_name -> symbols
self.by_type: Dict[SymbolType, List[Symbol]] = {} # type -> symbols
# Initialize parsers
self._parsers = {ext: parser() for ext, parser in self.PARSERS.items()}
def index_file(self, file_path: str) -> List[Symbol]:
"""Index symbols from a single file.
Args:
file_path: Path to the file
Returns:
List of extracted symbols
"""
ext = os.path.splitext(file_path)[1].lower()
parser = self._parsers.get(ext)
if not parser:
return []
try:
with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
content = f.read()
rel_path = os.path.relpath(file_path, self.workspace_path)
symbols = parser.parse(content, rel_path)
# Store in indexes
self.symbols[rel_path] = symbols
for sym in symbols:
# By name
if sym.name not in self.by_name:
self.by_name[sym.name] = []
self.by_name[sym.name].append(sym)
# By type
if sym.symbol_type not in self.by_type:
self.by_type[sym.symbol_type] = []
self.by_type[sym.symbol_type].append(sym)
return symbols
except Exception as e:
logger.debug(f"Error indexing {file_path}: {e}")
return []
def index(self) -> int:
"""Index all supported files in workspace.
Returns:
Total number of symbols indexed
"""
total_symbols = 0
for root, dirs, files in os.walk(self.workspace_path):
# Skip hidden directories
dirs[:] = [d for d in dirs if not d.startswith('.')]
for filename in files:
ext = os.path.splitext(filename)[1].lower()
if ext in self.PARSERS:
file_path = os.path.join(root, filename)
symbols = self.index_file(file_path)
total_symbols += len(symbols)
logger.info(f"Indexed {total_symbols} symbols from {len(self.symbols)} files")
return total_symbols
def find_by_name(self, name: str, exact: bool = False) -> List[Symbol]:
"""Find symbols by name.
Args:
name: Symbol name to search
exact: If True, match exact name
Returns:
List of matching symbols
"""
if exact:
return self.by_name.get(name, [])
results = []
name_lower = name.lower()
for sym_name, symbols in self.by_name.items():
if name_lower in sym_name.lower():
results.extend(symbols)
return results
def find_by_type(self, symbol_type: SymbolType) -> List[Symbol]:
"""Find symbols by type.
Args:
symbol_type: Type of symbol
Returns:
List of matching symbols
"""
return self.by_type.get(symbol_type, [])
def find_in_file(self, file_path: str) -> List[Symbol]:
"""Find all symbols in a file.
Args:
file_path: Path to the file
Returns:
List of symbols in the file
"""
return self.symbols.get(file_path, [])
def get_stats(self) -> Dict[str, Any]:
"""Get indexer statistics.
Returns:
Dictionary with stats
"""
type_counts = {t.value: len(syms) for t, syms in self.by_type.items()}
return {
"total_files": len(self.symbols),
"total_symbols": sum(len(s) for s in self.symbols.values()),
"by_type": type_counts
}
@@ -0,0 +1,325 @@
"""
Parallel execution engine for Fast Context.
Provides async parallel execution of search tools with:
- Configurable parallelism (default: 8 concurrent calls)
- Timeout handling per tool call
- Result aggregation
- Error isolation (one failure doesn't affect others)
"""
import asyncio
from concurrent.futures import ThreadPoolExecutor
from typing import List, Dict, Any, Optional, Callable
import logging
import time
logger = logging.getLogger(__name__)
class ToolCallBatch:
"""Batch of tool calls to execute in parallel.
Attributes:
tasks: List of task dictionaries
max_size: Maximum batch size (default: 8)
"""
def __init__(self, max_size: int = 8):
"""Initialize batch.
Args:
max_size: Maximum number of tasks in batch
"""
self.tasks: List[Dict[str, Any]] = []
self.max_size = max_size
def add(self, tool_name: str, **kwargs) -> bool:
"""Add a tool call to the batch.
Args:
tool_name: Name of the tool to call
**kwargs: Tool arguments
Returns:
True if added, False if batch is full
"""
if self.is_full:
return False
self.tasks.append({
"tool": tool_name,
"args": kwargs
})
return True
@property
def is_full(self) -> bool:
"""Check if batch is at max capacity."""
return len(self.tasks) >= self.max_size
def clear(self) -> None:
"""Clear all tasks from batch."""
self.tasks.clear()
class ParallelExecutor:
"""Executes search tools in parallel.
Uses asyncio for concurrent execution with a thread pool
for CPU-bound operations.
Attributes:
max_parallel: Maximum concurrent executions
timeout: Timeout per tool call in seconds
"""
def __init__(
self,
max_parallel: int = 8,
timeout: float = 30.0
):
"""Initialize executor.
Args:
max_parallel: Maximum concurrent tool calls
timeout: Timeout per tool call in seconds
"""
self.max_parallel = max_parallel
self.timeout = timeout
self._executor: Optional[ThreadPoolExecutor] = None
def _get_executor(self) -> ThreadPoolExecutor:
"""Get or create thread pool executor."""
if self._executor is None:
self._executor = ThreadPoolExecutor(max_workers=self.max_parallel)
return self._executor
def _get_tool_function(self, tool_name: str) -> Optional[Callable]:
"""Get tool function by name.
Args:
tool_name: Name of the tool
Returns:
Tool function or None if not found
"""
# Import here to avoid circular imports
from praisonaiagents.context.fast.search_tools import (
grep_search,
glob_search,
read_file,
list_directory
)
tools = {
"grep_search": grep_search,
"glob_search": glob_search,
"read_file": read_file,
"list_directory": list_directory
}
return tools.get(tool_name)
async def _execute_single(
self,
task: Dict[str, Any],
loop: asyncio.AbstractEventLoop
) -> Any:
"""Execute a single tool call.
Args:
task: Task dictionary with 'tool' and 'args'
loop: Event loop for thread execution
Returns:
Tool result or error dictionary
"""
tool_name = task.get("tool")
args = task.get("args", {})
tool_func = self._get_tool_function(tool_name)
if tool_func is None:
return {"error": f"Unknown tool: {tool_name}"}
try:
# Run in thread pool to avoid blocking
executor = self._get_executor()
result = await asyncio.wait_for(
loop.run_in_executor(executor, lambda: tool_func(**args)),
timeout=self.timeout
)
return result
except asyncio.TimeoutError:
return {"error": f"Timeout executing {tool_name}", "timeout": True}
except Exception as e:
logger.debug(f"Error executing {tool_name}: {e}")
return {"error": str(e)}
async def execute(self, tasks: List[Dict[str, Any]]) -> List[Any]:
"""Execute multiple tool calls in parallel.
Args:
tasks: List of task dictionaries, each with 'tool' and 'args'
Returns:
List of results in same order as tasks
"""
if not tasks:
return []
loop = asyncio.get_event_loop()
# Create semaphore to limit concurrency
semaphore = asyncio.Semaphore(self.max_parallel)
async def bounded_execute(task: Dict[str, Any]) -> Any:
async with semaphore:
return await self._execute_single(task, loop)
# Execute all tasks concurrently
results = await asyncio.gather(
*[bounded_execute(task) for task in tasks],
return_exceptions=True
)
# Convert exceptions to error dicts
processed_results = []
for result in results:
if isinstance(result, Exception):
processed_results.append({"error": str(result)})
else:
processed_results.append(result)
return processed_results
def execute_sync(self, tasks: List[Dict[str, Any]]) -> List[Any]:
"""Synchronous wrapper for execute.
Args:
tasks: List of task dictionaries
Returns:
List of results
"""
try:
loop = asyncio.get_event_loop()
if loop.is_running():
# If already in async context, create new loop
import concurrent.futures
with concurrent.futures.ThreadPoolExecutor() as pool:
future = pool.submit(asyncio.run, self.execute(tasks))
return future.result()
else:
return loop.run_until_complete(self.execute(tasks))
except RuntimeError:
# No event loop, create one
return asyncio.run(self.execute(tasks))
def close(self) -> None:
"""Close the executor and release resources."""
if self._executor is not None:
self._executor.shutdown(wait=False)
self._executor = None
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
class ParallelSearchCoordinator:
"""Coordinates parallel search operations across multiple turns.
Implements the Fast Context search strategy:
- Up to 8 parallel tool calls per turn
- Maximum 4 turns
- Aggregates results across turns
"""
def __init__(
self,
max_parallel: int = 8,
max_turns: int = 4,
timeout: float = 30.0
):
"""Initialize coordinator.
Args:
max_parallel: Max parallel calls per turn
max_turns: Maximum number of turns
timeout: Timeout per tool call
"""
self.max_parallel = max_parallel
self.max_turns = max_turns
self.timeout = timeout
self.executor = ParallelExecutor(max_parallel, timeout)
# Track execution stats
self.total_tool_calls = 0
self.turns_used = 0
self.start_time: Optional[float] = None
self.end_time: Optional[float] = None
def start(self) -> None:
"""Start timing the search."""
self.start_time = time.perf_counter()
self.total_tool_calls = 0
self.turns_used = 0
def stop(self) -> None:
"""Stop timing the search."""
self.end_time = time.perf_counter()
@property
def elapsed_ms(self) -> int:
"""Get elapsed time in milliseconds."""
if self.start_time is None:
return 0
end = self.end_time or time.perf_counter()
return int((end - self.start_time) * 1000)
async def execute_turn(self, batch: ToolCallBatch) -> List[Any]:
"""Execute a single turn of parallel tool calls.
Args:
batch: Batch of tool calls
Returns:
List of results
"""
if self.turns_used >= self.max_turns:
logger.warning("Maximum turns reached")
return []
self.turns_used += 1
self.total_tool_calls += len(batch.tasks)
results = await self.executor.execute(batch.tasks)
return results
def execute_turn_sync(self, batch: ToolCallBatch) -> List[Any]:
"""Synchronous wrapper for execute_turn.
Args:
batch: Batch of tool calls
Returns:
List of results
"""
try:
loop = asyncio.get_event_loop()
if loop.is_running():
import concurrent.futures
with concurrent.futures.ThreadPoolExecutor() as pool:
future = pool.submit(asyncio.run, self.execute_turn(batch))
return future.result()
else:
return loop.run_until_complete(self.execute_turn(batch))
except RuntimeError:
return asyncio.run(self.execute_turn(batch))
def close(self) -> None:
"""Close resources."""
self.executor.close()
@@ -0,0 +1,210 @@
"""
Result classes for Fast Context.
These dataclasses represent the output of Fast Context searches,
providing structured file and line range information.
"""
from dataclasses import dataclass, field
from typing import List, Optional
import time
@dataclass
class LineRange:
"""Represents a range of lines in a file.
Attributes:
start: Starting line number (1-indexed)
end: Ending line number (1-indexed, inclusive)
content: Optional content of the lines
relevance_score: Score indicating relevance (0.0-1.0)
"""
start: int
end: int
content: Optional[str] = None
relevance_score: float = 1.0
def __post_init__(self):
if self.start < 1:
self.start = 1
if self.end < self.start:
self.end = self.start
@property
def line_count(self) -> int:
"""Number of lines in this range."""
return self.end - self.start + 1
def overlaps(self, other: "LineRange") -> bool:
"""Check if this range overlaps with another."""
return not (self.end < other.start or self.start > other.end)
def merge(self, other: "LineRange") -> "LineRange":
"""Merge with another overlapping range."""
if not self.overlaps(other):
raise ValueError("Cannot merge non-overlapping ranges")
return LineRange(
start=min(self.start, other.start),
end=max(self.end, other.end),
content=None, # Content needs to be re-fetched
relevance_score=max(self.relevance_score, other.relevance_score)
)
@dataclass
class FileMatch:
"""Represents a file match from Fast Context search.
Attributes:
path: Absolute or relative path to the file
line_ranges: List of relevant line ranges in the file
relevance_score: Overall relevance score (0.0-1.0)
match_count: Number of pattern matches found
"""
path: str
line_ranges: List[LineRange] = field(default_factory=list)
relevance_score: float = 1.0
match_count: int = 0
def add_line_range(self, line_range: LineRange) -> None:
"""Add a line range, merging with existing overlapping ranges."""
# Find overlapping ranges
overlapping = []
non_overlapping = []
for existing in self.line_ranges:
if existing.overlaps(line_range):
overlapping.append(existing)
else:
non_overlapping.append(existing)
# Merge all overlapping ranges
merged = line_range
for overlap in overlapping:
merged = merged.merge(overlap)
self.line_ranges = non_overlapping + [merged]
self.line_ranges.sort(key=lambda r: r.start)
@property
def total_lines(self) -> int:
"""Total number of lines across all ranges."""
return sum(r.line_count for r in self.line_ranges)
@dataclass
class FastContextResult:
"""Result of a Fast Context search operation.
Attributes:
files: List of matching files with line ranges
query: The original search query
search_time_ms: Time taken for the search in milliseconds
turns_used: Number of turns used by the subagent
total_tool_calls: Total number of tool calls made
from_cache: Whether result was retrieved from cache
"""
files: List[FileMatch] = field(default_factory=list)
query: str = ""
search_time_ms: int = 0
turns_used: int = 0
total_tool_calls: int = 0
from_cache: bool = False
_start_time: float = field(default=0.0, repr=False)
def start_timer(self) -> None:
"""Start the search timer."""
self._start_time = time.perf_counter()
def stop_timer(self) -> None:
"""Stop the timer and record elapsed time."""
if self._start_time > 0:
elapsed = time.perf_counter() - self._start_time
self.search_time_ms = int(elapsed * 1000)
def add_file(self, file_match: FileMatch) -> None:
"""Add a file match, merging with existing if same path."""
for existing in self.files:
if existing.path == file_match.path:
# Merge line ranges
for lr in file_match.line_ranges:
existing.add_line_range(lr)
existing.relevance_score = max(
existing.relevance_score,
file_match.relevance_score
)
existing.match_count += file_match.match_count
return
self.files.append(file_match)
def sort_by_relevance(self) -> None:
"""Sort files by relevance score (highest first)."""
self.files.sort(key=lambda f: f.relevance_score, reverse=True)
@property
def total_files(self) -> int:
"""Total number of files found."""
return len(self.files)
@property
def total_lines(self) -> int:
"""Total number of lines across all files."""
return sum(f.total_lines for f in self.files)
def to_context_string(self, max_files: int = 10, include_content: bool = True) -> str:
"""Convert result to a context string for the main agent.
Args:
max_files: Maximum number of files to include
include_content: Whether to include line content
Returns:
Formatted context string
"""
if not self.files:
return "No relevant files found."
lines = [f"Found {self.total_files} relevant file(s):"]
for i, file_match in enumerate(self.files[:max_files]):
lines.append(f"\n## {file_match.path}")
if file_match.line_ranges:
for lr in file_match.line_ranges:
lines.append(f" Lines {lr.start}-{lr.end}")
if include_content and lr.content:
# Indent content
content_lines = lr.content.split('\n')
for cl in content_lines:
lines.append(f" {cl}")
if len(self.files) > max_files:
lines.append(f"\n... and {len(self.files) - max_files} more files")
return '\n'.join(lines)
def to_dict(self) -> dict:
"""Convert to dictionary for serialization."""
return {
"files": [
{
"path": f.path,
"relevance_score": f.relevance_score,
"match_count": f.match_count,
"line_ranges": [
{
"start": lr.start,
"end": lr.end,
"relevance_score": lr.relevance_score
}
for lr in f.line_ranges
]
}
for f in self.files
],
"query": self.query,
"search_time_ms": self.search_time_ms,
"turns_used": self.turns_used,
"total_tool_calls": self.total_tool_calls,
"from_cache": self.from_cache
}
@@ -0,0 +1,368 @@
"""
Search Strategy for Fast Context.
Orchestrates multi-turn search with:
- Turn 1-3: Exploration (grep, glob, directory listing)
- Turn 4: Final answer (file list + line ranges)
- Early termination if sufficient context found
"""
import logging
from dataclasses import dataclass, field
from typing import List, Dict, Any, Optional, Set
from enum import Enum
from praisonaiagents.context.fast.result import FastContextResult, FileMatch, LineRange
from praisonaiagents.context.fast.parallel_executor import ToolCallBatch
logger = logging.getLogger(__name__)
class SearchPhase(Enum):
"""Phases of the search strategy."""
DISCOVERY = "discovery" # Turn 1: Discover structure
EXPLORATION = "exploration" # Turn 2-3: Deep search
REFINEMENT = "refinement" # Turn 4: Final answer
@dataclass
class SearchState:
"""Tracks the state of a multi-turn search.
Attributes:
query: Original search query
current_turn: Current turn number (1-indexed)
max_turns: Maximum turns allowed
phase: Current search phase
discovered_files: Files found so far
explored_paths: Paths already explored
result: Accumulated search result
should_terminate: Whether to terminate early
"""
query: str
current_turn: int = 0
max_turns: int = 4
phase: SearchPhase = SearchPhase.DISCOVERY
discovered_files: Set[str] = field(default_factory=set)
explored_paths: Set[str] = field(default_factory=set)
result: FastContextResult = field(default_factory=FastContextResult)
should_terminate: bool = False
# Thresholds for early termination
min_files_for_termination: int = 5
max_files_before_refinement: int = 20
def advance_turn(self) -> None:
"""Advance to the next turn and update phase."""
self.current_turn += 1
if self.current_turn == 1:
self.phase = SearchPhase.DISCOVERY
elif self.current_turn < self.max_turns:
self.phase = SearchPhase.EXPLORATION
else:
self.phase = SearchPhase.REFINEMENT
def check_early_termination(self) -> bool:
"""Check if search should terminate early.
Returns:
True if search should terminate
"""
# Terminate if we have enough high-quality results
if len(self.discovered_files) >= self.min_files_for_termination:
high_relevance = sum(
1 for f in self.result.files
if f.relevance_score >= 0.8
)
if high_relevance >= 3:
self.should_terminate = True
return True
# Terminate if we've explored everything
if self.current_turn >= 2 and not self._has_unexplored_paths():
self.should_terminate = True
return True
return False
def _has_unexplored_paths(self) -> bool:
"""Check if there are unexplored paths."""
# Simple heuristic: if we found files but haven't explored many paths
return len(self.discovered_files) > len(self.explored_paths) * 2
class SearchStrategy:
"""Orchestrates multi-turn search strategy.
Implements the Fast Context search approach:
- Turn 1: Discovery - understand codebase structure
- Turn 2-3: Exploration - deep search for relevant code
- Turn 4: Refinement - finalize results
Supports early termination when sufficient context is found.
"""
def __init__(
self,
workspace_path: str,
max_turns: int = 4,
max_parallel: int = 8
):
"""Initialize search strategy.
Args:
workspace_path: Root directory for searches
max_turns: Maximum search turns
max_parallel: Maximum parallel tool calls per turn
"""
self.workspace_path = workspace_path
self.max_turns = max_turns
self.max_parallel = max_parallel
def create_state(self, query: str) -> SearchState:
"""Create a new search state.
Args:
query: Search query
Returns:
New SearchState instance
"""
state = SearchState(query=query, max_turns=self.max_turns)
state.result.query = query
return state
def plan_discovery_turn(self, state: SearchState) -> ToolCallBatch:
"""Plan the discovery turn (Turn 1).
Focus on understanding codebase structure:
- List root directory
- Find common file patterns
- Initial grep for query terms
Args:
state: Current search state
Returns:
Batch of tool calls to execute
"""
batch = ToolCallBatch(max_size=self.max_parallel)
query = state.query
# List root directory
batch.add("list_directory", dir_path=self.workspace_path, recursive=False)
# Find Python files (most common)
batch.add("glob_search", search_path=self.workspace_path, pattern="**/*.py", max_results=20)
# Initial grep for query terms
query_words = query.lower().split()
for word in query_words[:3]: # First 3 words
if len(word) >= 3: # Skip short words
batch.add("grep_search", search_path=self.workspace_path, pattern=word, max_results=10)
if batch.is_full:
break
return batch
def plan_exploration_turn(self, state: SearchState) -> ToolCallBatch:
"""Plan an exploration turn (Turn 2-3).
Deep search based on discovery results:
- Search in discovered directories
- Read promising files
- Follow references
Args:
state: Current search state
Returns:
Batch of tool calls to execute
"""
batch = ToolCallBatch(max_size=self.max_parallel)
# Read top files found so far
for file_match in state.result.files[:4]:
if file_match.path not in state.explored_paths:
batch.add("read_file", filepath=file_match.path, context_lines=5)
state.explored_paths.add(file_match.path)
if batch.is_full:
break
# Search in unexplored directories
unexplored_dirs = self._find_unexplored_dirs(state)
for dir_path in unexplored_dirs[:2]:
batch.add("list_directory", dir_path=dir_path, recursive=True, max_depth=2)
if batch.is_full:
break
# Additional grep searches with refined patterns
if not batch.is_full:
refined_patterns = self._generate_refined_patterns(state)
for pattern in refined_patterns[:2]:
batch.add("grep_search", search_path=self.workspace_path, pattern=pattern, max_results=15)
if batch.is_full:
break
return batch
def plan_refinement_turn(self, state: SearchState) -> ToolCallBatch:
"""Plan the refinement turn (Turn 4).
Finalize results:
- Read remaining important files
- Get specific line ranges
Args:
state: Current search state
Returns:
Batch of tool calls to execute
"""
batch = ToolCallBatch(max_size=self.max_parallel)
# Read top files that haven't been fully explored
for file_match in state.result.files[:self.max_parallel]:
if file_match.path not in state.explored_paths:
# Read specific line ranges if available
if file_match.line_ranges:
for lr in file_match.line_ranges[:2]:
batch.add(
"read_file",
filepath=file_match.path,
start_line=max(1, lr.start - 5),
end_line=lr.end + 5
)
if batch.is_full:
break
else:
batch.add("read_file", filepath=file_match.path)
state.explored_paths.add(file_match.path)
if batch.is_full:
break
return batch
def plan_next_turn(self, state: SearchState) -> Optional[ToolCallBatch]:
"""Plan the next turn based on current state.
Args:
state: Current search state
Returns:
Batch of tool calls, or None if search should end
"""
state.advance_turn()
# Check for early termination
if state.should_terminate or state.current_turn > state.max_turns:
return None
if state.check_early_termination():
return None
# Plan based on phase
if state.phase == SearchPhase.DISCOVERY:
return self.plan_discovery_turn(state)
elif state.phase == SearchPhase.EXPLORATION:
return self.plan_exploration_turn(state)
else: # REFINEMENT
return self.plan_refinement_turn(state)
def process_results(
self,
state: SearchState,
tool_results: List[Any]
) -> None:
"""Process results from a turn and update state.
Args:
state: Current search state
tool_results: Results from tool execution
"""
for result in tool_results:
if isinstance(result, list):
# Grep or glob results
for item in result:
if isinstance(item, dict):
path = item.get("path", "")
if path:
state.discovered_files.add(path)
# Create or update file match
file_match = FileMatch(
path=path,
relevance_score=0.8,
match_count=1
)
# Add line range if available
line_num = item.get("line_number")
if line_num:
file_match.add_line_range(LineRange(
start=max(1, line_num - 2),
end=line_num + 2,
content=item.get("context"),
relevance_score=0.8
))
state.result.add_file(file_match)
elif isinstance(result, dict):
# Read file or list directory result
if result.get("success"):
path = result.get("path", "")
if path:
state.explored_paths.add(path)
# Handle directory listing
entries = result.get("entries", [])
for entry in entries:
if not entry.get("is_dir"):
state.discovered_files.add(entry.get("path", ""))
def _find_unexplored_dirs(self, state: SearchState) -> List[str]:
"""Find directories that haven't been explored.
Args:
state: Current search state
Returns:
List of unexplored directory paths
"""
import os
# Extract unique directories from discovered files
dirs = set()
for file_path in state.discovered_files:
dir_path = os.path.dirname(file_path)
if dir_path and dir_path not in state.explored_paths:
dirs.add(dir_path)
return list(dirs)[:5] # Limit to 5
def _generate_refined_patterns(self, state: SearchState) -> List[str]:
"""Generate refined search patterns based on results.
Args:
state: Current search state
Returns:
List of refined search patterns
"""
patterns = []
query_words = state.query.lower().split()
# Combine query words
if len(query_words) >= 2:
patterns.append(f"{query_words[0]}.*{query_words[1]}")
# Add common code patterns
for word in query_words[:2]:
if len(word) >= 3:
patterns.append(f"def {word}")
patterns.append(f"class {word}")
return patterns[:4]
@@ -0,0 +1,636 @@
"""
Search tools for Fast Context.
Provides cross-platform compatible search tools:
- grep_search: Pattern search in files
- glob_search: File pattern matching
- read_file: Read file contents with line range support
- list_directory: List directory contents
These tools are designed to be fast, safe, and cross-platform compatible.
"""
import os
import re
import fnmatch
from pathlib import Path
from typing import List, Dict, Any, Optional, Set
import logging
logger = logging.getLogger(__name__)
def _load_gitignore_patterns(root_path: str) -> Set[str]:
"""Load patterns from .gitignore file.
Args:
root_path: Root directory to search for .gitignore
Returns:
Set of gitignore patterns
"""
patterns = set()
gitignore_path = Path(root_path) / ".gitignore"
if gitignore_path.exists():
try:
with open(gitignore_path, 'r', encoding='utf-8', errors='ignore') as f:
for line in f:
line = line.strip()
# Skip empty lines and comments
if line and not line.startswith('#'):
patterns.add(line)
except Exception:
pass
# Also check for .praisonignore
praisonignore_path = Path(root_path) / ".praisonignore"
if praisonignore_path.exists():
try:
with open(praisonignore_path, 'r', encoding='utf-8', errors='ignore') as f:
for line in f:
line = line.strip()
if line and not line.startswith('#'):
patterns.add(line)
except Exception:
pass
return patterns
def _should_ignore(path: str, root_path: str, patterns: Set[str]) -> bool:
"""Check if a path should be ignored based on gitignore patterns.
Args:
path: Path to check
root_path: Root directory
patterns: Set of gitignore patterns
Returns:
True if path should be ignored
"""
if not patterns:
return False
rel_path = os.path.relpath(path, root_path)
path_parts = Path(rel_path).parts
for pattern in patterns:
# Handle directory patterns (ending with /)
if pattern.endswith('/'):
dir_pattern = pattern.rstrip('/')
if any(fnmatch.fnmatch(part, dir_pattern) for part in path_parts):
return True
# Handle ** patterns
elif '**' in pattern:
# Convert gitignore pattern to fnmatch pattern
fnmatch_pattern = pattern.replace('**/', '*').replace('**', '*')
if fnmatch.fnmatch(rel_path, fnmatch_pattern):
return True
# Handle simple patterns
else:
# Check against filename
if fnmatch.fnmatch(os.path.basename(path), pattern):
return True
# Check against relative path
if fnmatch.fnmatch(rel_path, pattern):
return True
# Check if any path component matches
if any(fnmatch.fnmatch(part, pattern) for part in path_parts):
return True
return False
def _is_binary_file(filepath: str) -> bool:
"""Check if a file is binary.
Args:
filepath: Path to the file
Returns:
True if file appears to be binary
"""
try:
with open(filepath, 'rb') as f:
chunk = f.read(8192)
# Check for null bytes (common in binary files)
if b'\x00' in chunk:
return True
# Check for high ratio of non-text characters
text_chars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f})
non_text = sum(1 for byte in chunk if byte not in text_chars)
if len(chunk) > 0 and non_text / len(chunk) > 0.3:
return True
except Exception:
return True
return False
def grep_search(
search_path: str,
pattern: str,
is_regex: bool = False,
case_sensitive: bool = False,
include_patterns: Optional[List[str]] = None,
exclude_patterns: Optional[List[str]] = None,
max_results: int = 100,
context_lines: int = 0,
respect_gitignore: bool = True
) -> List[Dict[str, Any]]:
"""Search for pattern in files.
Args:
search_path: Directory or file to search
pattern: Search pattern (string or regex)
is_regex: If True, treat pattern as regex
case_sensitive: If True, search is case sensitive
include_patterns: Glob patterns to include (e.g., ["*.py"])
exclude_patterns: Glob patterns to exclude (e.g., ["**/test_*"])
max_results: Maximum number of results to return
context_lines: Number of context lines before/after match
respect_gitignore: If True, respect .gitignore patterns
Returns:
List of match dictionaries with path, line_number, content, context
"""
results = []
search_path = os.path.abspath(search_path)
# Compile pattern
if is_regex:
flags = 0 if case_sensitive else re.IGNORECASE
try:
regex = re.compile(pattern, flags)
except re.error as e:
logger.warning(f"Invalid regex pattern: {e}")
return []
else:
if case_sensitive:
regex = re.compile(re.escape(pattern))
else:
regex = re.compile(re.escape(pattern), re.IGNORECASE)
# Load gitignore patterns
gitignore_patterns = set()
if respect_gitignore:
gitignore_patterns = _load_gitignore_patterns(search_path)
# Get files to search
if os.path.isfile(search_path):
files_to_search = [search_path]
else:
files_to_search = []
for root, dirs, files in os.walk(search_path):
# Filter directories in-place to skip ignored ones
if respect_gitignore:
dirs[:] = [d for d in dirs if not _should_ignore(
os.path.join(root, d), search_path, gitignore_patterns
)]
for filename in files:
filepath = os.path.join(root, filename)
# Check gitignore
if respect_gitignore and _should_ignore(filepath, search_path, gitignore_patterns):
continue
# Check include patterns
if include_patterns:
rel_path = os.path.relpath(filepath, search_path)
if not any(fnmatch.fnmatch(rel_path, p) or fnmatch.fnmatch(filename, p)
for p in include_patterns):
continue
# Check exclude patterns
if exclude_patterns:
rel_path = os.path.relpath(filepath, search_path)
if any(fnmatch.fnmatch(rel_path, p) or fnmatch.fnmatch(filename, p)
for p in exclude_patterns):
continue
files_to_search.append(filepath)
# Search files
for filepath in files_to_search:
if len(results) >= max_results:
break
# Skip binary files
if _is_binary_file(filepath):
continue
try:
with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
lines = f.readlines()
for i, line in enumerate(lines):
if len(results) >= max_results:
break
if regex.search(line):
# Get context
start_ctx = max(0, i - context_lines)
end_ctx = min(len(lines), i + context_lines + 1)
context = ''.join(lines[start_ctx:end_ctx])
results.append({
"path": os.path.relpath(filepath, search_path),
"absolute_path": filepath,
"line_number": i + 1, # 1-indexed
"content": line.rstrip('\n\r'),
"context": context.rstrip('\n\r') if context_lines > 0 else None,
"context_start": start_ctx + 1 if context_lines > 0 else None,
"context_end": end_ctx if context_lines > 0 else None
})
except Exception as e:
logger.debug(f"Error reading {filepath}: {e}")
continue
return results
def glob_search(
search_path: str,
pattern: str,
max_results: int = 100,
include_dirs: bool = False,
respect_gitignore: bool = True
) -> List[Dict[str, Any]]:
"""Search for files matching glob pattern.
Args:
search_path: Directory to search
pattern: Glob pattern (e.g., "**/*.py", "src/*.js")
max_results: Maximum number of results
include_dirs: If True, include directories in results
respect_gitignore: If True, respect .gitignore patterns
Returns:
List of file info dictionaries with path, size, is_dir
"""
results = []
search_path = os.path.abspath(search_path)
# Load gitignore patterns
gitignore_patterns = set()
if respect_gitignore:
gitignore_patterns = _load_gitignore_patterns(search_path)
# Use pathlib for glob
root = Path(search_path)
try:
for match in root.glob(pattern):
if len(results) >= max_results:
break
# Check gitignore
if respect_gitignore and _should_ignore(str(match), search_path, gitignore_patterns):
continue
# Skip directories unless requested
if match.is_dir() and not include_dirs:
continue
# Skip binary files for file results
if match.is_file() and _is_binary_file(str(match)):
continue
try:
stat = match.stat()
results.append({
"path": str(match.relative_to(root)),
"absolute_path": str(match),
"size": stat.st_size if match.is_file() else None,
"is_dir": match.is_dir(),
"modified": stat.st_mtime
})
except Exception:
continue
except Exception as e:
logger.debug(f"Error in glob search: {e}")
return results
def read_file(
filepath: str,
start_line: Optional[int] = None,
end_line: Optional[int] = None,
context_lines: int = 0,
max_lines: int = 500
) -> Dict[str, Any]:
"""Read file contents with optional line range.
Args:
filepath: Path to the file
start_line: Starting line (1-indexed, inclusive)
end_line: Ending line (1-indexed, inclusive)
context_lines: Additional context lines before/after range
max_lines: Maximum lines to return
Returns:
Dictionary with content, line info, and metadata
"""
filepath = os.path.abspath(filepath)
if not os.path.exists(filepath):
return {
"success": False,
"error": f"File not found: {filepath}",
"path": filepath
}
if os.path.isdir(filepath):
return {
"success": False,
"error": f"Path is a directory: {filepath}",
"path": filepath
}
if _is_binary_file(filepath):
return {
"success": False,
"error": f"Binary file: {filepath}",
"path": filepath
}
try:
with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
lines = f.readlines()
total_lines = len(lines)
# Determine line range
if start_line is None:
start_line = 1
if end_line is None:
end_line = total_lines
# Apply context
actual_start = max(1, start_line - context_lines)
actual_end = min(total_lines, end_line + context_lines)
# Limit lines
if actual_end - actual_start + 1 > max_lines:
actual_end = actual_start + max_lines - 1
# Extract content (convert to 0-indexed)
content_lines = lines[actual_start - 1:actual_end]
content = ''.join(content_lines)
return {
"success": True,
"path": filepath,
"content": content.rstrip('\n'),
"start_line": actual_start,
"end_line": actual_end,
"total_lines": total_lines,
"lines_returned": len(content_lines)
}
except Exception as e:
return {
"success": False,
"error": str(e),
"path": filepath
}
def list_directory(
dir_path: str,
recursive: bool = False,
max_depth: int = 10,
max_entries: int = 500,
include_hidden: bool = False,
respect_gitignore: bool = True
) -> Dict[str, Any]:
"""List directory contents.
Args:
dir_path: Directory path
recursive: If True, list recursively
max_depth: Maximum recursion depth
max_entries: Maximum entries to return
include_hidden: If True, include hidden files/dirs
respect_gitignore: If True, respect .gitignore patterns
Returns:
Dictionary with entries list and metadata
"""
dir_path = os.path.abspath(dir_path)
if not os.path.exists(dir_path):
return {
"success": False,
"error": f"Directory not found: {dir_path}",
"path": dir_path
}
if not os.path.isdir(dir_path):
return {
"success": False,
"error": f"Not a directory: {dir_path}",
"path": dir_path
}
# Load gitignore patterns
gitignore_patterns = set()
if respect_gitignore:
gitignore_patterns = _load_gitignore_patterns(dir_path)
entries = []
def _list_dir(path: str, depth: int):
if depth > max_depth or len(entries) >= max_entries:
return
try:
for entry in os.scandir(path):
if len(entries) >= max_entries:
break
# Skip hidden files unless requested
if not include_hidden and entry.name.startswith('.'):
continue
# Check gitignore
if respect_gitignore and _should_ignore(entry.path, dir_path, gitignore_patterns):
continue
try:
stat = entry.stat()
entries.append({
"name": entry.name,
"path": os.path.relpath(entry.path, dir_path),
"absolute_path": entry.path,
"is_dir": entry.is_dir(),
"size": stat.st_size if entry.is_file() else None,
"modified": stat.st_mtime
})
except Exception:
continue
# Recurse into directories
if recursive and entry.is_dir():
_list_dir(entry.path, depth + 1)
except PermissionError:
pass
except Exception as e:
logger.debug(f"Error listing {path}: {e}")
_list_dir(dir_path, 0)
# Sort entries: directories first, then by name
entries.sort(key=lambda e: (not e["is_dir"], e["name"].lower()))
return {
"success": True,
"path": dir_path,
"entries": entries,
"total_entries": len(entries),
"truncated": len(entries) >= max_entries
}
# Tool definitions for LLM function calling
FAST_CONTEXT_TOOLS = [
{
"name": "grep_search",
"description": "Search for a pattern in files within a directory. Returns matching lines with file paths and line numbers.",
"parameters": {
"type": "object",
"properties": {
"search_path": {
"type": "string",
"description": "Directory or file path to search"
},
"pattern": {
"type": "string",
"description": "Search pattern (text or regex)"
},
"is_regex": {
"type": "boolean",
"description": "If true, treat pattern as regex",
"default": False
},
"case_sensitive": {
"type": "boolean",
"description": "If true, search is case sensitive",
"default": False
},
"include_patterns": {
"type": "array",
"items": {"type": "string"},
"description": "Glob patterns to include (e.g., ['*.py'])"
},
"max_results": {
"type": "integer",
"description": "Maximum results to return",
"default": 50
}
},
"required": ["search_path", "pattern"]
}
},
{
"name": "glob_search",
"description": "Find files matching a glob pattern. Returns file paths and metadata.",
"parameters": {
"type": "object",
"properties": {
"search_path": {
"type": "string",
"description": "Directory to search"
},
"pattern": {
"type": "string",
"description": "Glob pattern (e.g., '**/*.py', 'src/*.js')"
},
"max_results": {
"type": "integer",
"description": "Maximum results to return",
"default": 50
}
},
"required": ["search_path", "pattern"]
}
},
{
"name": "read_file",
"description": "Read contents of a file, optionally with specific line range.",
"parameters": {
"type": "object",
"properties": {
"filepath": {
"type": "string",
"description": "Path to the file"
},
"start_line": {
"type": "integer",
"description": "Starting line number (1-indexed)"
},
"end_line": {
"type": "integer",
"description": "Ending line number (1-indexed)"
},
"context_lines": {
"type": "integer",
"description": "Additional context lines",
"default": 0
}
},
"required": ["filepath"]
}
},
{
"name": "list_directory",
"description": "List contents of a directory.",
"parameters": {
"type": "object",
"properties": {
"dir_path": {
"type": "string",
"description": "Directory path"
},
"recursive": {
"type": "boolean",
"description": "If true, list recursively",
"default": False
},
"max_depth": {
"type": "integer",
"description": "Maximum recursion depth",
"default": 3
}
},
"required": ["dir_path"]
}
}
]
def execute_tool(tool_name: str, **kwargs) -> Dict[str, Any]:
"""Execute a search tool by name.
Args:
tool_name: Name of the tool to execute
**kwargs: Tool arguments
Returns:
Tool result dictionary
"""
tools = {
"grep_search": grep_search,
"glob_search": glob_search,
"read_file": read_file,
"list_directory": list_directory
}
if tool_name not in tools:
return {"error": f"Unknown tool: {tool_name}"}
try:
return tools[tool_name](**kwargs)
except Exception as e:
return {"error": str(e)}
+1 -1
View File
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "praisonaiagents"
version = "0.0.189"
version = "0.0.190"
description = "Praison AI agents for completing complex tasks with Self Reflection Agents"
requires-python = ">=3.10"
authors = [
Whitespace-only changes.
@@ -0,0 +1,252 @@
"""
Tests for FastContext high-level API.
"""
import pytest
import tempfile
import time
from pathlib import Path
@pytest.fixture
def temp_workspace():
"""Create a temporary workspace with test files."""
with tempfile.TemporaryDirectory() as tmpdir:
# Create directory structure
src_dir = Path(tmpdir) / "src"
src_dir.mkdir()
(src_dir / "main.py").write_text("""
def main():
print("Hello, World!")
authenticate_user()
def authenticate_user():
# Authentication logic here
pass
class UserService:
def login(self, username, password):
return True
""")
(src_dir / "utils.py").write_text("""
import os
def get_config():
return {"debug": True}
def validate_input(data):
if not data:
raise ValueError("Empty data")
return True
""")
auth_dir = src_dir / "auth"
auth_dir.mkdir()
(auth_dir / "handler.py").write_text("""
from typing import Optional
class AuthHandler:
def __init__(self):
self.token = None
def authenticate(self, credentials: dict) -> bool:
return True
def logout(self):
self.token = None
""")
yield tmpdir
class TestFastContextInit:
"""Tests for FastContext initialization."""
def test_default_initialization(self, temp_workspace):
"""Test default initialization."""
from praisonaiagents.context.fast import FastContext
fc = FastContext(workspace_path=temp_workspace)
assert fc.workspace_path == temp_workspace
assert fc.max_turns == 4
assert fc.max_parallel == 8
assert fc.model == "gpt-4o-mini"
assert fc.cache_enabled is True
def test_custom_initialization(self, temp_workspace):
"""Test custom initialization."""
from praisonaiagents.context.fast import FastContext
fc = FastContext(
workspace_path=temp_workspace,
max_turns=3,
max_parallel=4,
model="claude-3-haiku-20240307",
cache_enabled=False
)
assert fc.max_turns == 3
assert fc.max_parallel == 4
assert fc.model == "claude-3-haiku-20240307"
assert fc.cache_enabled is False
def test_context_manager(self, temp_workspace):
"""Test context manager usage."""
from praisonaiagents.context.fast import FastContext
with FastContext(workspace_path=temp_workspace) as fc:
result = fc.search("def")
assert result.total_files > 0
class TestFastContextSearch:
"""Tests for FastContext search functionality."""
def test_search_basic(self, temp_workspace):
"""Test basic search."""
from praisonaiagents.context.fast import FastContext
fc = FastContext(workspace_path=temp_workspace)
result = fc.search("authenticate")
assert result.total_files > 0
file_paths = [f.path for f in result.files]
assert any("main.py" in p for p in file_paths)
def test_search_with_patterns(self, temp_workspace):
"""Test search with include patterns."""
from praisonaiagents.context.fast import FastContext
fc = FastContext(workspace_path=temp_workspace)
result = fc.search("def", include_patterns=["**/auth/*.py"])
for f in result.files:
assert "auth" in f.path
def test_search_no_results(self, temp_workspace):
"""Test search with no matches."""
from praisonaiagents.context.fast import FastContext
fc = FastContext(workspace_path=temp_workspace)
result = fc.search("nonexistent_xyz_123")
assert result.total_files == 0
class TestFastContextCache:
"""Tests for FastContext caching."""
def test_cache_hit(self, temp_workspace):
"""Test cache hit on repeated query."""
from praisonaiagents.context.fast import FastContext
fc = FastContext(workspace_path=temp_workspace, cache_enabled=True)
# First search
result1 = fc.search("authenticate")
assert result1.from_cache is False
# Second search (should hit cache)
result2 = fc.search("authenticate")
assert result2.from_cache is True
def test_cache_disabled(self, temp_workspace):
"""Test cache disabled."""
from praisonaiagents.context.fast import FastContext
fc = FastContext(workspace_path=temp_workspace, cache_enabled=False)
result1 = fc.search("authenticate")
result2 = fc.search("authenticate")
assert result1.from_cache is False
assert result2.from_cache is False
def test_clear_cache(self, temp_workspace):
"""Test clearing cache."""
from praisonaiagents.context.fast import FastContext
fc = FastContext(workspace_path=temp_workspace, cache_enabled=True)
fc.search("authenticate")
fc.clear_cache()
result = fc.search("authenticate")
assert result.from_cache is False
class TestFastContextFiles:
"""Tests for FastContext file operations."""
def test_search_files(self, temp_workspace):
"""Test searching for files."""
from praisonaiagents.context.fast import FastContext
fc = FastContext(workspace_path=temp_workspace)
result = fc.search_files("**/*.py")
assert result.total_files >= 3
def test_read_context(self, temp_workspace):
"""Test reading file context."""
from praisonaiagents.context.fast import FastContext
fc = FastContext(workspace_path=temp_workspace)
content = fc.read_context("src/main.py")
assert content is not None
assert "def main():" in content
def test_read_context_with_lines(self, temp_workspace):
"""Test reading specific lines."""
from praisonaiagents.context.fast import FastContext
fc = FastContext(workspace_path=temp_workspace)
content = fc.read_context("src/main.py", start_line=1, end_line=5, context_lines=0)
assert content is not None
lines = content.split("\n")
assert len(lines) <= 5
class TestFastContextAgentIntegration:
"""Tests for FastContext agent integration."""
def test_get_context_for_agent(self, temp_workspace):
"""Test getting formatted context for agent."""
from praisonaiagents.context.fast import FastContext
fc = FastContext(workspace_path=temp_workspace)
context = fc.get_context_for_agent("authenticate")
assert "Relevant Code Context" in context
assert "authenticate" in context.lower()
def test_get_context_for_agent_no_results(self, temp_workspace):
"""Test context for agent with no results."""
from praisonaiagents.context.fast import FastContext
fc = FastContext(workspace_path=temp_workspace)
context = fc.get_context_for_agent("nonexistent_xyz_123")
assert "No relevant code found" in context
class TestFastSearchFunction:
"""Tests for fast_search convenience function."""
def test_fast_search(self, temp_workspace):
"""Test fast_search function."""
from praisonaiagents.context.fast.fast_context import fast_search
result = fast_search("authenticate", workspace_path=temp_workspace)
assert result.total_files > 0
if __name__ == "__main__":
pytest.main([__file__, "-v"])
@@ -0,0 +1,205 @@
"""
Tests for FastContextAgent class.
"""
import pytest
import tempfile
from pathlib import Path
@pytest.fixture
def temp_workspace():
"""Create a temporary workspace with test files."""
with tempfile.TemporaryDirectory() as tmpdir:
# Create directory structure
src_dir = Path(tmpdir) / "src"
src_dir.mkdir()
(src_dir / "main.py").write_text("""
def main():
print("Hello, World!")
authenticate_user()
def authenticate_user():
# Authentication logic here
pass
class UserService:
def login(self, username, password):
return True
""")
(src_dir / "utils.py").write_text("""
import os
def get_config():
return {"debug": True}
def validate_input(data):
if not data:
raise ValueError("Empty data")
return True
""")
auth_dir = src_dir / "auth"
auth_dir.mkdir()
(auth_dir / "handler.py").write_text("""
from typing import Optional
class AuthHandler:
def __init__(self):
self.token = None
def authenticate(self, credentials: dict) -> bool:
return True
def logout(self):
self.token = None
""")
yield tmpdir
class TestFastContextAgentInit:
"""Tests for FastContextAgent initialization."""
def test_default_initialization(self, temp_workspace):
"""Test default initialization."""
from praisonaiagents.context.fast.fast_context_agent import FastContextAgent
agent = FastContextAgent(workspace_path=temp_workspace)
assert agent.workspace_path == temp_workspace
assert agent.max_turns == 4
assert agent.max_parallel == 8
assert agent.model == "gpt-4o-mini"
def test_custom_initialization(self, temp_workspace):
"""Test custom initialization."""
from praisonaiagents.context.fast.fast_context_agent import FastContextAgent
agent = FastContextAgent(
workspace_path=temp_workspace,
max_turns=3,
max_parallel=4,
model="claude-3-haiku-20240307"
)
assert agent.max_turns == 3
assert agent.max_parallel == 4
assert agent.model == "claude-3-haiku-20240307"
class TestFastContextAgentSearch:
"""Tests for FastContextAgent search functionality."""
def test_search_returns_result(self, temp_workspace):
"""Test that search returns FastContextResult."""
from praisonaiagents.context.fast.fast_context_agent import FastContextAgent
from praisonaiagents.context.fast.result import FastContextResult
agent = FastContextAgent(workspace_path=temp_workspace)
# Use simple search without LLM
result = agent.search_simple("authenticate")
assert isinstance(result, FastContextResult)
assert result.total_files > 0
def test_search_finds_relevant_files(self, temp_workspace):
"""Test that search finds relevant files."""
from praisonaiagents.context.fast.fast_context_agent import FastContextAgent
agent = FastContextAgent(workspace_path=temp_workspace)
result = agent.search_simple("authenticate")
# Should find files containing "authenticate"
file_paths = [f.path for f in result.files]
assert any("main.py" in p for p in file_paths)
assert any("handler.py" in p for p in file_paths)
def test_search_with_file_pattern(self, temp_workspace):
"""Test search with file pattern filter."""
from praisonaiagents.context.fast.fast_context_agent import FastContextAgent
agent = FastContextAgent(workspace_path=temp_workspace)
result = agent.search_simple("def", include_patterns=["**/auth/*.py"])
# Should only find in auth directory
for f in result.files:
assert "auth" in f.path
def test_search_records_timing(self, temp_workspace):
"""Test that search records timing information."""
from praisonaiagents.context.fast.fast_context_agent import FastContextAgent
agent = FastContextAgent(workspace_path=temp_workspace)
result = agent.search_simple("def")
# search_time_ms >= 0 is valid (can be 0 for very fast searches)
assert result.search_time_ms >= 0
assert result.turns_used == 1
assert result.total_tool_calls == 1
def test_search_no_results(self, temp_workspace):
"""Test search with no matches."""
from praisonaiagents.context.fast.fast_context_agent import FastContextAgent
agent = FastContextAgent(workspace_path=temp_workspace)
result = agent.search_simple("nonexistent_pattern_xyz123")
assert result.total_files == 0
class TestFastContextAgentTools:
"""Tests for FastContextAgent tool definitions."""
def test_get_tools(self, temp_workspace):
"""Test getting tool definitions."""
from praisonaiagents.context.fast.fast_context_agent import FastContextAgent
agent = FastContextAgent(workspace_path=temp_workspace)
tools = agent.get_tools()
assert len(tools) == 4
tool_names = [t["name"] for t in tools]
assert "grep_search" in tool_names
assert "glob_search" in tool_names
assert "read_file" in tool_names
assert "list_directory" in tool_names
def test_execute_tool(self, temp_workspace):
"""Test executing a tool directly."""
from praisonaiagents.context.fast.fast_context_agent import FastContextAgent
agent = FastContextAgent(workspace_path=temp_workspace)
result = agent.execute_tool("glob_search", pattern="**/*.py")
assert len(result) >= 3 # At least 3 .py files
class TestFastContextAgentLLMSearch:
"""Tests for FastContextAgent LLM-powered search."""
def test_search_falls_back_to_simple(self, temp_workspace):
"""Test that search falls back to simple when LLM not available."""
from praisonaiagents.context.fast.fast_context_agent import FastContextAgent
agent = FastContextAgent(workspace_path=temp_workspace)
# search() should fall back to search_simple when LLM is not available
result = agent.search("find authentication code")
# Should still return results (from simple search fallback)
assert result is not None
assert result.query == "find authentication code"
if __name__ == "__main__":
pytest.main([__file__, "-v"])
@@ -0,0 +1,269 @@
"""
Tests for Fast Context result classes.
"""
import pytest
import time
class TestLineRange:
"""Tests for LineRange dataclass."""
def test_line_range_creation(self):
"""Test basic LineRange creation."""
from praisonaiagents.context.fast.result import LineRange
lr = LineRange(start=10, end=20)
assert lr.start == 10
assert lr.end == 20
assert lr.content is None
assert lr.relevance_score == 1.0
def test_line_range_with_content(self):
"""Test LineRange with content."""
from praisonaiagents.context.fast.result import LineRange
lr = LineRange(start=1, end=3, content="line1\nline2\nline3")
assert lr.content == "line1\nline2\nline3"
def test_line_range_invalid_start(self):
"""Test LineRange corrects invalid start."""
from praisonaiagents.context.fast.result import LineRange
lr = LineRange(start=0, end=5)
assert lr.start == 1 # Corrected to 1
def test_line_range_invalid_end(self):
"""Test LineRange corrects invalid end."""
from praisonaiagents.context.fast.result import LineRange
lr = LineRange(start=10, end=5)
assert lr.end == 10 # Corrected to start
def test_line_count(self):
"""Test line_count property."""
from praisonaiagents.context.fast.result import LineRange
lr = LineRange(start=10, end=20)
assert lr.line_count == 11
def test_overlaps_true(self):
"""Test overlaps returns True for overlapping ranges."""
from praisonaiagents.context.fast.result import LineRange
lr1 = LineRange(start=10, end=20)
lr2 = LineRange(start=15, end=25)
assert lr1.overlaps(lr2) is True
assert lr2.overlaps(lr1) is True
def test_overlaps_false(self):
"""Test overlaps returns False for non-overlapping ranges."""
from praisonaiagents.context.fast.result import LineRange
lr1 = LineRange(start=10, end=20)
lr2 = LineRange(start=25, end=30)
assert lr1.overlaps(lr2) is False
def test_overlaps_adjacent(self):
"""Test overlaps for adjacent ranges."""
from praisonaiagents.context.fast.result import LineRange
lr1 = LineRange(start=10, end=20)
lr2 = LineRange(start=21, end=30)
assert lr1.overlaps(lr2) is False
def test_merge(self):
"""Test merging overlapping ranges."""
from praisonaiagents.context.fast.result import LineRange
lr1 = LineRange(start=10, end=20, relevance_score=0.8)
lr2 = LineRange(start=15, end=25, relevance_score=0.9)
merged = lr1.merge(lr2)
assert merged.start == 10
assert merged.end == 25
assert merged.relevance_score == 0.9 # Max of both
def test_merge_non_overlapping_raises(self):
"""Test merge raises for non-overlapping ranges."""
from praisonaiagents.context.fast.result import LineRange
lr1 = LineRange(start=10, end=20)
lr2 = LineRange(start=25, end=30)
with pytest.raises(ValueError):
lr1.merge(lr2)
class TestFileMatch:
"""Tests for FileMatch dataclass."""
def test_file_match_creation(self):
"""Test basic FileMatch creation."""
from praisonaiagents.context.fast.result import FileMatch
fm = FileMatch(path="/path/to/file.py")
assert fm.path == "/path/to/file.py"
assert fm.line_ranges == []
assert fm.relevance_score == 1.0
assert fm.match_count == 0
def test_add_line_range(self):
"""Test adding line ranges."""
from praisonaiagents.context.fast.result import FileMatch, LineRange
fm = FileMatch(path="/path/to/file.py")
fm.add_line_range(LineRange(start=10, end=20))
assert len(fm.line_ranges) == 1
assert fm.line_ranges[0].start == 10
def test_add_overlapping_line_ranges(self):
"""Test adding overlapping line ranges merges them."""
from praisonaiagents.context.fast.result import FileMatch, LineRange
fm = FileMatch(path="/path/to/file.py")
fm.add_line_range(LineRange(start=10, end=20))
fm.add_line_range(LineRange(start=15, end=25))
assert len(fm.line_ranges) == 1
assert fm.line_ranges[0].start == 10
assert fm.line_ranges[0].end == 25
def test_add_non_overlapping_line_ranges(self):
"""Test adding non-overlapping line ranges keeps them separate."""
from praisonaiagents.context.fast.result import FileMatch, LineRange
fm = FileMatch(path="/path/to/file.py")
fm.add_line_range(LineRange(start=10, end=20))
fm.add_line_range(LineRange(start=30, end=40))
assert len(fm.line_ranges) == 2
def test_total_lines(self):
"""Test total_lines property."""
from praisonaiagents.context.fast.result import FileMatch, LineRange
fm = FileMatch(path="/path/to/file.py")
fm.add_line_range(LineRange(start=10, end=20)) # 11 lines
fm.add_line_range(LineRange(start=30, end=35)) # 6 lines
assert fm.total_lines == 17
class TestFastContextResult:
"""Tests for FastContextResult dataclass."""
def test_result_creation(self):
"""Test basic result creation."""
from praisonaiagents.context.fast.result import FastContextResult
result = FastContextResult(query="find authentication")
assert result.query == "find authentication"
assert result.files == []
assert result.search_time_ms == 0
def test_timer(self):
"""Test timer functionality."""
from praisonaiagents.context.fast.result import FastContextResult
result = FastContextResult()
result.start_timer()
time.sleep(0.1) # 100ms
result.stop_timer()
assert result.search_time_ms >= 100
assert result.search_time_ms < 200 # Should be close to 100ms
def test_add_file(self):
"""Test adding files."""
from praisonaiagents.context.fast.result import FastContextResult, FileMatch
result = FastContextResult()
result.add_file(FileMatch(path="/path/to/file.py"))
assert result.total_files == 1
def test_add_duplicate_file_merges(self):
"""Test adding duplicate file path merges them."""
from praisonaiagents.context.fast.result import (
FastContextResult, FileMatch, LineRange
)
result = FastContextResult()
fm1 = FileMatch(path="/path/to/file.py", relevance_score=0.8)
fm1.add_line_range(LineRange(start=10, end=20))
result.add_file(fm1)
fm2 = FileMatch(path="/path/to/file.py", relevance_score=0.9)
fm2.add_line_range(LineRange(start=30, end=40))
result.add_file(fm2)
assert result.total_files == 1
assert len(result.files[0].line_ranges) == 2
assert result.files[0].relevance_score == 0.9
def test_sort_by_relevance(self):
"""Test sorting files by relevance."""
from praisonaiagents.context.fast.result import FastContextResult, FileMatch
result = FastContextResult()
result.add_file(FileMatch(path="/low.py", relevance_score=0.3))
result.add_file(FileMatch(path="/high.py", relevance_score=0.9))
result.add_file(FileMatch(path="/mid.py", relevance_score=0.6))
result.sort_by_relevance()
assert result.files[0].path == "/high.py"
assert result.files[1].path == "/mid.py"
assert result.files[2].path == "/low.py"
def test_to_context_string(self):
"""Test converting result to context string."""
from praisonaiagents.context.fast.result import (
FastContextResult, FileMatch, LineRange
)
result = FastContextResult()
fm = FileMatch(path="/path/to/file.py")
fm.add_line_range(LineRange(start=10, end=20))
result.add_file(fm)
context = result.to_context_string()
assert "1 relevant file" in context
assert "/path/to/file.py" in context
assert "Lines 10-20" in context
def test_to_context_string_empty(self):
"""Test context string for empty result."""
from praisonaiagents.context.fast.result import FastContextResult
result = FastContextResult()
context = result.to_context_string()
assert "No relevant files found" in context
def test_to_dict(self):
"""Test converting result to dictionary."""
from praisonaiagents.context.fast.result import (
FastContextResult, FileMatch, LineRange
)
result = FastContextResult(query="test", search_time_ms=100, turns_used=2)
fm = FileMatch(path="/file.py", relevance_score=0.8)
fm.add_line_range(LineRange(start=10, end=20))
result.add_file(fm)
d = result.to_dict()
assert d["query"] == "test"
assert d["search_time_ms"] == 100
assert d["turns_used"] == 2
assert len(d["files"]) == 1
assert d["files"][0]["path"] == "/file.py"
if __name__ == "__main__":
pytest.main([__file__, "-v"])
@@ -0,0 +1,261 @@
"""
Tests for File Indexer.
"""
import pytest
import tempfile
import os
from pathlib import Path
@pytest.fixture
def temp_workspace():
"""Create a temporary workspace with test files."""
with tempfile.TemporaryDirectory() as tmpdir:
# Create directory structure
src_dir = Path(tmpdir) / "src"
src_dir.mkdir()
(src_dir / "main.py").write_text("def main(): pass")
(src_dir / "utils.py").write_text("def util(): pass")
(src_dir / "config.json").write_text('{"key": "value"}')
auth_dir = src_dir / "auth"
auth_dir.mkdir()
(auth_dir / "handler.py").write_text("class Handler: pass")
# Create .gitignore
(Path(tmpdir) / ".gitignore").write_text("__pycache__/\n*.pyc\n")
# Create ignored directory
cache_dir = Path(tmpdir) / "__pycache__"
cache_dir.mkdir()
(cache_dir / "cached.pyc").write_text("cached")
yield tmpdir
class TestFileIndexerInit:
"""Tests for FileIndexer initialization."""
def test_default_initialization(self, temp_workspace):
"""Test default initialization."""
from praisonaiagents.context.fast.indexer import FileIndexer
indexer = FileIndexer(workspace_path=temp_workspace)
assert indexer.workspace_path == temp_workspace
assert indexer.respect_gitignore is True
assert indexer.total_files == 0
def test_custom_extensions(self, temp_workspace):
"""Test custom extensions."""
from praisonaiagents.context.fast.indexer import FileIndexer
indexer = FileIndexer(
workspace_path=temp_workspace,
extensions={".py"}
)
assert indexer.extensions == {".py"}
class TestFileIndexerIndex:
"""Tests for FileIndexer indexing."""
def test_index_files(self, temp_workspace):
"""Test indexing files."""
from praisonaiagents.context.fast.indexer import FileIndexer
indexer = FileIndexer(workspace_path=temp_workspace)
count = indexer.index()
assert count >= 3 # At least main.py, utils.py, handler.py
assert indexer.total_files >= 3
def test_index_respects_gitignore(self, temp_workspace):
"""Test that indexing respects .gitignore."""
from praisonaiagents.context.fast.indexer import FileIndexer
indexer = FileIndexer(workspace_path=temp_workspace, respect_gitignore=True)
indexer.index()
# Should not include __pycache__ files
for path in indexer.files:
assert "__pycache__" not in path
assert not path.endswith(".pyc")
def test_index_by_extension(self, temp_workspace):
"""Test indexing organizes by extension."""
from praisonaiagents.context.fast.indexer import FileIndexer
indexer = FileIndexer(workspace_path=temp_workspace)
indexer.index()
assert ".py" in indexer.by_extension
assert len(indexer.by_extension[".py"]) >= 3
def test_index_with_extension_filter(self, temp_workspace):
"""Test indexing with extension filter."""
from praisonaiagents.context.fast.indexer import FileIndexer
indexer = FileIndexer(workspace_path=temp_workspace, extensions={".py"})
indexer.index()
# Should only have .py files
for path in indexer.files:
assert path.endswith(".py")
class TestFileIndexerFind:
"""Tests for FileIndexer find methods."""
def test_find_by_pattern(self, temp_workspace):
"""Test finding files by pattern."""
from praisonaiagents.context.fast.indexer import FileIndexer
indexer = FileIndexer(workspace_path=temp_workspace)
indexer.index()
results = indexer.find_by_pattern("**/*.py")
assert len(results) >= 3
for info in results:
assert info.path.endswith(".py")
def test_find_by_extension(self, temp_workspace):
"""Test finding files by extension."""
from praisonaiagents.context.fast.indexer import FileIndexer
indexer = FileIndexer(workspace_path=temp_workspace)
indexer.index()
results = indexer.find_by_extension(".py")
assert len(results) >= 3
def test_find_by_extension_without_dot(self, temp_workspace):
"""Test finding files by extension without dot."""
from praisonaiagents.context.fast.indexer import FileIndexer
indexer = FileIndexer(workspace_path=temp_workspace)
indexer.index()
results = indexer.find_by_extension("py")
assert len(results) >= 3
def test_find_by_name_exact(self, temp_workspace):
"""Test finding files by exact name."""
from praisonaiagents.context.fast.indexer import FileIndexer
indexer = FileIndexer(workspace_path=temp_workspace)
indexer.index()
results = indexer.find_by_name("main.py", exact=True)
assert len(results) == 1
assert results[0].path.endswith("main.py")
def test_find_by_name_partial(self, temp_workspace):
"""Test finding files by partial name."""
from praisonaiagents.context.fast.indexer import FileIndexer
indexer = FileIndexer(workspace_path=temp_workspace)
indexer.index()
results = indexer.find_by_name("handler")
assert len(results) >= 1
assert any("handler" in r.path for r in results)
def test_get_file(self, temp_workspace):
"""Test getting file by path."""
from praisonaiagents.context.fast.indexer import FileIndexer
indexer = FileIndexer(workspace_path=temp_workspace)
indexer.index()
# Find a file first
results = indexer.find_by_name("main.py", exact=True)
assert len(results) > 0
# Get by path
file_info = indexer.get_file(results[0].path)
assert file_info is not None
assert file_info.path == results[0].path
class TestFileIndexerStats:
"""Tests for FileIndexer statistics."""
def test_get_stats(self, temp_workspace):
"""Test getting index statistics."""
from praisonaiagents.context.fast.indexer import FileIndexer
indexer = FileIndexer(workspace_path=temp_workspace)
indexer.index()
stats = indexer.get_stats()
assert stats["total_files"] >= 3
assert stats["total_size_bytes"] > 0
assert ".py" in stats["extensions"]
def test_is_stale(self, temp_workspace):
"""Test stale check."""
from praisonaiagents.context.fast.indexer import FileIndexer
indexer = FileIndexer(workspace_path=temp_workspace)
# Before indexing, should be stale
assert indexer.is_stale() is True
indexer.index()
# After indexing, should not be stale
assert indexer.is_stale(max_age=60.0) is False
class TestFileIndexerPersistence:
"""Tests for FileIndexer save/load."""
def test_save_and_load(self, temp_workspace):
"""Test saving and loading index."""
from praisonaiagents.context.fast.indexer import FileIndexer
indexer = FileIndexer(workspace_path=temp_workspace)
indexer.index()
# Save index
index_path = os.path.join(temp_workspace, ".fast_context_index.json")
indexer.save_index(index_path)
assert os.path.exists(index_path)
# Load into new indexer
indexer2 = FileIndexer(workspace_path=temp_workspace)
loaded = indexer2.load_index(index_path)
assert loaded is True
assert indexer2.total_files == indexer.total_files
def test_load_wrong_workspace(self, temp_workspace):
"""Test loading index from wrong workspace."""
from praisonaiagents.context.fast.indexer import FileIndexer
indexer = FileIndexer(workspace_path=temp_workspace)
indexer.index()
index_path = os.path.join(temp_workspace, ".fast_context_index.json")
indexer.save_index(index_path)
# Try to load with different workspace
indexer2 = FileIndexer(workspace_path="/different/path")
loaded = indexer2.load_index(index_path)
assert loaded is False
if __name__ == "__main__":
pytest.main([__file__, "-v"])
@@ -0,0 +1,205 @@
"""
Tests for Fast Context parallel executor.
"""
import pytest
import asyncio
import time
import tempfile
from pathlib import Path
@pytest.fixture
def temp_workspace():
"""Create a temporary workspace with test files."""
with tempfile.TemporaryDirectory() as tmpdir:
# Create some test files
for i in range(5):
(Path(tmpdir) / f"file{i}.py").write_text(f"# File {i}\ndef func{i}(): pass")
yield tmpdir
class TestParallelExecutor:
"""Tests for ParallelExecutor class."""
@pytest.mark.asyncio
async def test_execute_single_task(self, temp_workspace):
"""Test executing a single task."""
from praisonaiagents.context.fast.parallel_executor import ParallelExecutor
from praisonaiagents.context.fast.search_tools import glob_search
executor = ParallelExecutor(max_parallel=8)
tasks = [
{"tool": "glob_search", "args": {"search_path": temp_workspace, "pattern": "*.py"}}
]
results = await executor.execute(tasks)
assert len(results) == 1
assert len(results[0]) == 5 # 5 .py files
@pytest.mark.asyncio
async def test_execute_multiple_tasks_parallel(self, temp_workspace):
"""Test executing multiple tasks in parallel."""
from praisonaiagents.context.fast.parallel_executor import ParallelExecutor
executor = ParallelExecutor(max_parallel=8)
tasks = [
{"tool": "glob_search", "args": {"search_path": temp_workspace, "pattern": "*.py"}},
{"tool": "grep_search", "args": {"search_path": temp_workspace, "pattern": "def"}},
{"tool": "list_directory", "args": {"dir_path": temp_workspace}},
]
start = time.perf_counter()
results = await executor.execute(tasks)
elapsed = time.perf_counter() - start
assert len(results) == 3
# All tasks should complete (parallel execution)
assert all(r is not None for r in results)
@pytest.mark.asyncio
async def test_execute_with_timeout(self, temp_workspace):
"""Test execution with timeout."""
from praisonaiagents.context.fast.parallel_executor import ParallelExecutor
executor = ParallelExecutor(max_parallel=8, timeout=0.001) # Very short timeout
# This should timeout for slow operations
tasks = [
{"tool": "glob_search", "args": {"search_path": temp_workspace, "pattern": "**/*.py"}}
]
results = await executor.execute(tasks)
# Should return results (either success or timeout error)
assert len(results) == 1
@pytest.mark.asyncio
async def test_execute_respects_max_parallel(self, temp_workspace):
"""Test that max_parallel limit is respected."""
from praisonaiagents.context.fast.parallel_executor import ParallelExecutor
executor = ParallelExecutor(max_parallel=2)
# Create more tasks than max_parallel
tasks = [
{"tool": "glob_search", "args": {"search_path": temp_workspace, "pattern": "*.py"}}
for _ in range(5)
]
results = await executor.execute(tasks)
assert len(results) == 5
# All should complete successfully
assert all(len(r) == 5 for r in results)
@pytest.mark.asyncio
async def test_execute_handles_errors(self, temp_workspace):
"""Test that errors in one task don't affect others."""
from praisonaiagents.context.fast.parallel_executor import ParallelExecutor
executor = ParallelExecutor(max_parallel=8)
tasks = [
{"tool": "glob_search", "args": {"search_path": temp_workspace, "pattern": "*.py"}},
{"tool": "read_file", "args": {"filepath": "/nonexistent/file.py"}}, # Will fail
{"tool": "list_directory", "args": {"dir_path": temp_workspace}},
]
results = await executor.execute(tasks)
assert len(results) == 3
# First and third should succeed
assert len(results[0]) == 5
assert results[1]["success"] is False # Error result
assert results[2]["success"] is True
@pytest.mark.asyncio
async def test_execute_unknown_tool(self, temp_workspace):
"""Test handling of unknown tool."""
from praisonaiagents.context.fast.parallel_executor import ParallelExecutor
executor = ParallelExecutor(max_parallel=8)
tasks = [
{"tool": "unknown_tool", "args": {}}
]
results = await executor.execute(tasks)
assert len(results) == 1
assert "error" in results[0]
@pytest.mark.asyncio
async def test_execute_empty_tasks(self):
"""Test executing empty task list."""
from praisonaiagents.context.fast.parallel_executor import ParallelExecutor
executor = ParallelExecutor(max_parallel=8)
results = await executor.execute([])
assert results == []
def test_execute_sync(self, temp_workspace):
"""Test synchronous execution wrapper."""
from praisonaiagents.context.fast.parallel_executor import ParallelExecutor
executor = ParallelExecutor(max_parallel=8)
tasks = [
{"tool": "glob_search", "args": {"search_path": temp_workspace, "pattern": "*.py"}}
]
results = executor.execute_sync(tasks)
assert len(results) == 1
assert len(results[0]) == 5
class TestToolCallBatch:
"""Tests for ToolCallBatch class."""
def test_batch_creation(self):
"""Test creating a tool call batch."""
from praisonaiagents.context.fast.parallel_executor import ToolCallBatch
batch = ToolCallBatch()
batch.add("grep_search", search_path="/path", pattern="test")
batch.add("glob_search", search_path="/path", pattern="*.py")
assert len(batch.tasks) == 2
def test_batch_max_size(self):
"""Test batch respects max size."""
from praisonaiagents.context.fast.parallel_executor import ToolCallBatch
batch = ToolCallBatch(max_size=2)
batch.add("grep_search", search_path="/path", pattern="test1")
batch.add("grep_search", search_path="/path", pattern="test2")
# Should not add beyond max_size
added = batch.add("grep_search", search_path="/path", pattern="test3")
assert added is False
assert len(batch.tasks) == 2
def test_batch_is_full(self):
"""Test is_full property."""
from praisonaiagents.context.fast.parallel_executor import ToolCallBatch
batch = ToolCallBatch(max_size=2)
assert batch.is_full is False
batch.add("grep_search", search_path="/path", pattern="test1")
assert batch.is_full is False
batch.add("grep_search", search_path="/path", pattern="test2")
assert batch.is_full is True
if __name__ == "__main__":
pytest.main([__file__, "-v"])
@@ -0,0 +1,336 @@
"""
Tests for Fast Context search tools.
"""
import pytest
import tempfile
import os
from pathlib import Path
@pytest.fixture
def temp_workspace():
"""Create a temporary workspace with test files."""
with tempfile.TemporaryDirectory() as tmpdir:
# Create directory structure
src_dir = Path(tmpdir) / "src"
src_dir.mkdir()
# Create Python files
(src_dir / "main.py").write_text("""
def main():
print("Hello, World!")
authenticate_user()
def authenticate_user():
# Authentication logic here
pass
class UserService:
def login(self, username, password):
return True
""")
(src_dir / "utils.py").write_text("""
import os
def get_config():
return {"debug": True}
def validate_input(data):
if not data:
raise ValueError("Empty data")
return True
""")
# Create subdirectory
auth_dir = src_dir / "auth"
auth_dir.mkdir()
(auth_dir / "handler.py").write_text("""
from typing import Optional
class AuthHandler:
def __init__(self):
self.token = None
def authenticate(self, credentials: dict) -> bool:
# Authenticate user
return True
def logout(self):
self.token = None
""")
# Create a .gitignore
(Path(tmpdir) / ".gitignore").write_text("""
__pycache__/
*.pyc
.env
""")
# Create ignored directory
cache_dir = Path(tmpdir) / "__pycache__"
cache_dir.mkdir()
(cache_dir / "cached.pyc").write_text("cached content")
yield tmpdir
class TestGrepSearch:
"""Tests for grep_search function."""
def test_grep_simple_pattern(self, temp_workspace):
"""Test grep with simple pattern."""
from praisonaiagents.context.fast.search_tools import grep_search
results = grep_search(temp_workspace, "authenticate")
assert len(results) > 0
# Should find in main.py and handler.py
paths = [r["path"] for r in results]
assert any("main.py" in p for p in paths)
assert any("handler.py" in p for p in paths)
def test_grep_regex_pattern(self, temp_workspace):
"""Test grep with regex pattern."""
from praisonaiagents.context.fast.search_tools import grep_search
results = grep_search(temp_workspace, r"def \w+\(", is_regex=True)
assert len(results) > 0
# Should find function definitions
def test_grep_case_insensitive(self, temp_workspace):
"""Test grep case insensitive search."""
from praisonaiagents.context.fast.search_tools import grep_search
results = grep_search(temp_workspace, "AUTHENTICATE", case_sensitive=False)
assert len(results) > 0
def test_grep_case_sensitive(self, temp_workspace):
"""Test grep case sensitive search."""
from praisonaiagents.context.fast.search_tools import grep_search
results = grep_search(temp_workspace, "AUTHENTICATE", case_sensitive=True)
assert len(results) == 0 # No uppercase AUTHENTICATE exists
def test_grep_with_file_pattern(self, temp_workspace):
"""Test grep with file pattern filter."""
from praisonaiagents.context.fast.search_tools import grep_search
results = grep_search(temp_workspace, "def", include_patterns=["*.py"])
assert len(results) > 0
# All results should be .py files
for r in results:
assert r["path"].endswith(".py")
def test_grep_with_exclude_pattern(self, temp_workspace):
"""Test grep with exclude pattern."""
from praisonaiagents.context.fast.search_tools import grep_search
results = grep_search(temp_workspace, "def", exclude_patterns=["**/auth/*"])
# Should not include auth directory files
for r in results:
assert "auth" not in r["path"]
def test_grep_max_results(self, temp_workspace):
"""Test grep with max results limit."""
from praisonaiagents.context.fast.search_tools import grep_search
results = grep_search(temp_workspace, "def", max_results=2)
assert len(results) <= 2
def test_grep_no_matches(self, temp_workspace):
"""Test grep with no matches."""
from praisonaiagents.context.fast.search_tools import grep_search
results = grep_search(temp_workspace, "nonexistent_pattern_xyz")
assert len(results) == 0
def test_grep_respects_gitignore(self, temp_workspace):
"""Test grep respects .gitignore patterns."""
from praisonaiagents.context.fast.search_tools import grep_search
results = grep_search(temp_workspace, "cached", respect_gitignore=True)
# Should not find content in __pycache__
for r in results:
assert "__pycache__" not in r["path"]
class TestGlobSearch:
"""Tests for glob_search function."""
def test_glob_all_python_files(self, temp_workspace):
"""Test glob to find all Python files."""
from praisonaiagents.context.fast.search_tools import glob_search
results = glob_search(temp_workspace, "**/*.py")
assert len(results) >= 3 # main.py, utils.py, handler.py
for r in results:
assert r["path"].endswith(".py")
def test_glob_specific_directory(self, temp_workspace):
"""Test glob in specific directory."""
from praisonaiagents.context.fast.search_tools import glob_search
results = glob_search(temp_workspace, "src/auth/*.py")
assert len(results) == 1
assert "handler.py" in results[0]["path"]
def test_glob_by_name(self, temp_workspace):
"""Test glob by filename."""
from praisonaiagents.context.fast.search_tools import glob_search
results = glob_search(temp_workspace, "**/main.py")
assert len(results) == 1
assert "main.py" in results[0]["path"]
def test_glob_no_matches(self, temp_workspace):
"""Test glob with no matches."""
from praisonaiagents.context.fast.search_tools import glob_search
results = glob_search(temp_workspace, "**/*.java")
assert len(results) == 0
def test_glob_max_results(self, temp_workspace):
"""Test glob with max results."""
from praisonaiagents.context.fast.search_tools import glob_search
results = glob_search(temp_workspace, "**/*.py", max_results=2)
assert len(results) <= 2
def test_glob_respects_gitignore(self, temp_workspace):
"""Test glob respects .gitignore."""
from praisonaiagents.context.fast.search_tools import glob_search
results = glob_search(temp_workspace, "**/*.pyc", respect_gitignore=True)
assert len(results) == 0 # .pyc is in .gitignore
class TestReadFile:
"""Tests for read_file function."""
def test_read_entire_file(self, temp_workspace):
"""Test reading entire file."""
from praisonaiagents.context.fast.search_tools import read_file
result = read_file(os.path.join(temp_workspace, "src", "main.py"))
assert result["success"] is True
assert "def main():" in result["content"]
assert result["total_lines"] > 0
def test_read_file_with_line_range(self, temp_workspace):
"""Test reading specific line range."""
from praisonaiagents.context.fast.search_tools import read_file
result = read_file(
os.path.join(temp_workspace, "src", "main.py"),
start_line=1,
end_line=3
)
assert result["success"] is True
assert result["start_line"] == 1
assert result["end_line"] == 3
lines = result["content"].split("\n")
assert len(lines) <= 3
def test_read_nonexistent_file(self, temp_workspace):
"""Test reading nonexistent file."""
from praisonaiagents.context.fast.search_tools import read_file
result = read_file(os.path.join(temp_workspace, "nonexistent.py"))
assert result["success"] is False
assert "error" in result
def test_read_file_with_context(self, temp_workspace):
"""Test reading with context lines."""
from praisonaiagents.context.fast.search_tools import read_file
result = read_file(
os.path.join(temp_workspace, "src", "main.py"),
start_line=5,
end_line=5,
context_lines=2
)
assert result["success"] is True
# Should include 2 lines before and after line 5
class TestListDirectory:
"""Tests for list_directory function."""
def test_list_directory(self, temp_workspace):
"""Test listing directory contents."""
from praisonaiagents.context.fast.search_tools import list_directory
result = list_directory(temp_workspace)
assert result["success"] is True
assert len(result["entries"]) > 0
# Should contain src directory
names = [e["name"] for e in result["entries"]]
assert "src" in names
def test_list_directory_recursive(self, temp_workspace):
"""Test listing directory recursively."""
from praisonaiagents.context.fast.search_tools import list_directory
result = list_directory(temp_workspace, recursive=True)
assert result["success"] is True
# Should find files in subdirectories
paths = [e["path"] for e in result["entries"]]
assert any("main.py" in p for p in paths)
assert any("handler.py" in p for p in paths)
def test_list_directory_max_depth(self, temp_workspace):
"""Test listing with max depth."""
from praisonaiagents.context.fast.search_tools import list_directory
result = list_directory(temp_workspace, recursive=True, max_depth=1)
assert result["success"] is True
# Should not find files in auth subdirectory (depth 2)
paths = [e["path"] for e in result["entries"]]
assert not any("handler.py" in p for p in paths)
def test_list_nonexistent_directory(self, temp_workspace):
"""Test listing nonexistent directory."""
from praisonaiagents.context.fast.search_tools import list_directory
result = list_directory(os.path.join(temp_workspace, "nonexistent"))
assert result["success"] is False
def test_list_directory_respects_gitignore(self, temp_workspace):
"""Test listing respects .gitignore."""
from praisonaiagents.context.fast.search_tools import list_directory
result = list_directory(temp_workspace, recursive=True, respect_gitignore=True)
# Should not include __pycache__
paths = [e["path"] for e in result["entries"]]
assert not any("__pycache__" in p for p in paths)
if __name__ == "__main__":
pytest.main([__file__, "-v"])
@@ -0,0 +1,323 @@
"""
Tests for MCP HTTP Stream Transport features.
Tests the HTTPStreamTransport implementation including:
- Protocol version header (Mcp-Protocol-Version)
- Session management (Mcp-Session-Id)
- Resumability (Last-Event-ID, retry delay)
- Session termination (HTTP DELETE)
- SSE event parsing
"""
import pytest
class TestHTTPStreamTransportInit:
"""Test HTTPStreamTransport initialization."""
def test_transport_has_protocol_version_header(self):
"""Test that transport includes Mcp-Protocol-Version header."""
from praisonaiagents.mcp.mcp_http_stream import HTTPStreamTransport
transport = HTTPStreamTransport('https://api.example.com/mcp')
assert 'Mcp-Protocol-Version' in transport.headers
assert transport.headers['Mcp-Protocol-Version'] == '2025-03-26'
def test_transport_default_protocol_version(self):
"""Test default protocol version for backward compatibility."""
from praisonaiagents.mcp.mcp_http_stream import HTTPStreamTransport
assert HTTPStreamTransport.DEFAULT_PROTOCOL_VERSION == '2025-03-26'
def test_transport_custom_protocol_version(self):
"""Test custom protocol version via options."""
from praisonaiagents.mcp.mcp_http_stream import HTTPStreamTransport
transport = HTTPStreamTransport(
'https://api.example.com/mcp',
options={'protocol_version': '2025-11-25'}
)
assert transport.headers['Mcp-Protocol-Version'] == '2025-11-25'
def test_transport_has_accept_header(self):
"""Test that transport includes correct Accept header."""
from praisonaiagents.mcp.mcp_http_stream import HTTPStreamTransport
transport = HTTPStreamTransport('https://api.example.com/mcp')
assert 'Accept' in transport.headers
assert 'application/json' in transport.headers['Accept']
assert 'text/event-stream' in transport.headers['Accept']
def test_transport_has_session_id_when_provided(self):
"""Test that transport includes session ID when provided."""
from praisonaiagents.mcp.mcp_http_stream import HTTPStreamTransport
transport = HTTPStreamTransport(
'https://api.example.com/mcp',
session_id='test-session-123'
)
assert 'Mcp-Session-Id' in transport.headers
assert transport.headers['Mcp-Session-Id'] == 'test-session-123'
def test_transport_no_session_id_when_not_provided(self):
"""Test that transport doesn't include session ID when not provided."""
from praisonaiagents.mcp.mcp_http_stream import HTTPStreamTransport
transport = HTTPStreamTransport('https://api.example.com/mcp')
assert 'Mcp-Session-Id' not in transport.headers
class TestHTTPStreamTransportResumability:
"""Test HTTPStreamTransport resumability features."""
def test_transport_has_last_event_id_tracking(self):
"""Test that transport tracks last event ID."""
from praisonaiagents.mcp.mcp_http_stream import HTTPStreamTransport
transport = HTTPStreamTransport('https://api.example.com/mcp')
assert hasattr(transport, 'last_event_id')
assert transport.last_event_id is None
def test_transport_has_retry_delay_tracking(self):
"""Test that transport tracks retry delay."""
from praisonaiagents.mcp.mcp_http_stream import HTTPStreamTransport
transport = HTTPStreamTransport('https://api.example.com/mcp')
assert hasattr(transport, '_retry_delay_ms')
assert transport._retry_delay_ms == 3000 # Default 3 seconds
def test_transport_updates_last_event_id(self):
"""Test that last_event_id can be updated."""
from praisonaiagents.mcp.mcp_http_stream import HTTPStreamTransport
transport = HTTPStreamTransport('https://api.example.com/mcp')
transport.last_event_id = 'event-123'
assert transport.last_event_id == 'event-123'
def test_transport_updates_retry_delay(self):
"""Test that retry delay can be updated."""
from praisonaiagents.mcp.mcp_http_stream import HTTPStreamTransport
transport = HTTPStreamTransport('https://api.example.com/mcp')
transport._retry_delay_ms = 5000
assert transport._retry_delay_ms == 5000
class TestHTTPStreamTransportSessionTermination:
"""Test HTTPStreamTransport session termination."""
def test_transport_has_terminate_session_method(self):
"""Test that transport has terminate_session method."""
from praisonaiagents.mcp.mcp_http_stream import HTTPStreamTransport
transport = HTTPStreamTransport('https://api.example.com/mcp')
assert hasattr(transport, 'terminate_session')
assert callable(transport.terminate_session)
@pytest.mark.asyncio
async def test_terminate_session_returns_false_without_session(self):
"""Test that terminate_session returns False without session."""
from praisonaiagents.mcp.mcp_http_stream import HTTPStreamTransport
transport = HTTPStreamTransport('https://api.example.com/mcp')
# No session, should return False
result = await transport.terminate_session()
assert result is False
@pytest.mark.asyncio
async def test_terminate_session_returns_false_without_client_session(self):
"""Test that terminate_session returns False without aiohttp session."""
from praisonaiagents.mcp.mcp_http_stream import HTTPStreamTransport
transport = HTTPStreamTransport(
'https://api.example.com/mcp',
session_id='test-session'
)
# _session is None (not initialized)
result = await transport.terminate_session()
assert result is False
class TestHTTPStreamTransportAutoDetection:
"""Test transport auto-detection in MCP class."""
def test_http_url_uses_http_stream_transport(self):
"""Test that HTTP URLs use HTTPStreamTransport."""
from praisonaiagents.mcp.mcp_transport import get_transport_type
assert get_transport_type('http://localhost:8080/mcp') == 'http_stream'
assert get_transport_type('https://api.example.com/mcp') == 'http_stream'
def test_sse_url_uses_sse_transport(self):
"""Test that SSE URLs use SSE transport."""
from praisonaiagents.mcp.mcp_transport import get_transport_type
assert get_transport_type('http://localhost:8080/sse') == 'sse'
assert get_transport_type('https://api.example.com/sse') == 'sse'
def test_websocket_url_uses_websocket_transport(self):
"""Test that WebSocket URLs use WebSocket transport."""
from praisonaiagents.mcp.mcp_transport import get_transport_type
assert get_transport_type('ws://localhost:8080/mcp') == 'websocket'
assert get_transport_type('wss://api.example.com/mcp') == 'websocket'
def test_command_string_uses_stdio_transport(self):
"""Test that command strings use stdio transport."""
from praisonaiagents.mcp.mcp_transport import get_transport_type
assert get_transport_type('npx @modelcontextprotocol/server-memory') == 'stdio'
assert get_transport_type('python3 server.py') == 'stdio'
class TestSSEEventParsing:
"""Test SSE event parsing for resumability."""
def test_parse_sse_event_with_id(self):
"""Test parsing SSE event with id field."""
from praisonaiagents.mcp.mcp_session import parse_sse_event
event_text = "id: event-123\ndata: {\"test\": true}"
result = parse_sse_event(event_text)
assert result['id'] == 'event-123'
assert result['data'] == '{"test": true}'
def test_parse_sse_event_with_retry(self):
"""Test parsing SSE event with retry field."""
from praisonaiagents.mcp.mcp_session import parse_sse_event
event_text = "retry: 5000\ndata: {\"test\": true}"
result = parse_sse_event(event_text)
assert result['retry'] == 5000
assert result['data'] == '{"test": true}'
def test_parse_sse_event_with_all_fields(self):
"""Test parsing SSE event with all fields."""
from praisonaiagents.mcp.mcp_session import parse_sse_event
event_text = "id: event-456\nretry: 3000\ndata: {\"jsonrpc\": \"2.0\"}"
result = parse_sse_event(event_text)
assert result['id'] == 'event-456'
assert result['retry'] == 3000
assert result['data'] == '{"jsonrpc": "2.0"}'
def test_parse_sse_event_multiline_data(self):
"""Test parsing SSE event with multiline data."""
from praisonaiagents.mcp.mcp_session import parse_sse_event
event_text = "data: line1\ndata: line2"
result = parse_sse_event(event_text)
assert result['data'] == 'line1\nline2'
class TestResumabilityManager:
"""Test ResumabilityManager for SSE stream recovery."""
def test_manager_tracks_event_ids(self):
"""Test that manager tracks event IDs per stream."""
from praisonaiagents.mcp.mcp_session import ResumabilityManager
manager = ResumabilityManager()
manager.set_last_event_id('stream1', 'event-100')
manager.set_last_event_id('stream2', 'event-200')
assert manager.get_last_event_id('stream1') == 'event-100'
assert manager.get_last_event_id('stream2') == 'event-200'
def test_manager_returns_none_for_unknown_stream(self):
"""Test that manager returns None for unknown stream."""
from praisonaiagents.mcp.mcp_session import ResumabilityManager
manager = ResumabilityManager()
assert manager.get_last_event_id('unknown') is None
def test_manager_generates_resume_headers(self):
"""Test that manager generates Last-Event-ID header."""
from praisonaiagents.mcp.mcp_session import ResumabilityManager
manager = ResumabilityManager()
manager.set_last_event_id('stream1', 'event-123')
headers = manager.get_resume_headers('stream1')
assert 'Last-Event-ID' in headers
assert headers['Last-Event-ID'] == 'event-123'
def test_manager_tracks_retry_delay(self):
"""Test that manager tracks retry delay."""
from praisonaiagents.mcp.mcp_session import ResumabilityManager
manager = ResumabilityManager()
manager.set_retry_delay(5000)
assert manager.get_retry_delay() == 5000
def test_manager_default_retry_delay(self):
"""Test default retry delay."""
from praisonaiagents.mcp.mcp_session import ResumabilityManager
manager = ResumabilityManager()
# Default should be 3000ms
assert manager.get_retry_delay() == 3000
class TestEventStore:
"""Test EventStore for message replay."""
def test_store_events(self):
"""Test storing events."""
from praisonaiagents.mcp.mcp_session import EventStore
store = EventStore(max_events=100)
store.store('stream1', 'event-1', {'data': 'test1'})
store.store('stream1', 'event-2', {'data': 'test2'})
events = store.get_all_events('stream1')
assert len(events) == 2
def test_get_events_after_id(self):
"""Test getting events after specific ID."""
from praisonaiagents.mcp.mcp_session import EventStore
store = EventStore(max_events=100)
store.store('stream1', 'event-1', {'data': 'test1'})
store.store('stream1', 'event-2', {'data': 'test2'})
store.store('stream1', 'event-3', {'data': 'test3'})
events = store.get_events_after('stream1', 'event-1')
assert len(events) == 2
assert events[0]['data'] == 'test2'
assert events[1]['data'] == 'test3'
def test_store_respects_max_events(self):
"""Test that store respects max_events limit."""
from praisonaiagents.mcp.mcp_session import EventStore
store = EventStore(max_events=3)
for i in range(5):
store.store('stream1', f'event-{i}', {'data': f'test{i}'})
events = store.get_all_events('stream1')
assert len(events) == 3
if __name__ == "__main__":
pytest.main([__file__, "-v"])
+1 -1
View File
@@ -2422,7 +2422,7 @@ wheels = [
[[package]]
name = "praisonaiagents"
version = "0.0.189"
version = "0.0.190"
source = { editable = "." }
dependencies = [
{ name = "aiohttp" },
+67
View File
@@ -1098,6 +1098,73 @@ agent = Agent(
| Prompt Caching | OpenAI (auto), Anthropic, Bedrock, Deepseek |
## MCP (Model Context Protocol)
PraisonAI supports MCP Protocol Revision 2025-11-25 with multiple transports.
### MCP Client (Consume MCP Servers)
```python
from praisonaiagents import Agent, MCP
# stdio - Local NPX/Python servers
agent = Agent(tools=MCP("npx @modelcontextprotocol/server-memory"))
# Streamable HTTP - Production servers
agent = Agent(tools=MCP("https://api.example.com/mcp"))
# WebSocket - Real-time bidirectional
agent = Agent(tools=MCP("wss://api.example.com/mcp", auth_token="token"))
# SSE (Legacy) - Backward compatibility
agent = Agent(tools=MCP("http://localhost:8080/sse"))
# With environment variables
agent = Agent(
tools=MCP(
command="npx",
args=["-y", "@modelcontextprotocol/server-brave-search"],
env={"BRAVE_API_KEY": "your-key"}
)
)
```
### MCP Server (Expose Tools as MCP Server)
Expose your Python functions as MCP tools for Claude Desktop, Cursor, and other MCP clients:
```python
from praisonaiagents.mcp import ToolsMCPServer
def search_web(query: str, max_results: int = 5) -> dict:
"""Search the web for information."""
return {"results": [f"Result for {query}"]}
def calculate(expression: str) -> dict:
"""Evaluate a mathematical expression."""
return {"result": eval(expression)}
# Create and run MCP server
server = ToolsMCPServer(name="my-tools")
server.register_tools([search_web, calculate])
server.run() # stdio for Claude Desktop
# server.run_sse(host="0.0.0.0", port=8080) # SSE for web clients
```
### MCP Features
| Feature | Description |
|---------|-------------|
| **Session Management** | Automatic `Mcp-Session-Id` handling |
| **Protocol Versioning** | `Mcp-Protocol-Version` header |
| **Resumability** | SSE stream recovery via `Last-Event-ID` |
| **Security** | Origin validation, DNS rebinding prevention |
| **WebSocket** | Auto-reconnect with exponential backoff |
## Development:
Below is used for development only.
+2 -2
View File
@@ -3,8 +3,8 @@ class Praisonai < Formula
desc "AI tools for various AI applications"
homepage "https://github.com/MervinPraison/PraisonAI"
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.10.tar.gz"
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.10.tar.gz | shasum -a 256`.split.first
url "https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.11.tar.gz"
sha256 `curl -sL https://github.com/MervinPraison/PraisonAI/archive/refs/tags/v2.3.11.tar.gz | shasum -a 256`.split.first
license "MIT"
depends_on "[email protected]"
+1 -1
View File
@@ -57,7 +57,7 @@ class CloudDeployer:
file.write("FROM python:3.11-slim\n")
file.write("WORKDIR /app\n")
file.write("COPY . .\n")
file.write("RUN pip install flask praisonai==2.3.10 gunicorn markdown\n")
file.write("RUN pip install flask praisonai==2.3.11 gunicorn markdown\n")
file.write("EXPOSE 8080\n")
file.write('CMD ["gunicorn", "-b", "0.0.0.0:8080", "api:app"]\n')
+1 -1
View File
@@ -1 +1 @@
__version__ = "2.3.10"
__version__ = "2.3.11"
+1 -1
View File
@@ -12,7 +12,7 @@ dependencies = [
"rich>=13.7",
"markdown>=3.5",
"pyparsing>=3.0.0",
"praisonaiagents>=0.0.189",
"praisonaiagents>=0.0.190",
"python-dotenv>=0.19.0",
"instructor>=1.3.3",
"PyYAML>=6.0",
+4 -4
View File
@@ -4197,7 +4197,7 @@ requires-dist = [
{ name = "praisonai-tools", marker = "extra == 'autogen'", specifier = ">=0.0.22" },
{ name = "praisonai-tools", marker = "extra == 'autogen-v4'", specifier = ">=0.0.22" },
{ name = "praisonai-tools", marker = "extra == 'crewai'", specifier = ">=0.0.22" },
{ name = "praisonaiagents", specifier = ">=0.0.189" },
{ name = "praisonaiagents", specifier = ">=0.0.190" },
{ name = "pyautogen", marker = "extra == 'autogen'", specifier = "==0.2.29" },
{ name = "pydantic", marker = "extra == 'chat'", specifier = "<=2.10.1" },
{ name = "pydantic", marker = "extra == 'code'", specifier = "<=2.10.1" },
@@ -4252,7 +4252,7 @@ wheels = [
[[package]]
name = "praisonaiagents"
version = "0.0.189"
version = "0.0.190"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "aiohttp" },
@@ -4261,9 +4261,9 @@ dependencies = [
{ name = "pydantic" },
{ name = "rich" },
]
sdist = { url = "https://files.pythonhosted.org/packages/62/ed/bd468ae67dea1489590fb306d50b02fbe2da5462745b2c8519a095b81fc1/praisonaiagents-0.0.189.tar.gz", hash = "sha256:a788b540b2b1f3418fb98013f1b2aba92d3010797d852507325735e2eb3fd944", size = 461258, upload-time = "2025-12-16T09:47:12.238Z" }
sdist = { url = "https://files.pythonhosted.org/packages/23/1d/d258091ca26f8ea3794b68835ec3ee43245bdc24e3bd4e7b0606cdf5139e/praisonaiagents-0.0.190.tar.gz", hash = "sha256:a596c56fc956c96f8927fbe6e08eb2ca58e7c2624b3f0bc7b2ed6655b9f7c084", size = 492228, upload-time = "2025-12-16T10:43:29.059Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/3a/70/2ea1d2a8baa02ae5a277616bbcc2f70c8a2a25a068d4cd5a8083eaf759cb/praisonaiagents-0.0.189-py3-none-any.whl", hash = "sha256:158693b6c68b9d23df64ee092648992222718279c47cac7b96e1007bf869ad8c", size = 437480, upload-time = "2025-12-16T09:47:09.846Z" },
{ url = "https://files.pythonhosted.org/packages/22/8a/6b1905e99452711c5631e34ca42bfa193025e122d12ec9800c0415eb5d1e/praisonaiagents-0.0.190-py3-none-any.whl", hash = "sha256:0481deb62fb0a9a6599ab973bd2ce91f3e700986326dca122a8e8a00850f2154", size = 474458, upload-time = "2025-12-16T10:43:26.984Z" },
]
[[package]]