run-llama/template-workflow-classify-extract-sec
1
0
Fork 0
mirror of https://github.com/run-llama/template-workflow-classify-extract-sec.git synced 2026-07-01 21:54:02 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge commit '446fde4fe401bc46166454174cab1ecd5783a915' as 'templates/classify-extract-sec'

Browse Source
This commit is contained in:
Adrian Lyjak
2025-11-04 17:28:46 -05:00
parent 783dbf4f76 4c74d0fd8a
commit 9433b0dab5
253 changed files with 2 additions and 14689 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.cursor/mcp.json
-11
View File
@@ -1,11 +0,0 @@
{
"mcpServers": {
"llama-index-docs": {
"url": "https://developers.llamaindex.ai/mcp"
},
"tmpl": {
"command": "uv",
"args": ["run", "tmpl", "mcp-stdio"]
}
}
}
.github/templates-remotes.yml
-56
View File
@@ -1,56 +0,0 @@
templates:
data-extraction:
remote: data-extraction
url: https://github.com/run-llama/template-workflow-data-extraction.git
branch: main
version: 0.2.1
document-qa:
remote: document-qa
url: https://github.com/run-llama/template-workflow-document-qa.git
branch: main
version: 0.2.2
basic-ui:
remote: basic-ui
url: https://github.com/run-llama/template-workflow-basic-ui.git
branch: main
version: 0.2.2
showcase:
remote: showcase
url: https://github.com/run-llama/template-workflow-showcase.git
branch: main
version: 0.1.3
basic:
remote: basic
url: https://github.com/run-llama/template-workflow-basic.git
branch: main
version: 0.2.0
document-parsing:
remote: document-parsing
url: https://github.com/run-llama/template-workflow-document-parsing.git
branch: main
version: 0.1.0
human-in-the-loop:
remote: human-in-the-loop
url: https://github.com/run-llama/template-workflow-human-in-the-loop.git
branch: main
version: 0.1.0
invoice-extraction:
remote: invoice-extraction
url: https://github.com/run-llama/template-workflow-invoice-extraction.git
branch: main
version: 0.1.0
rag:
remote: rag
url: https://github.com/run-llama/template-workflow-rag.git
branch: main
version: 0.1.0
web-scraping:
remote: web-scraping
url: https://github.com/run-llama/template-workflow-web-scraping.git
branch: main
version: 0.1.0
classify-extract-sec:
remote: classify-extract-sec
url: https://github.com/run-llama/template-workflow-classify-extract-sec.git
branch: main
version: 0.1.0
.github/workflows/ci.yml
-87
View File
@@ -1,87 +0,0 @@
name: templates-ci
on:
pull_request: {}
push:
branches: [ main ]
workflow_dispatch: {}
jobs:
root-test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: astral-sh/setup-uv@v5
with:
enable-cache: true
- name: Root lint and test
run: |
uv run hatch run lint-check
uv run hatch run format-check
uv run hatch run typecheck
uv run hatch run test
enumerate:
runs-on: ubuntu-latest
outputs:
all_json: ${{ steps.list.outputs.all_json }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: astral-sh/setup-uv@v6
with:
enable-cache: true
- id: list
name: List all templates
run: |
echo -n 'all_json=' >> "$GITHUB_OUTPUT"
uv run python -c 'import json; from tmpl.config import MAPPING_DATA; print(json.dumps(list(MAPPING_DATA.keys())))' >> "$GITHUB_OUTPUT"
detect:
runs-on: ubuntu-latest
outputs:
changed_json: ${{ steps.changes.outputs.json }}
has_changes: ${{ steps.changes.outputs.has_changes }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: astral-sh/setup-uv@v6
with:
enable-cache: true
- id: changes
name: Detect changed templates
run: |
uv run tmpl changed --format json --github-output
validate:
needs: [detect, enumerate]
if: needs.detect.outputs.has_changes == 'true' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
template: ${{ fromJSON(github.event_name == 'workflow_dispatch' && needs.enumerate.outputs.all_json || needs.detect.outputs.changed_json) }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: astral-sh/setup-uv@v5
with:
enable-cache: true
- uses: actions/setup-node@v5
with:
node-version: '22'
- name: Enable Corepack
run: corepack enable
- name: Validate template
env:
TEMPLATE: ${{ matrix.template }}
run: |
uv run tmpl check-workflows "$TEMPLATE"
uv run tmpl check-python "$TEMPLATE"
uv run tmpl check-javascript "$TEMPLATE"
.github/workflows/export-template-metrics.yml
-33
View File
@@ -1,33 +0,0 @@
name: Export Template Repo Metrics
on:
workflow_dispatch:
schedule:
- cron: '17 3 * * *'
jobs:
export:
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.13'
- name: Install uv
uses: astral-sh/setup-uv@v6
- name: Install project
run: uv sync --frozen
- name: Export metrics to PostHog
if: ${{ !cancelled() }}
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT }}
POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
run: uv run tmpl export-metrics
.github/workflows/mirror.yml
-97
View File
@@ -1,97 +0,0 @@
name: mirror-templates
on:
push:
branches: [main]
paths:
- "templates/**"
- "scripts/**"
- ".github/templates-remotes.yml"
workflow_dispatch:
inputs:
template_name:
description: 'Template name to force mirror (bypasses change detection)'
required: true
type: choice
options:
- data-extraction
- document-qa
- basic-ui
- basic
- document-parsing
- human-in-the-loop
- invoice-extraction
- rag
- web-scraping
jobs:
mirror:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
token: ${{ secrets.GH_PAT }}
- name: Configure Git
run: |
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
- uses: astral-sh/setup-uv@v6
with:
enable-cache: true
- id: changes
name: Detect changed templates (vs before)
if: github.event_name == 'push'
env:
BEFORE: ${{ github.event.before }}
SHA: ${{ github.sha }}
run: |
uv run tmpl changed --base "$BEFORE" --head "$SHA" --format json --github-output
# Also consider changes to the templates remotes mapping as a signal of changes
if git diff --name-only "$BEFORE" "$SHA" -- .github/templates-remotes.yml | grep -q .; then
echo "has_changes=true" >> "$GITHUB_OUTPUT"
fi
- name: Set manual template for workflow_dispatch
if: github.event_name == 'workflow_dispatch'
run: |
echo "json=[\"${{ github.event.inputs.template_name }}\"]" >> $GITHUB_OUTPUT
echo "has_changes=true" >> $GITHUB_OUTPUT
id: manual
- name: Mirror changed templates (automatic)
if: github.event_name == 'push' && steps.changes.outputs.has_changes == 'true'
env:
CHANGED_JSON: ${{ steps.changes.outputs.json }}
run: |
echo "$CHANGED_JSON" | jq -r '.[]' | while read -r t; do
echo "Mirroring $t"
uv run tmpl mirror "$t"
done
- name: Ensure remote tags exist for all versions (after auto mirror)
if: github.event_name == 'push' && steps.changes.outputs.has_changes == 'true'
run: |
uv run tmpl tag-versions
- name: Mirror template (manual)
if: github.event_name == 'workflow_dispatch'
env:
CHANGED_JSON: ${{ steps.manual.outputs.json }}
run: |
echo "$CHANGED_JSON" | jq -r '.[]' | while read -r t; do
echo "Force mirroring $t"
uv run tmpl mirror "$t"
done
- name: Ensure remote tags exist for all versions (after manual mirror)
if: github.event_name == 'workflow_dispatch'
run: |
uv run tmpl tag-versions
- name: No changes to mirror
if: github.event_name == 'push' && steps.changes.outputs.has_changes != 'true'
run: echo "No template changes detected. Skipping mirroring."
.gitignore
-12
View File
@@ -1,12 +0,0 @@
# Monorepo templates
__pycache__/
*.pyc
.env
.venv
/.copier-answers.yml
node_modules
build/
dist/
*egg-info/
claude_memory.db
.DS_Store
.python-version
-1
View File
@@ -1 +0,0 @@
3.13
AGENTS.md
-51
View File
@@ -1,51 +0,0 @@
Getting Started
---------------
This project exclusively uses `uv` for package management.
To run commands, use the following format:
```bash
uv run <command>
```
or, source the .venv/bin/activate and run the command directly:
```bash
source .venv/bin/activate
<command>
```
The main entrypoint is `tmpl`.
To get started, run:
```bash
uv run tmpl --help
```
Validating CI:
-------------------
When editing a template. Make changes in the templates/<template-name> directory.
You can validate the template via the `tmpl` CLI:
```bash
# format, typecheck, and lint before pushing to template
uv run tmpl check-python <template-name> --fix
uv run tmpl check-javascript <template-name> --fix
# Validate workflows still validate
uv run tmpl check-workflows <template-name>
```
When editing the src/tmpl code, make sure it passes it's own checks:
```bash
uv run hatch run all-fix
# this runs all of the following
uv run hatch run format
uv run hatch run lint
uv run hatch run typecheck
uv run hatch run test
```
AGENTS.modified.md
-304
View File
@@ -1,304 +0,0 @@
# LlamaCloud Services Integration Guide for Coding Agents
## Project Setup & Environment
### Package Management
This project uses `uv` for package management. All commands should be run with:
```bash
uv run <command>
```
Or activate the virtual environment first:
```bash
source .venv/bin/activate
<command>
```
### Installing LlamaCloud Services
Add LlamaCloud Services to your project:
```bash
uv add llama-cloud-services
```
### API Key Configuration
Before using any LlamaCloud services, set your API key as an environment variable:
```bash
export LLAMA_CLOUD_API_KEY="llx-..."
```
The API key does not need to be passed directly to service constructors—it will be read from the environment.
---
## LlamaCloud Services Overview
LlamaCloud provides four main services for intelligent document processing:
1. **LlamaParse** - Parse and extract text, charts, tables, and images from unstructured files
2. **LlamaClassify** - Automatically categorize documents using natural-language rules
3. **LlamaExtract** - Extract structured data following specific patterns
4. **LlamaCloud Index** - Store, index, and retrieve documents for RAG applications
---
## Service Integration Examples
### 1. LlamaParse - Document Parsing
#### Basic Usage
```python
from llama_cloud_services import LlamaParse
parser = LlamaParse(
parse_mode="parse_page_with_agent",
model="openai-gpt-4-1-mini",
high_res_ocr=True,
adaptive_long_table=True,
outlined_table_extraction=True,
output_tables_as_HTML=True,
result_type="markdown",
project_id=project_id,
organization_id=organization_id,
)
# Sync parsing
result = parser.parse("./my_file.pdf")
# Async parsing
result = await parser.aparse("./my_file.pdf")
# Batch processing
results = parser.parse(["./file1.pdf", "./file2.pdf"])
```
#### Working with Parse Results
```python
# Get different document formats
markdown_docs = result.get_markdown_documents(split_by_page=True)
text_docs = result.get_text_documents(split_by_page=False)
image_docs = result.get_image_documents(
include_screenshot_images=True,
image_download_dir="./images"
)
# Extract tables
result = parser.get_json_result("./my_file.pdf")
tables = parser.get_tables(result)
```
#### Parse Mode Presets
```python
# Cost-Effective Mode
parser = LlamaParse(
parse_mode="parse_page_with_llm",
high_res_ocr=True,
adaptive_long_table=True,
outlined_table_extraction=True,
output_tables_as_HTML=True,
result_type="markdown",
)
# Agentic Mode (Default) - Recommended
parser = LlamaParse(
parse_mode="parse_page_with_agent",
model="openai-gpt-4-1-mini",
high_res_ocr=True,
adaptive_long_table=True,
outlined_table_extraction=True,
output_tables_as_HTML=True,
result_type="markdown",
)
# Agentic Plus Mode - Highest Quality
parser = LlamaParse(
parse_mode="parse_page_with_agent",
model="anthropic-sonnet-4.0",
high_res_ocr=True,
adaptive_long_table=True,
outlined_table_extraction=True,
output_tables_as_HTML=True,
result_type="markdown",
)
```
---
### 2. LlamaExtract - Structured Data Extraction
#### Quick Start
```python
from llama_cloud_services import LlamaExtract
from llama_cloud import ExtractConfig, ExtractMode, ExtractTarget
from pydantic import BaseModel, Field
# Initialize
extractor = LlamaExtract(
show_progress=True,
check_interval=5,
project_id=project_id,
organization_id=organization_id,
)
# Define schema
class Resume(BaseModel):
name: str = Field(description="Full name of candidate")
email: str = Field(description="Email address")
skills: list[str] = Field(description="Technical skills")
# Configure extraction
config = ExtractConfig(
extraction_mode=ExtractMode.MULTIMODAL,
extraction_target=ExtractTarget.PER_DOC,
system_prompt="<context>",
cite_sources=True,
use_reasoning=True,
confidence_scores=True,
)
# Extract
result = extractor.extract(Resume, config, "resume.pdf")
print(result.data)
```
#### Extraction Modes
- **FAST** - Fastest processing, simple documents, no OCR
- **BALANCED** - Good speed/accuracy tradeoff
- **MULTIMODAL** - Recommended for visually rich documents
- **PREMIUM** - Highest accuracy with advanced OCR
#### Multiple Input Types
```python
# From file path
result = extractor.extract(Resume, config, "resume.pdf")
# From file handle
with open("resume.pdf", "rb") as f:
result = extractor.extract(Resume, config, f)
# From text content
from llama_cloud_services.extract import SourceText
text = "Name: John Doe\nEmail: john@example.com"
result = extractor.extract(Resume, config, SourceText(text_content=text))
```
#### Async Extraction
```python
import asyncio
async def extract_documents():
# Single file
result = await extractor.aextract(Resume, config, "resume.pdf")
# Queue multiple jobs
jobs = await extractor.queue_extraction(
Resume, config, ["resume1.pdf", "resume2.pdf"]
)
# Get results
results = [agent.get_extraction_run_for_job(job.id) for job in jobs]
return results
```
#### Extraction Agents (Advanced)
For reusable extraction workflows:
```python
# Create agent
agent = extractor.create_agent(
name="resume-parser",
data_schema=Resume,
config=config
)
# Use agent
result = agent.extract("resume.pdf")
# Batch processing
jobs = await agent.queue_extraction(["resume1.pdf", "resume2.pdf"])
# Manage agents
agents = extractor.list_agents()
agent = extractor.get_agent(name="resume-parser")
extractor.delete_agent(agent.id)
```
---
### 3. LlamaClassify - Document Classification
Automatically categorize documents before extraction:
```python
from llama_cloud_services.beta.classifier.client import ClassifyClient
from llama_cloud.types import ClassifierRule
classifier = ClassifyClient.from_api_key(api_key)
# Define classification rules
rules = [
ClassifierRule(
type="invoice",
description="Documents with line items, prices, and payment terms",
),
ClassifierRule(
type="contract",
description="Legal agreements with terms and signatures",
),
]
# Classify PDF directly
result = await classifier.aclassify_file_path(
rules=rules,
file_input_path="document.pdf",
)
classification = result.items[0].result
print(f"Type: {classification.type}")
print(f"Confidence: {classification.confidence:.2%}")
```
#### Parse → Classify → Extract Workflow
```python
# 1. Parse
parser = LlamaParse(result_type="markdown")
parse_result = await parser.aparse("document.pdf")
markdown_content = await parse_result.aget_markdown()
# 2. Classify
classification = await classifier.aclassify_file_path(
rules=rules, file_input_path="temp.md"
)
doc_type = classification.items[0].result.type
# 3. Extract with appropriate schema
if doc_type == "invoice":
schema = InvoiceSchema
elif doc_type == "contract":
schema = ContractSchema
source_text = SourceText(text_content=markdown_content)
extraction_result = extractor.extract(schema, config, source_text)
```
---
## Best Practices for Agents
1. **Always set API key in environment** - Don't hardcode credentials
2. **Use appropriate parse modes** - Balance cost and quality (Agentic Mode recommended)
3. **Choose extraction modes wisely** - MULTIMODAL recommended for most documents
4. **Classify before extracting** - Route different document types to appropriate schemas
5. **Use agents for repeated workflows** - Create extraction agents for reusable patterns
6. **Handle async properly** - Use `aparse`, `aextract` for better performance
7. **Validate code before committing** - Run `uv run hatch run all-fix`
---
## Supported File Types
- **Documents**: PDF, Word (.docx)
- **Text**: .txt, .csv, .json, .html, .md
- **Images**: .png, .jpg, .jpeg
README.md
+2 -119
View File
@@ -1,119 +1,2 @@
Template Manager (tmpl)
-----------------------
This directory contains the template monorepo manager, exposed via the `tmpl` CLI. You develop, validate, and publish templates directly under `templates/`
Use `uv` to run commands:
```bash
uv run tmpl --help
```
### CLI commands
- **list**: Show configured templates and (optionally) remotes/branches.
- `uv run tmpl list`
- `uv run tmpl list --detail`
- **changed**: Detect which templates changed between two refs (used by CI).
- `uv run tmpl changed --base <ref> --head <ref> --format json`
- **clone**: Initial import of a template as a git subtree.
- `uv run tmpl clone <template>`
- **merge**: Pull upstream changes from the template's remote (git subtree pull).
- `uv run tmpl merge <template>`
- **mirror**: Push `templates/<template>` to its upstream remote (git subtree push).
- `uv run tmpl mirror <template>`
- **check-python**: Run Python format/lint/type checks within `templates/<template>`.
- `uv run tmpl check-python <template> [--fix]`
- **check-javascript**: Run JS/TS formatting/lint checks within `templates/<template>/ui` if present.
- `uv run tmpl check-javascript <template> [--fix]`
- **check-workflows**: Validate workflows declared in the template's `pyproject.toml`.
- `uv run tmpl check-workflows <template>`
- **version**: Interactively bump versions for changed templates in `.github/templates-remotes.yml`.
- `uv run tmpl version [--base <ref> --head <ref>]`
- **tag-versions**: Ensure remote `vX.Y.Z` tags exist for templates with configured versions.
- `uv run tmpl tag-versions [--dry-run]`
- **init-scripts**: Add helpful dev scripts to the template (Python and JS/TS).
- `uv run tmpl init-scripts <template>`
- **export-metrics**: Export GitHub repo traffic to PostHog (see Metrics section).
- `uv run tmpl export-metrics [--dry-run] [--print] [--backfill]`
- **all**: Run any subcommand across all templates.
- `uv run tmpl all check-python --fix`
### Development workflow
Develop directly in the template, validate, and then release:
1. **Develop**: make changes inside `templates/<template>`
- Python: `uv run tmpl check-python <template> --fix`
- JS/TS (if present): `uv run tmpl check-javascript <template> --fix`
- Workflows: `uv run tmpl check-workflows <template>`
2. **Bump version** (when needed):
- `uv run tmpl version [--base <ref> --head <ref>]`
3. **Commit and open a PR**
4. **Release**: after merge to `main`, the mirror workflow pushes to the upstream template repo; or run manually:
- `uv run tmpl mirror <template>`
Tip: See the root `AGENTS.md` for a quick-start and project-level checks.
### Versioning and tagging
- Versions are stored in `.github/templates-remotes.yml`.
- Bump versions for changed templates: `uv run tmpl version [--base <ref> --head <ref>]`.
- List changed templates: `uv run tmpl changed --format json`.
- CI (`.github/workflows/mirror.yml`) on push to `main`: detect changes, mirror each changed template, then run `tag-versions`. Manual dispatch mirrors the chosen template then tags.
### GitHub Actions
This repo includes workflows that use the CLI under the hood:
- **templates-ci** (`.github/workflows/ci.yml`)
- Triggers on PRs and pushes that touch templates.
- Steps:
- Detect changed templates via `uv run tmpl changed --format json --github-output`.
- For each changed template: run `check-python`, `check-javascript`, and `check-workflows`.
- **mirror-templates** (`.github/workflows/mirror.yml`)
- Triggers on push to `main` and via manual dispatch.
- Uses `uv run tmpl changed` to detect changed templates on push; or a manual input.
- Mirrors each changed template using `uv run tmpl mirror <template>` to its configured upstream. Requires `GH_PAT` with push permission.
- **export-template-metrics** (`.github/workflows/export-template-metrics.yml`)
- Scheduled and manually runnable job to export metrics to PostHog.
- Runs `uv run tmpl export-metrics` with `GH_PAT` and `POSTHOG_API_KEY`.
### PostHog metrics
The `export-metrics` command fetches 14-day GitHub traffic (clones and unique cloners) for each configured template repository and emits PostHog events.
- **Required environment**:
- GitHub: `GITHUB_TOKEN` or `GITHUB_PAT`
- PostHog: `POSTHOG_API_KEY` (or `POSTHOG_PROJECT_API_KEY`), optional `POSTHOG_HOST` (defaults to `https://us.posthog.com`)
- **Events emitted**:
- `template_repo_clones_total`
- properties: `template`, `owner`, `repo`, `clones_total`, `clones_unique_total`, `days_count`, `window_start`, `window_end`, `dedupe_ts`
- timestamp: end of the window
- `template_repo_clones_daily`
- properties: `template`, `owner`, `repo`, `day`, `clones_day`, `clones_uniques_day`, `dedupe_ts`
- timestamp: that day's timestamp; with `--backfill` it emits one per day in the window; otherwise only the most recent day
- **Examples**:
- Dry run and print results locally: `uv run tmpl export-metrics --dry-run --print`
- Backfill daily events for the entire 14-day window: `uv run tmpl export-metrics --backfill`
### MCP Server
There's an experimental template search MCP server that can be used to search the templates directory for files relevant to a query.
It is autoconfigured for cursor in `.cursor/mcp.json`.
```json
"tmpl": {
"command": "tmpl",
"args": ["mcp-stdio"]
}
```
To use it, you will need to install `tmpl` globally. You can do this by running `uv tool install .` in this directory.
To add it to a template during development (it's files will be gitignored), run `tmpl init-agents-mcp`.
# template-workflow-classify-extract-sec
Llama Index Workflow Template
meta-templates/agents-mcp-ui/ui/AGENTS.md
-30
View File
@@ -1,30 +0,0 @@
This is a vite built react project. It uses tailwindcss for styling, react-router, and is meant to be built into a static single page application (SPA).
When applicable, use the `@llamaindex/ui` library to build the UI. It provides a React component library for building UI components that interact with the workflow server.
`@llamaindex/ui` provides:
- react hooks for interacting with the workflow server, such as `useWorkflowRun`, `useWorkflowHandler`, `useWorkflowEvents`, etc. Search the llama-index-ui documentation via the MCP tool for more details.
- Document centric components, such as pdf viewers, as well as components specifically integrated with LlamaCloud, such as extracted data viewers.
- It provides ShadCN based primitive components, such as button, card, etc.
The components are currently undocumented. Explore its node_modules and LSP completions to find and use the components you need.
When passing files to the workflow server, first upload them to LlamaCloud, and pass the file reference to the workflow server in the StartEvent. See the `FileUpload` component. Make use of other patterns from other templates. Use the `tmpl` MCP tool to find examples.
Never use the workflow API directly. Always interact with it through the hooks.
### System Tools:
- Install missing dependencies. Use tools at your disposal to find correct dependencies.
- Use `pnpm` for js code dependencies.
### Linting:
Make sure to run lints, tests, etc, to validate that the code is correct and runnable
- `build` - runs the vite build process
- `lint` - runs the typescript compiler to check for errors
- `format` - runs the prettier formatter to format the code
- `format-check` - runs the prettier formatter to check the code for formatting errors
- `all-check` - runs the linting, formatting, and build processes
- `all-fix` - runs the linting, formatting, and build processes, and fixes any errors. Prefer using this over `all-check`, as it will just fix things, and tell you if there's anything additional that needs to be fixed.
meta-templates/agents-mcp/.claude/settings.json
-6
View File
@@ -1,6 +0,0 @@
{
"enabledMcpjsonServers": [
"tmpl"
],
"enableAllProjectMcpServers": true
}
meta-templates/agents-mcp/.cursor/mcp.json
-11
View File
@@ -1,11 +0,0 @@
{
"mcpServers": {
"llama-index-docs": {
"url": "https://developers.llamaindex.ai/mcp"
},
"tmpl": {
"command": "tmpl",
"args": ["mcp-stdio"]
}
}
}
meta-templates/agents-mcp/.mcp.json
-16
View File
@@ -1,16 +0,0 @@
{
"mcpServers": {
"tmpl": {
"type": "stdio",
"command": "tmpl",
"args": [
"mcp-stdio"
],
"env": {}
},
"llama-index-docs": {
"type": "http",
"url": "https://developers.llamaindex.ai/mcp"
}
}
}
meta-templates/agents-mcp/AGENTS.md
-77
View File
@@ -1,77 +0,0 @@
This is a LlamaAgents powered project. It is meant to be modified and adapted to novel use cases.
When implementing a new workflow, or adding functionality, always first search for high-quality reference implementations. Make use of the `tmpl` `search-templates` tool to find matching snippets. It uses keyword based search. These are the best source of examples, as they are similar apps achieving similar goals.
When you cannot find examples, always reference the llama-index-docs MCP tool when integrating with llama-index libraries or llama-cloud. This will ensure you are using the latest syntax and library names to install, etc.
The llama-index-docs MCP tool documents:
- workflows - LlamaIndex's library for building durable agentic applications
- LlamaAgents - a suite of tools for building agents, across workflows and LlamaCloud
- `llamactl` - the CLI for developing and deploying LlamaAgents, powered by LlamaIndex workflows
- LlamaAgents configuration
- Agent Data - a way to store ad-hoc data in LlamaCloud
- LlamaCloud - best in class primitives for parsing file formats to text, extracting structured data from docs or text, and indexing and querying for retrieval.
- LlamaIndex - LlamaIndex's core OSS framework for creating generative AI applications. Abstracts away common LLM providers, and other generate AI application integrations, and common application patterns.
Toolchain:
Package Manager: `uv` is the package manager for this project. It is used to install dependencies, run tests, and build the project. Install dependencies with `uv add <package>`.
Linting: `hatch`/`ruff`/`ty` - run the provided linters with the hatch commands
e.g. `uv run hatch run format` to format the code.
All commands:
- `format` - format the code with ruff
- `format-check` - check the code with ruff
- `lint` - lint the code with ruff
- `lint-check` - check the code with ruff
- `typecheck` - typecheck the code with ty
- `test` - run the tests with pytest
- `all-check` - run all the checks
- `all-fix` - run all the fixes. Prefer using this over `all-check`,
as it will just fix things, and tell you if there's anything additional
that needs to be fixed.
System Tools:
- Install missing dependencies. Use tools at your disposal to find correct dependencies.
- Use `uv` for python code. Make sure you have its venv activated, or prefix commands with `uv run`.
- If there is a `/ui` directory, use the configured javascript package manager to install dependencies. Refer to the version in the `package.json`'s `packageManager` field.
Coding Style:
- Do not make messy or defensive code. Avoid unnecessary try catches. Do not add try catches or other optional imports. Do not use getattr/setattr. These are signs of underlying issues, or an incorrectly configured linter or environment.
- When adding new code or installing a dependency, use the latest version, and make sure to use the latest interface, looking up the interfaces where applicable.
- Don't pass through silent failures, better to explicitly fail if you can
- Err on the side of generating LLM-powered flows instead of heavy code/heuristic based decision making - especially in cases where you're dealing with a lot of text inputs and want the logic to be generalizable.
- Don't mix environment variable parsing, and client or service configuration into the main code files. Organize dependencies and env var configuration into a stand-alone file. E.g. `config.py`. If dependencies grow, split up the files further.
- Add and run unit tests to validate your changes.
Workflows and Workflows served via `llamactl` CLI:
- workflow state and events are stored durably. Avoid storing large amounts of data in the state, or passing around large amounts of data in events.
- Try to split steps up into different workflow steps if possible, instead of putting too much logic per workflow step.
- Data that should be published to the client should be written with `ctx.write_event_to_stream()` from the workflow's steps.
- Prefer to use typed APIs with the workflows library:
- Subclass `StartEvent`, `StopEvent`, `InputRequiredEvent`, `HumanResponseEvent` with custom
types in order to clearly define the expected input and output of the workflow step.
- When using the Workflow state store, create a custom state type, and use typed Context, e.g. `Context[MyStateType]`, then calls to `state = await ctx.store.get_state()` and `async with ctx.store.edit_state() as state` will have `MyStateType` as the type of the `state` variable.
- Use annotated resource parameters in workflow steps for dependency injection. e.g. `def my_step(ctx: Context, ev: StartEvent, llm: Annotated[OpenAI, Resource(get_llm)])`
- When adding new workflows that should be exposed, make sure to configure the pyproject.toml for the `llamactl` CLI with a `[tool.llamadeploy]` section.
- Usually, just configure workflows with a None timeout unless they really should have a timeout.
LlamaCloud:
- Use llama cloud services (Parsing, Extracting, Indexes, Agent Data) where relevant. LlamaAgent projects, like this one, are meant to be built on top of these services.
- Make use of Agent Data to store useful information that may want to be queried by a client, without requiring a workflow as a go-between
- Types in LlamaExtract's extracted schema should generally be optional - to allow LlamaExtract room to fail (it can sometimes return None for fields, and if the field is typed as required the script will break).
- AsyncAgentDataClient has a generic for the schema of the data. Make use of it to facilitate type safety and autocomplete. For example `AsyncAgentDataClient[ExtractedData[InvoiceData]]` for extracted data, or `AsyncAgentDataClient[SomeOtherDataSchema]` for misc data.
LlamaIndex
- If you use an LLM, use llama_index's openai, and `PromptTemplate` abstractions. Use an LLMs structured prediction functions for simple and fast typed outputs where the detailed parsing and high-accuracy of LlamaExtract is not necessary.
- Prefer using `FunctionAgent` for modelling chat interactions powered by tools, such as retrieval.
Integrations:
- When using llama-cloud-services make sure to add `llama_cloud = true` to the `[tool.llamadeploy]` section of `pyproject.toml`. This will set appropriate environment variables to enable the use of llama-cloud-services in the workflow automatically when using `llamactl`.
- When building llama cloud based workflows, pass around llama cloud file references, rather than file paths. Reference and search the other existing templates to understand llama cloud file integrations to upload and download files.
- When storing results from LlamaExtract, use `ExtractedData.from_extraction_result(extraction_result)`. `ExtractedData` parses, and has fields to store: citations, confidence scores, and track human corrections to the extracted data.
- When adding new environment variables that are critical to the functionality of the workflow, update the `pyproject.toml` with `required_env_vars = ["NEW_ENV_VAR"]` in the `[tool.llamadeploy]` section. This will prompt users to input them before starting the workflow.
packages/coding-agent/.gitignore
-5
View File
@@ -1,5 +0,0 @@
.claude/
claude_memory.db
CLAUDE.md
.mcp.json
sessions.db
packages/coding-agent/README.md
-25
View File
@@ -1,25 +0,0 @@
# coding-agent
To assist you with workflows.
## Try it out (at your own risk)
1. Export `ANTHROPIC_API_KEY` in your environment:
```bash
export ANTHROPIC_API_KEY="my-secret-key"
```
2. Install the package locally:
```bash
uv pip install .
```
3. Run the agent:
```bash
coder claude
```
And follow the TUI prompts to get started.
packages/coding-agent/pyproject.toml
-31
View File
@@ -1,31 +0,0 @@
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project]
name = "coding-agent"
version = "0.1.0"
description = "An agent that can edit and perform QA on workflows"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"aiosqlite>=0.21.0",
"claude-agent-sdk>=0.1.6",
"llama-index-workflows>=2.9.1",
"prompt-toolkit>=3.0.52",
"rich>=14.1.0",
"rich-gradient>=0.3.6",
"vibe-llama-core>=0.2.0",
]
[tool.hatch.build.targets.wheel]
only-include = ["src/coding_agent"]
[tool.hatch.build.targets.wheel.force-include]
"../../AGENTS.modified.md" = "coding_agent/AGENTS.md"
[tool.hatch.build.targets.wheel.sources]
"src" = ""
[project.scripts]
coder = "coding_agent.main:main"
packages/coding-agent/query.sql
-11
View File
@@ -1,11 +0,0 @@
-- name: GetMessages :many
SELECT (message_role, content) FROM chat_sessions
WHERE session_id = ?;
-- name: CreateMessage :one
INSERT INTO chat_sessions (
session_id, message_role, content
) VALUES (
?, ?, ?
)
RETURNING *;
packages/coding-agent/schema.sql
-8
View File
@@ -1,8 +0,0 @@
-- Sessions table
CREATE TABLE chat_sessions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
session_id TEXT NOT NULL,
message_role TEXT NOT NULL,
content TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
packages/coding-agent/sqlc.yaml
-18
View File
@@ -1,18 +0,0 @@
# filename: sqlc.yaml
version: "2"
plugins:
- name: python
wasm:
url: https://github.com/rayakame/sqlc-gen-better-python/releases/download/v0.4.5/sqlc-gen-better-python.wasm
sha256: 9b42f7cffda942388470e777af957ec8ab03cbea9124ff8c49bef05f9d00cb66
sql:
- engine: "sqlite"
queries: "query.sql"
schema: "schema.sql"
codegen:
- out: "src/coding_agent/persistence"
plugin: python
options:
package: "coding_agent.persistence"
emit_init_file: true
sql_driver: "aiosqlite"
packages/coding-agent/src/coding_agent/__init__.py
View File
packages/coding-agent/src/coding_agent/adapters/__init__.py
-3
View File
@@ -1,3 +0,0 @@
from .claude_code import ClaudeCodingAgent
__all__ = ["ClaudeCodingAgent"]
packages/coding-agent/src/coding_agent/adapters/claude_code/__init__.py
-3
View File
@@ -1,3 +0,0 @@
from .agent import ClaudeCodingAgent
__all__ = ["ClaudeCodingAgent"]
packages/coding-agent/src/coding_agent/adapters/claude_code/agent.py
-192
View File
@@ -1,192 +0,0 @@
from os import getenv as os_getenv
from json import load as load_json
from pathlib import Path
from typing import AsyncIterator, AsyncGenerator, Any
from contextlib import asynccontextmanager
from claude_agent_sdk import ClaudeSDKClient
from claude_agent_sdk.types import (
McpHttpServerConfig,
McpSSEServerConfig,
McpStdioServerConfig,
AssistantMessage,
TextBlock,
ResultMessage,
ClaudeAgentOptions,
ToolUseBlock,
ToolResultBlock,
ThinkingBlock,
PermissionMode,
ToolPermissionContext,
PermissionResultAllow,
PermissionResultDeny,
)
from vibe_llama_core.docs.utils import get_claude_code_skills, write_file
from .memory.mcp import sdk_mcp_server as memory_mcp
from .templates import sdk_mcp_server as templates_mcp
from .memory.db import insert_record, MemoryMatch, init_db
from ...agent import BaseCodingAgent, SkillType, Message
from .errors import MalformedMCPSpecification, UnsupportedMCPType, MissingAPIKey
from .instructions import BASE_SYSTEM_PROMPT, get_claude_md
async def _can_use_tool(t: str, d: dict[str, Any], c: ToolPermissionContext):
if t != "Bash" and "Bash" not in d:
return PermissionResultAllow()
else:
return PermissionResultDeny()
class ClaudeCodingAgent(BaseCodingAgent):
def __init__(
self,
tools: list[str],
mcp_servers_file: str,
system_prompt: str,
skills: list[SkillType] | None = None,
permissions: PermissionMode = "acceptEdits",
cwd: str | Path | None = None,
model: str = "claude-sonnet-4-5",
max_turns: int | None = None,
) -> None:
if not os_getenv("ANTHROPIC_API_KEY"):
raise MissingAPIKey(
"You need to provide an ANTHROPIC_API_KEY in your environment."
)
if system_prompt != "":
system_prompt = (
BASE_SYSTEM_PROMPT + "\n\nFURTHER INSTRUCTIONS:\n" + system_prompt
)
super().__init__(
"Claude Code", tools, mcp_servers_file, permissions, system_prompt
)
self.options = ClaudeAgentOptions(
allowed_tools=self.tools,
system_prompt=self.system_prompt,
permission_mode=self.permissions,
max_turns=max_turns,
model=model,
cwd=cwd or Path.cwd(),
disallowed_tools=["Bash", "Glob"],
)
self.skills = skills
self._is_db_inited = False
@asynccontextmanager
async def _get_client(self) -> AsyncIterator[ClaudeSDKClient]:
async with ClaudeSDKClient(self.options) as client:
yield client
def _mcp_config_from_file(self) -> None:
with open(self.mcp_file, "r") as f:
mcp_config = load_json(f)
if "mcpServers" not in mcp_config:
raise MalformedMCPSpecification(
"Could not find `mcpServers` in the MCP specification file"
)
servers = {}
for k, v in mcp_config["mcpServers"].items():
if v.get("type", "") == "http":
servers.update({k: McpHttpServerConfig(**v)})
elif v.get("type", "") == "sse":
servers.update({k: McpSSEServerConfig(**v)})
elif v.get("type", "") == "stdio":
servers.update({k: McpStdioServerConfig(**v)})
else:
raise UnsupportedMCPType(f"Unsupported MCP type: {v.get('type', '')}")
servers.update({"memory": memory_mcp})
servers.update({"templates": templates_mcp})
self.options.allowed_tools.extend(
[
"__mcp__memory__get_memory",
"__mcp__templates__get_template",
"__mcp__templates_download_template",
"__mcp__templates_search_templates",
]
)
self.options.can_use_tool = _can_use_tool
self.options.mcp_servers = servers
return None
async def warmup(self, skills: list[SkillType] | None = None) -> None:
if not self._is_db_inited:
await init_db()
self._is_db_inited = True
return await super().warmup(skills)
async def generate(self, prompt: str) -> AsyncGenerator[Message, Any]:
if self.options.permission_mode == "plan":
if self.permissions != "plan":
self.options.permission_mode = self.permissions
else:
self.options.permission_mode = "acceptEdits"
await self.warmup(self.skills)
async def gen():
async for message in self._query(prompt=prompt):
if message["type"] == "final_result":
await insert_record(
MemoryMatch(content=message["result"] or "No result")
)
yield message
return gen()
async def plan(self, prompt: str) -> AsyncGenerator[Message, Any]:
await self.warmup(self.skills)
self.options.permission_mode = "plan"
async def gen():
async for message in self._query(prompt=prompt):
if message["type"] == "final_result":
await insert_record(
MemoryMatch(content=message["result"] or "No result")
)
yield message
return gen()
async def _query(self, prompt: str) -> AsyncIterator[Message]:
async with self._get_client() as client:
await client.query(prompt=prompt)
async for message in client.receive_response():
if isinstance(message, AssistantMessage):
blocks = message.content
for block in blocks:
if isinstance(block, TextBlock):
yield {"text": block.text, "type": "text"}
elif isinstance(block, ThinkingBlock):
yield {"thinking": block.thinking, "type": "thinking"}
elif isinstance(block, ToolUseBlock):
yield {
"id": block.id,
"input": block.input,
"name": block.name,
"type": "tool_call",
}
elif isinstance(block, ToolResultBlock):
yield {
"id": block.tool_use_id,
"result": block.content,
"error": block.is_error,
"type": "tool_result",
}
elif isinstance(message, ResultMessage):
yield {
"result": message.result,
"metadata": {
"error": message.is_error,
"subtype": message.subtype,
"duration_ms": message.duration_ms,
"turns": message.num_turns,
"total_cost_usd": message.total_cost_usd,
},
"type": "final_result",
}
else:
pass
async def _get_rules(self, skills: list[SkillType] | None = None) -> None:
await get_claude_code_skills(skills=skills, overwrite_files=True) # type: ignore
claude_md = get_claude_md()
write_file("CLAUDE.md", claude_md, overwrite_file=True, service_url="")
packages/coding-agent/src/coding_agent/adapters/claude_code/errors.py
-10
View File
@@ -1,10 +0,0 @@
class MalformedMCPSpecification(Exception):
"""Raise when the MCP configuration file is malformed/misconfigured"""
class UnsupportedMCPType(Exception):
"""Raise when the MCP type is not supported"""
class MissingAPIKey(Exception):
"""Raise when ANTHROPIC_API_KEY is missing from the environment"""
packages/coding-agent/src/coding_agent/adapters/claude_code/instructions.py
-33
View File
@@ -1,33 +0,0 @@
from importlib.resources import files
from pathlib import Path
BASE_SYSTEM_PROMPT = """\
You are **Coder**, a skilled Python coding assistant with expertise in **LlamaIndex Workflows**, in particular with **Document Extraction** workflows leveraging **LlamaCloud Services**.
Your task is to follow the user's request around the document extraction use case they are presenting you with, and do your best to fulfill it using:
- The documentation available to you through the **CLAUDE.md** file (**ALWAYS**)
- The **MCP tools** available to you - focus especially on the memory tool (to retrieve past interactions and plans) and on the templates tool, to download and explore the code of templates when the user asks you to start from a template.
- Your skills, available in the folder `.claude/skills/`
**IMPORTANT INSTRUCTIONS:**
- If the user asks you to perform a task that is not related with document extraction, politely refuse to carry it out.
- Take inspiration from existing templates, leveraging the templates MCP tools.
- ALWAYS trust the documentation over your own assumptions.
- Conduct only MINIMAL project exploration unless the user explicitly requests it.
- Be concise: your code should do exactly what the user asks—nothing more, nothing less.
- Be precise: if the user requests a Python script, produce only that. The user will specify if additional output is needed.
"""
def get_claude_md():
file = files("coding_agent").joinpath("AGENTS.md")
if file.exists():
return file.read_text(encoding="utf-8")
else:
# editable mode
file = Path(__file__).parents[6] / "meta-templates" / "agents-mcp" / "AGENTS.md"
if not file.exists():
raise FileNotFoundError(
"AGENTS.md not found in either the installed directory, nor in the expected path for editable mode"
)
return file.read_text(encoding="utf-8")
packages/coding-agent/src/coding_agent/adapters/claude_code/memory/__init__.py
View File
packages/coding-agent/src/coding_agent/adapters/claude_code/memory/db/__init__.py
-4
View File
@@ -1,4 +0,0 @@
from .query import get_matches, insert_record, init_db
from .models import MemoryMatch
__all__ = ["get_matches", "insert_record", "MemoryMatch", "init_db"]
packages/coding-agent/src/coding_agent/adapters/claude_code/memory/db/constants.py
-10
View File
@@ -1,10 +0,0 @@
CREATE_VIRTUAL_TABLE_STATEMENT = """CREATE VIRTUAL TABLE IF NOT EXISTS memory
USING FTS5(content);
"""
GET_MATCHES_STATEMENT = """SELECT *
FROM memory
WHERE memory MATCH ?;
"""
INSERT_RECORD_STATEMENT = """INSERT INTO memory (content) VALUES (?);"""
packages/coding-agent/src/coding_agent/adapters/claude_code/memory/db/models.py
-6
View File
@@ -1,6 +0,0 @@
from dataclasses import dataclass
@dataclass
class MemoryMatch:
content: str
packages/coding-agent/src/coding_agent/adapters/claude_code/memory/db/query.py
-40
View File
@@ -1,40 +0,0 @@
import aiosqlite
from contextlib import asynccontextmanager
from typing import AsyncIterator
from .constants import (
CREATE_VIRTUAL_TABLE_STATEMENT,
GET_MATCHES_STATEMENT,
INSERT_RECORD_STATEMENT,
)
from .models import MemoryMatch
@asynccontextmanager
async def sqlite_connection() -> AsyncIterator[aiosqlite.Connection]:
async with aiosqlite.connect("claude_memory.db") as db:
yield db
async def init_db() -> None:
async with sqlite_connection() as conn:
await conn.execute("PRAGMA journal_mode=WAL;")
await conn.execute(CREATE_VIRTUAL_TABLE_STATEMENT)
await conn.commit()
async def get_matches(pattern: str) -> list[MemoryMatch]:
async with sqlite_connection() as conn:
rows = await (await conn.execute(GET_MATCHES_STATEMENT, (pattern,))).fetchmany(
10
)
matches: list[MemoryMatch] = []
for row in rows:
matches.append(MemoryMatch(content=row[0]))
return matches
async def insert_record(record: MemoryMatch) -> None:
async with sqlite_connection() as conn:
await conn.execute(CREATE_VIRTUAL_TABLE_STATEMENT)
await conn.execute(INSERT_RECORD_STATEMENT, (record.content,))
await conn.commit()
packages/coding-agent/src/coding_agent/adapters/claude_code/memory/mcp/__init__.py
-3
View File
@@ -1,3 +0,0 @@
from .server import sdk_mcp_server
__all__ = ["sdk_mcp_server"]
packages/coding-agent/src/coding_agent/adapters/claude_code/memory/mcp/server.py
-18
View File
@@ -1,18 +0,0 @@
from claude_agent_sdk import tool, create_sdk_mcp_server
from typing import Any
from ..db import get_matches
from dataclasses import asdict
@tool(
name="get_memory",
description="Get relevant pieces of memory matching with a specific pattern",
input_schema={"pattern": str},
)
async def get_memory(args: dict[str, Any]) -> dict[str, Any]:
pattern = args.get("pattern", "")
results = await get_matches(pattern=pattern)
return {"matches": [asdict(result) for result in results]}
sdk_mcp_server = create_sdk_mcp_server(name="local_memory_mcp", tools=[get_memory])
packages/coding-agent/src/coding_agent/adapters/claude_code/templates/__init__.py
-3
View File
@@ -1,3 +0,0 @@
from .server import sdk_mcp_server
__all__ = ["sdk_mcp_server"]
packages/coding-agent/src/coding_agent/adapters/claude_code/templates/search.py
-389
View File
@@ -1,389 +0,0 @@
from __future__ import annotations
import math
import re
import os
from dataclasses import dataclass
from pathlib import Path
from typing import Iterable, List, Dict, Tuple, Optional
import fnmatch
# -----------------------------
# Search implementation (local)
# -----------------------------
TOKEN_RE = re.compile(r"\w+", re.UNICODE)
_include_ext = {
".py",
".ts",
".tsx",
".js",
".jsx",
".json",
".md",
".txt",
".yml",
".yaml",
".toml",
".css",
".html",
}
_exclude_dirs = {
"node_modules",
"dist",
"build",
".git",
".venv",
"venv",
"env",
".mypy_cache",
".pytest_cache",
".ruff_cache",
".cache",
".next",
"out",
}
_exclude_patterns = {
"*lock.json",
"*lock.yaml",
"*.lock",
}
def _iter_template_files(templates_dir: Path) -> Iterable[Path]:
"""Yield code/content files under templates directory, skipping heavy dirs."""
max_size_bytes = 1_000_000 # 1 MB cap per file
for root, dirnames, filenames in os.walk(templates_dir):
# Prune heavy/irrelevant directories in-place
dirnames[:] = [d for d in dirnames if d not in _exclude_dirs]
# In the function:
filenames[:] = [
f
for f in filenames
if not any(fnmatch.fnmatch(f, pattern) for pattern in _exclude_patterns)
]
for fname in filenames:
path = Path(root) / fname
if path.suffix.lower() not in _include_ext:
continue
try:
if path.stat().st_size > max_size_bytes:
continue
except OSError:
continue
yield path
def _tokenize(text: str) -> List[str]:
return [t.lower() for t in TOKEN_RE.findall(text)]
@dataclass
class Match:
line: int
text: str
before: List[str]
after: List[str]
@dataclass
class SearchResult:
path: str
score: float
matches: List[Match]
def _bm25_score(
query_tokens: List[str],
doc_tf: Dict[str, int],
avgdl: float,
doc_len: int,
N: int,
df: Dict[str, int],
k1: float = 1.5,
b: float = 0.75,
) -> float:
score = 0.0
for q in query_tokens:
if q not in df or df[q] == 0:
continue
n_q = df[q]
idf = math.log(1 + (N - n_q + 0.5) / (n_q + 0.5))
f_q = doc_tf.get(q, 0)
if f_q == 0:
continue
denom = f_q + k1 * (1 - b + b * (doc_len / (avgdl if avgdl > 0 else 1.0)))
score += idf * (f_q * (k1 + 1)) / denom
return score
def _find_snippets(
lines: List[str],
query_tokens: List[str],
max_matches: int = 3,
context_lines: int = 3,
) -> List[Match]:
matches: List[Match] = []
lowered_tokens = set(query_tokens)
for idx, line in enumerate(lines, start=1):
lower_line = line.lower()
# Check if ANY query token appears as a substring in the line
if any(tok in lower_line for tok in lowered_tokens):
cleaned = line.rstrip("\n")
# Skip if the cleaned line is empty or whitespace-only
if not cleaned or not cleaned.strip():
continue
start = max(1, idx - context_lines)
end = min(len(lines), idx + context_lines)
before = [
line_text.rstrip("\n") for line_text in lines[start - 1 : idx - 1]
]
after = [line_text.rstrip("\n") for line_text in lines[idx:end]]
matches.append(Match(line=idx, text=cleaned, before=before, after=after))
if len(matches) >= max_matches:
break
return matches
def _find_repo_root(start: Path) -> Path:
"""Find the repo root by looking for a templates dir with actual template subdirs.
To distinguish between src/tmpl/templates (just code) and the actual templates directory,
we check that the templates directory contains subdirectories with pyproject.toml files.
"""
for parent in [start, *start.parents]:
templates_dir = parent / "templates"
if not templates_dir.exists() or not templates_dir.is_dir():
continue
# Check if this templates dir has template subdirectories with pyproject.toml
# (i.e., it's the actual templates dir, not just a code directory)
has_template_projects = False
try:
for entry in templates_dir.iterdir():
if entry.is_dir() and (entry / "pyproject.toml").exists():
has_template_projects = True
break
except (OSError, PermissionError):
continue
if has_template_projects:
return parent
return start
def _detect_repo_root(explicit_root: Optional[Path]) -> Path:
# Prefer explicit root if provided
if explicit_root is not None:
root = _find_repo_root(explicit_root)
if (root / "templates").exists():
return root
# Next try current working directory
cwd_root = _find_repo_root(Path.cwd())
if (cwd_root / "templates").exists():
return cwd_root
# Fallback to module location
return _find_repo_root(Path(__file__).resolve())
def get_template(
relative_path: str,
start_line: Optional[int] = None,
end_line: Optional[int] = None,
max_lines: int = 1000,
) -> str:
"""Get a template file with line numbers.
Args:
relative_path: Path relative to templates directory
start_line: Optional starting line number (1-indexed, inclusive)
end_line: Optional ending line number (1-indexed, inclusive)
max_lines: Maximum number of lines to return (default 1000)
Returns:
File contents with line numbers prefixed to each line
"""
repo_root = _detect_repo_root(Path.cwd())
templates_dir = repo_root / "templates"
if not templates_dir.exists():
raise ValueError(f"Templates directory not found at {templates_dir}")
path = templates_dir / relative_path
if not path.exists():
raise ValueError(f"Template file not found at {path}")
content = path.read_text(encoding="utf-8", errors="ignore")
lines = content.splitlines()
total_lines = len(lines)
# Apply range filtering if specified
if start_line is not None or end_line is not None:
# Convert to 0-indexed
start_idx = (start_line - 1) if start_line is not None else 0
end_idx = end_line if end_line is not None else total_lines
# Clamp to valid range
start_idx = max(0, min(start_idx, total_lines))
end_idx = max(start_idx, min(end_idx, total_lines))
lines = lines[start_idx:end_idx]
line_offset = start_idx
else:
line_offset = 0
# Apply max_lines cap
lines_to_show = lines[:max_lines]
omitted_count = len(lines) - len(lines_to_show)
# Format with line numbers
result_lines = []
for i, line in enumerate(lines_to_show, start=line_offset + 1):
# Right-align line numbers to 6 characters for consistency
result_lines.append(f"{i:6}| {line}")
result = "\n".join(result_lines)
# Add omission notice if needed
if omitted_count > 0:
result += f"\n// {omitted_count} more line(s) omitted"
return result
def search_templates_impl(
query: str,
limit: int = 10,
root: Optional[Path] = None,
context_lines: int = 3,
) -> List[SearchResult]:
"""Search the repository templates directory for code matching the query.
Uses a lightweight BM25-like ranking across files. Returns top results with
a few matching line snippets.
"""
repo_root = _detect_repo_root(root)
templates_dir = repo_root / "templates"
if not templates_dir.exists():
return []
query_tokens = _tokenize(query)
if not query_tokens:
return []
# First pass to gather term frequencies and document stats
doc_tfs: Dict[Path, Dict[str, int]] = {}
doc_lens: Dict[Path, int] = {}
df: Dict[str, int] = {}
files: List[Path] = []
path_token_sets: Dict[Path, set[str]] = {}
for path in _iter_template_files(templates_dir):
try:
text = path.read_text(encoding="utf-8", errors="ignore")
except Exception:
continue
# Combine file content tokens with relative path tokens for better recall
rel_path = path.relative_to(repo_root).as_posix()
content_tokens = _tokenize(text)
path_tokens = _tokenize(rel_path)
tokens = content_tokens + path_tokens
if not tokens:
continue
files.append(path)
tf: Dict[str, int] = {}
for tok in tokens:
tf[tok] = tf.get(tok, 0) + 1
doc_tfs[path] = tf
doc_lens[path] = len(tokens)
path_token_sets[path] = set(path_tokens)
# Update document frequency only once per doc
seen = set(tf.keys())
for tok in seen:
df[tok] = df.get(tok, 0) + 1
if not files:
return []
N = len(files)
avgdl = sum(doc_lens.values()) / float(N)
# Score documents
scored: List[Tuple[Path, float]] = []
path_bonus = 2.0
for path in files:
score = _bm25_score(query_tokens, doc_tfs[path], avgdl, doc_lens[path], N, df)
# Add bonus for matches in file path tokens
if path in path_token_sets:
bonus_hits = sum(1 for q in query_tokens if q in path_token_sets[path])
if bonus_hits:
score += path_bonus * bonus_hits
if score > 0:
scored.append((path, score))
scored.sort(key=lambda x: x[1], reverse=True)
top = scored[: max(1, limit)]
results: List[SearchResult] = []
for path, score in top:
try:
lines = path.read_text(encoding="utf-8", errors="ignore").splitlines()
except Exception:
lines = []
matches = _find_snippets(lines, query_tokens, context_lines=context_lines)
results.append(
SearchResult(
path=str(path.relative_to(repo_root)),
score=round(score, 4),
matches=matches,
)
)
return results
# -----------------------------
# MCP tool wrappers
# -----------------------------
def _merge_line_entries(matches: List[Match]) -> Dict[int, Tuple[str, bool]]:
"""Merge overlapping contexts across matches.
Returns mapping of line_number -> (text, is_match).
"""
line_entries: Dict[int, Tuple[str, bool]] = {}
for m in matches:
# Before context
start_before = m.line - len(m.before)
for i, b in enumerate(m.before):
ln = start_before + i
if ln not in line_entries:
line_entries[ln] = (b, False)
# Matched line
existing = line_entries.get(m.line)
if existing is None or existing[1] is False:
line_entries[m.line] = (m.text, True)
# After context
for i, a in enumerate(m.after):
ln = m.line + 1 + i
if ln not in line_entries:
line_entries[ln] = (a, False)
return line_entries
def format_results_pretty(results: List[SearchResult]) -> str:
"""Render search results into a pretty textual format.
Uses '*' to mark matching lines and a blank space for context lines.
"""
parts: List[str] = []
for r in results:
parts.append(f"{r.path} (score {r.score})")
line_entries = _merge_line_entries(r.matches)
for ln in sorted(line_entries.keys()):
text, is_match = line_entries[ln]
indicator = "*" if is_match else " "
parts.append(f" {ln:>5}{indicator} {text}")
return "\n".join(parts)
packages/coding-agent/src/coding_agent/adapters/claude_code/templates/server.py
-56
View File
@@ -1,56 +0,0 @@
from __future__ import annotations
from .search import (
format_results_pretty,
search_templates_impl,
get_template as get_template_impl,
)
from typing import Any
from vibe_llama_core.templates.scaffold import ProjectName
from vibe_llama_core.templates import download_template as download_template_impl
from claude_agent_sdk import tool, create_sdk_mcp_server
@tool(
name="search_templates",
description="Search the templates directory for files relevant to a query. This tool is useful when developing LlamaAgent applications, in order to identify relevant best-practice code snippets to re-use or adapt to a new use case. Use only if you already have downlaoded templates.",
input_schema={"query": str, "context": int},
)
async def search_templates(args: dict[str, Any]) -> dict[str, Any]:
query = args.get("query", "")
context = args.get("context", 10)
results = search_templates_impl(query, context_lines=max(0, int(context)))
return {"results": format_results_pretty(results)}
@tool(
name="get_template",
description="Read the full content of a template file by path. Can only be used once you downloaded at least one template.\n\nArgs:\n\tpath: Path relative to templates directory (e.g. 'basic/src/basic/workflow.py')\n\tstart_line: Optional starting line number (1-indexed, inclusive)\n\tend_line: Optional ending line number (1-indexed, inclusive)\nReturns:\n\tFile contents with line numbers. Limited to 1000 lines by default.",
input_schema={"path": str, "start_line": int | None, "end_line": int | None},
)
async def get_template(args: dict[str, Any]) -> dict[str, Any]:
path = args.get("path", "")
start_line = args.get("start_line")
end_line = args.get("end_line")
return {
"results": get_template_impl(path, start_line=start_line, end_line=end_line)
}
@tool(
name="download_template",
description="Download a template by name (among the available ones).\n\nArgs:\n\tname (Literal['basic', 'document_parsing', 'human_in_the_loop', 'invoice_extraction', 'rag', 'web_scraping']): name of the template to download.\n\nReturns:\t\nPath to which the template was downloaded.",
input_schema={"name": ProjectName},
)
async def download_template(args: dict[str, Any]) -> dict[str, Any]:
name = args.get("name", "basic")
retval = await download_template_impl(request=name)
if "SUCCESS" in retval:
return {"path": f".vibe-llama/scaffold/{name}/"}
else:
return {"error": retval}
sdk_mcp_server = create_sdk_mcp_server(
name="templates_mcp", tools=[download_template, get_template, search_templates]
)
packages/coding-agent/src/coding_agent/agent/__init__.py
-4
View File
@@ -1,4 +0,0 @@
from .base import BaseCodingAgent
from .models import AgentType, SkillType, Message
__all__ = ["BaseCodingAgent", "AgentType", "SkillType", "Message"]
packages/coding-agent/src/coding_agent/agent/base.py
-57
View File
@@ -1,57 +0,0 @@
from abc import ABC, abstractmethod
from typing import Any, AsyncGenerator
from vibe_llama_core.docs import get_agent_rules
from .models import AgentType, SkillType, Message
from .errors import WarmUpError
class BaseCodingAgent(ABC):
def __init__(
self,
agent_type: AgentType,
tools: list[Any],
mcp_servers_file: str,
permissions: Any,
system_prompt: str,
**kwargs,
) -> None:
self.tools = tools
self.mcp_file = mcp_servers_file
self.system_prompt = system_prompt
self.permissions: Any = permissions
self.agent_type = agent_type
self._agent: Any | None = None
self._is_warmed_up = False
@abstractmethod
def _mcp_config_from_file(self) -> None: ...
@abstractmethod
async def generate(self, prompt: str) -> AsyncGenerator[Message, Any]: ...
@abstractmethod
async def plan(
self, prompt: str
) -> AsyncGenerator[Message, Any]: # returns the plan
...
async def warmup(self, skills: list[SkillType] | None = None) -> None:
if not self._is_warmed_up:
try:
self._mcp_config_from_file()
await self._get_rules(skills=skills)
except Exception as e:
raise WarmUpError(
f"An error occurred while fetching rules and skills for the agent: {e}"
)
self._is_warmed_up = True
return None
async def _get_rules(self, skills: list[SkillType] | None = None) -> None:
for service in ["llama-index-workflows"]:
await get_agent_rules(
agent=self.agent_type,
service=service, # type: ignore
skills=skills or [], # type: ignore
overwrite_files=True,
)
packages/coding-agent/src/coding_agent/agent/errors.py
-2
View File
@@ -1,2 +0,0 @@
class WarmUpError(Exception):
"""Raised when an error occurs during agent warm-up"""
packages/coding-agent/src/coding_agent/agent/models.py
-64
View File
@@ -1,64 +0,0 @@
from typing import Literal, TypedDict, Any
from typing_extensions import NotRequired
AgentType = Literal[
"GitHub Copilot",
"Claude Code",
"OpenAI Codex CLI",
"Jules",
"Cursor",
"Windsurf",
"Cline",
"Amp",
"Firebase Studio",
"Open Hands",
"Gemini CLI",
"Junie",
"AugmentCode",
"Kilo Code",
"OpenCode",
"Goose",
]
SkillType = Literal[
"PDF Parsing",
"Structured Data Extraction",
"Information Retrieval",
"Text Classification",
"Llamactl Usage",
]
class ThinkingMessage(TypedDict):
type: Literal["thinking"]
thinking: str
class TextMessage(TypedDict):
type: Literal["text"]
text: str
class ToolMessage(TypedDict):
type: Literal["tool_call"]
id: str
name: str
input: dict[str, Any]
class ToolResultMessage(TypedDict):
type: Literal["tool_result"]
id: str
result: Any | None
error: NotRequired[bool | None]
class FinalResultMessage(TypedDict):
type: Literal["final_result"]
result: str | None
metadata: NotRequired[dict[str, Any]]
Message = (
ThinkingMessage | FinalResultMessage | TextMessage | ToolMessage | ToolResultMessage
)
packages/coding-agent/src/coding_agent/main.py
-5
View File
@@ -1,5 +0,0 @@
from .tui import app
def main():
app()
packages/coding-agent/src/coding_agent/persistence/__init__.py
-4
View File
@@ -1,4 +0,0 @@
# Code generated by sqlc. DO NOT EDIT.
# versions:
# sqlc v1.30.0
# sqlc-gen-better-python v0.4.5
packages/coding-agent/src/coding_agent/persistence/models.py
-23
View File
@@ -1,23 +0,0 @@
# Code generated by sqlc. DO NOT EDIT.
# versions:
# sqlc v1.30.0
# sqlc-gen-better-python v0.4.5
from __future__ import annotations
__all__: collections.abc.Sequence[str] = ("ChatSession",)
import dataclasses
import typing
if typing.TYPE_CHECKING:
import collections.abc
import datetime
@dataclasses.dataclass()
class ChatSession:
id: int
session_id: str
message_role: str
content: str | None
created_at: datetime.datetime | None
packages/coding-agent/src/coding_agent/persistence/query.py
-114
View File
@@ -1,114 +0,0 @@
# Code generated by sqlc. DO NOT EDIT.
# versions:
# sqlc v1.30.0
# sqlc-gen-better-python v0.4.5
from __future__ import annotations
__all__: collections.abc.Sequence[str] = (
"QueryResults",
"create_message",
"get_messages",
)
import operator
import typing
if typing.TYPE_CHECKING:
import aiosqlite
import collections.abc
import sqlite3
QueryResultsArgsType: typing.TypeAlias = int | float | str | memoryview | None
from coding_agent.persistence import models
CREATE_MESSAGE: typing.Final[str] = """-- name: CreateMessage :one
INSERT INTO chat_sessions (
session_id, message_role, content
) VALUES (
?, ?, ?
)
RETURNING id, session_id, message_role, content, created_at
"""
GET_MESSAGES: typing.Final[str] = """-- name: GetMessages :many
SELECT (message_role, content) FROM chat_sessions
WHERE session_id = ?
"""
T = typing.TypeVar("T")
class QueryResults(typing.Generic[T]):
__slots__ = ("_args", "_conn", "_cursor", "_decode_hook", "_iterator", "_sql")
def __init__(
self,
conn: aiosqlite.Connection,
sql: str,
decode_hook: collections.abc.Callable[[sqlite3.Row], T],
*args: QueryResultsArgsType,
) -> None:
self._conn = conn
self._sql = sql
self._decode_hook = decode_hook
self._args = args
self._cursor: aiosqlite.Cursor | None = None
self._iterator: collections.abc.AsyncIterator[sqlite3.Row] | None = None
def __aiter__(self) -> QueryResults[T]:
return self
def __await__(
self,
) -> collections.abc.Generator[None, None, collections.abc.Sequence[T]]:
async def _wrapper() -> collections.abc.Sequence[T]:
result = await (await self._conn.execute(self._sql, self._args)).fetchall()
return [self._decode_hook(row) for row in result]
return _wrapper().__await__()
async def __anext__(self) -> T:
if self._cursor is None or self._iterator is None:
self._cursor: aiosqlite.Cursor | None = await self._conn.execute(
self._sql, self._args
)
self._iterator = self._cursor.__aiter__()
try:
record = await self._iterator.__anext__()
except StopAsyncIteration:
self._cursor = None
self._iterator = None
raise
return self._decode_hook(record)
async def create_message(
conn: aiosqlite.Connection,
*,
session_id: str,
message_role: str,
content: str | None,
) -> models.ChatSession | None:
row = await (
await conn.execute(CREATE_MESSAGE, (session_id, message_role, content))
).fetchone()
if row is None:
return None
return models.ChatSession(
id=row[0],
session_id=row[1],
message_role=row[2],
content=row[3],
created_at=row[4],
)
def get_messages(
conn: aiosqlite.Connection, *, session_id: str
) -> QueryResults[typing.Any]:
return QueryResults[typing.Any](
conn, GET_MESSAGES, operator.itemgetter(0), session_id
)
packages/coding-agent/src/coding_agent/tui/__init__.py
-3
View File
@@ -1,3 +0,0 @@
from .cli import app, code_with_claude
__all__ = ["app", "code_with_claude"]
packages/coding-agent/src/coding_agent/tui/cli.py
-118
View File
@@ -1,118 +0,0 @@
"""Fully lifted from https://click.palletsprojects.com/en/stable/extending-click/"""
import click
import json
from pathlib import Path
from rich import print as rprint
from rich.markdown import Markdown
from .utils import Configuration
from .ui import AgentUI
from ..agent import AgentType
from ..adapters.claude_code.agent import BASE_SYSTEM_PROMPT as CLAUDE_BASE_SYSTEM_PROMPT
class AliasedGroup(click.Group):
"""
Implements a subclass of Group that accepts a prefix for a command.
If there was a command called push, it would accept pus as an alias (so long as it was unique):
"""
def get_command(self, ctx: click.Context, cmd_name: str) -> click.Command | None:
rv = super().get_command(ctx, cmd_name)
if rv is not None:
return rv
matches = [x for x in self.list_commands(ctx) if x.startswith(cmd_name)]
if not matches:
return None
if len(matches) == 1:
return click.Group.get_command(self, ctx, matches[0])
ctx.fail(f"Too many matches: {', '.join(sorted(matches))}")
def resolve_command(
self, ctx: click.Context, args: list[str]
) -> tuple[str, click.Command, list[str]]:
# always return the full command name
_, cmd, args = super().resolve_command(ctx, args)
return cmd.name, cmd, args # type: ignore
@click.group(
help="Convert notebooks to python scripts and run them for testing purposes",
cls=AliasedGroup,
)
def app():
pass
@app.command("claude", help="Use Claude Code as the backend for the coding agent")
@click.option(
"--system",
is_flag=True,
default=False,
required=False,
help="Show the base system prompt and exit",
)
@click.option(
"--prompt",
default=None,
required=False,
help="The task to perform. If not specified, will prompt the user for a task",
)
@click.option(
"--mode",
default=None,
required=False,
help="The mode to use for the coding agent. Either 'build' or 'plan'. If not specified, will prompt the user for a mode",
)
def code_with_claude(
system: bool = False,
prompt: str | None = None,
mode: str | None = None,
) -> None:
if system:
rprint(Markdown(f"## SYSTEM PROMPT\n\n{CLAUDE_BASE_SYSTEM_PROMPT}"))
return None
else:
agent_type: AgentType = "Claude Code"
configuration: Configuration = {
"agent_specific_args": {
"model": "claude-sonnet-4-5",
"max_turns": None,
"permissions": "acceptEdits",
"cwd": ".",
"skills": [
"PDF Parsing",
"Structured Data Extraction",
"Text Classification",
"Llamactl Usage",
],
},
"enable_persistence": True,
"mcp_servers_file": ".mcp.json",
"system_prompt": "",
"tools": ["Read", "Write"],
"provided_prompt": prompt,
"provided_mode": mode,
}
if not Path(".mcp.json").exists():
with open(".mcp.json", "w") as f:
json.dump(
{
"mcpServers": {
"llama-index-docs": {
"type": "http",
"url": "https://developers.llamaindex.ai/mcp",
}
}
},
f,
indent=2,
)
ui = AgentUI(agent_type, configuration)
return ui.run()
packages/coding-agent/src/coding_agent/tui/constants.py
-27
View File
@@ -1,27 +0,0 @@
VERBS = [
"thinking",
"pondering",
"overclocking brain",
"debugging thoughts",
"reasoning",
"speculating",
"daydreaming",
"brainstorming",
"conceptualizing",
"plotting",
"calculating",
"theorizing",
"ruminating",
"processing",
"scheming",
"optimizing ideas",
"looping on it",
"hashing it out",
"context switching",
"garbage collecting memories",
"refactoring thoughts",
"bootstrapping plan",
"compiling theories",
"updating beliefs",
"buffering",
]
packages/coding-agent/src/coding_agent/tui/logo.py
-19
View File
@@ -1,19 +0,0 @@
from rich_gradient import Gradient
from rich.panel import Panel
from rich.console import Console
LOGO = """
██████╗ ██████╗ ██████╗ ███████╗ ██████╗
██╔════╝ ██╔═══██╗ ██╔══██╗ ██╔════╝ ██╔══██╗
██║ ██║ ██║ ██║ ██║ █████╗ ██████╔╝
██║ ██║ ██║ ██║ ██║ ██╔══╝ ██╔══██╗
╚██████╗ ╚██████╔╝ ██████╔╝ ███████╗ ██║ ██║
╚═════╝ ╚═════╝ ╚═════╝ ╚══════╝ ╚═╝ ╚═╝
"""
def print_logo(console: Console):
panel = Panel(LOGO, padding=(2, 2), border_style="bold")
console.print(Gradient(panel, rainbow=True), justify="center")
print()
print()
packages/coding-agent/src/coding_agent/tui/ui.py
-18
View File
@@ -1,18 +0,0 @@
import asyncio
from rich.console import Console
from .utils import run_workflow, Configuration, print_welcome_message
from ..agent import AgentType
from .logo import print_logo
class AgentUI:
def __init__(self, agent_type: AgentType, config: Configuration) -> None:
self.agent_type: AgentType = agent_type
self._console = Console()
self.config: Configuration = config
def run(self) -> None:
print_logo(self._console)
print_welcome_message(self._console, self.agent_type)
asyncio.run(run_workflow(self._console, self.config, self.agent_type))
packages/coding-agent/src/coding_agent/tui/utils.py
-191
View File
@@ -1,191 +0,0 @@
from random import randint
from typing import TypedDict, Any
from rich.console import Console
from rich.panel import Panel
from rich.markdown import Markdown
from rich.status import Status
from rich.pretty import Pretty
from prompt_toolkit import prompt as ask_user_input
from prompt_toolkit.key_binding import KeyBindings
from ..agent import Message, AgentType
from .constants import VERBS
from ..workflow.workflow import (
CodingAgentWorkflow,
InputConfig,
OutputSummary,
MessageEvent,
HumanDecision,
QuestionToHuman,
)
def prompt(
console: Console, question: str, options: list[str] | None, status: Status
) -> str:
status.stop()
if options:
for option in options:
question += f"\n- {option}"
question += "\n"
if not options:
kb = KeyBindings()
@kb.add("c-s")
def _(event):
# Accept input on Enter (or use a different key combo)
event.current_buffer.validate_and_handle()
question += "> _use `ctrl+s` to submit_\n"
multiline = True
else:
kb = KeyBindings()
@kb.add("enter")
def _(event):
# Accept input on Enter (or use a different key combo)
event.current_buffer.validate_and_handle()
multiline = False
console.print(
Panel(Markdown(question), border_style="blue", title="Human Input Required")
)
answer = ask_user_input("→ ", multiline=multiline, key_bindings=kb, in_thread=True)
console.print(
Panel(f"Your Answer: {answer}", title="Submitted", border_style="green")
)
status.start()
return answer
def display_message(console: Console, message: Message, status: Status):
if message["type"] == "text":
status.update(VERBS[randint(0, len(VERBS) - 1)] + "...")
status.stop()
console.print(
Panel(Markdown(message["text"]), title="Agent Message", title_align="left")
)
status.start()
elif message["type"] == "thinking":
status.update(VERBS[randint(0, len(VERBS) - 1)] + "...")
status.stop()
console.print(
Panel(
Markdown(message["thinking"]),
title="Agent Thoughts",
title_align="left",
border_style="yellow",
)
)
status.start()
elif message["type"] == "tool_call":
status.update(f"Calling tool {message['name']}...")
status.stop()
console.print(
Panel(
Pretty(message["input"]),
title="Tool Call Input",
title_align="left",
border_style="red",
)
)
status.start()
elif message["type"] == "tool_result":
status.update("Retrieving tool result...")
status.stop()
if "error" in message and message["error"]:
console.print(
Panel(
f"An error occurred while executing tool call {message['id']}",
title=f"Tool Result for {message['id']}",
title_align="left",
border_style="red",
)
)
else:
if isinstance(message["result"], str):
console.print(
Panel(
Markdown(message["result"]),
title=f"Tool Result for {message['id']}",
title_align="left",
border_style="cyan",
)
)
else:
console.print(
Panel(
Pretty(message["result"]),
title=f"Tool Result for {message['id']}",
title_align="left",
border_style="cyan",
)
)
status.start()
elif message["type"] == "final_result":
status.update("Getting the final result...")
status.stop()
res = message["result"] or "No result produced"
console.print(
Panel(
Markdown(res),
title="Final result",
title_align="left",
border_style="green",
)
)
status.update("Working on your request...")
status.start()
class Configuration(TypedDict):
tools: list[str]
mcp_servers_file: str
system_prompt: str
agent_specific_args: dict[str, Any]
enable_persistence: bool
provided_prompt: str | None
provided_mode: str | None
async def run_workflow(console: Console, config: Configuration, agent_type: AgentType):
wf = CodingAgentWorkflow(timeout=1800)
start_event = InputConfig(
agent_specific_options=config["agent_specific_args"],
enable_persistence=config["enable_persistence"],
mcp_file=config["mcp_servers_file"],
system_prompt=config["system_prompt"],
tools=config["tools"],
agent_type=agent_type,
provided_prompt=config["provided_prompt"],
provided_mode=config["provided_mode"],
)
handler = wf.run(start_event=start_event)
with Status("Starting to work on your request...") as status:
async for ev in handler.stream_events():
if isinstance(ev, MessageEvent):
display_message(message=ev.message, console=console, status=status)
elif isinstance(ev, QuestionToHuman):
answer = prompt(
console=console,
question=ev.question,
options=ev.options,
status=status,
)
handler.ctx.send_event(HumanDecision(response=answer, type=ev.type)) # type: ignore
elif isinstance(ev, OutputSummary):
console.print(Markdown("## OUTPUT SUMMARY"))
console.print(Markdown(ev.summary))
else:
pass
await handler
return None
def print_welcome_message(console: Console, agent_type: AgentType):
console.print(
Markdown(
f"## Welcome to Coder!\n\nHi, I am Coder, your assistant, based on {agent_type}, for everything concerning building and editing **LlamaIndex Workflows**.\n\n> _Tip: If you with to exit, do so by using the `/exit` command_\n\n---\n\n"
)
)
print()
packages/coding-agent/src/coding_agent/workflow/__init__.py
View File
packages/coding-agent/src/coding_agent/workflow/constants.py
-15
View File
@@ -1,15 +0,0 @@
from coding_agent.agent import AgentType, BaseCodingAgent
from coding_agent.adapters.claude_code import ClaudeCodingAgent
from typing import Type
SUPPORTED_AGENTS: dict[AgentType, Type[BaseCodingAgent]] = {
"Claude Code": ClaudeCodingAgent
}
CREATE_TABLE_STATEMENT = """CREATE TABLE IF NOT EXISTS chat_sessions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
session_id TEXT NOT NULL,
message_role TEXT NOT NULL,
content TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
);"""
packages/coding-agent/src/coding_agent/workflow/errors.py
-6
View File
@@ -1,6 +0,0 @@
class UnsupportAgentError(Exception):
"""Raise when an unsupported coding agent is requested by the user"""
class UnrecognizedMode(Exception):
"""Raise when the HumanDecision event has an unrecognized mode"""
packages/coding-agent/src/coding_agent/workflow/events.py
-70
View File
@@ -1,70 +0,0 @@
from typing import Any, Literal
from typing_extensions import Self
from pydantic import model_validator
from ..agent import AgentType, Message
from workflows.events import (
StartEvent,
StopEvent,
HumanResponseEvent,
InputRequiredEvent,
Event,
)
class InputConfig(StartEvent):
agent_specific_options: dict[str, Any]
enable_persistence: bool
agent_type: AgentType
mcp_file: str
system_prompt: str
tools: list[str]
class QuestionToHuman(InputRequiredEvent):
type: Literal[
"mode_choice", "prompt", "approve_mode_change", "approve_final_result"
]
question: str
options: list[str] | None = None
@model_validator(mode="after")
def validate_options_by_type(self) -> Self:
if self.type == "mode_choice":
self.options = ["build", "plan"]
elif self.type == "approve_final_result":
self.options = ["yes", "no"]
elif self.type == "approve_mode_change":
self.options = [
"yes",
"no, and provide details on what should be done further/differently in the current mode",
]
else:
self.options = None
return self
class HumanDecision(HumanResponseEvent):
type: Literal[
"mode_choice", "prompt", "approve_mode_change", "approve_final_result"
]
response: str
class MessageEvent(Event):
message: Message
class Plan(Event):
pass
class Build(Event):
pass
class Finalize(Event):
event: Literal["plan", "build"]
class OutputSummary(StopEvent):
summary: str
packages/coding-agent/src/coding_agent/workflow/models.py
-22
View File
@@ -1,22 +0,0 @@
from uuid import uuid4
from typing import Literal
from pydantic import BaseModel, Field
class ChatMessage(BaseModel):
content: str
role: Literal["user", "assistant", "result"]
def _get_session_id() -> str:
return str(uuid4())
class CodingWorkflowState(BaseModel):
session_id: str = Field(default_factory=_get_session_id)
chat_history: list[ChatMessage] = Field(default_factory=list)
enable_persistance: bool = False
current_mode: Literal["plan", "build"] | None = None
current_prompt: str | None = None
current_plan: str = ""
current_result: str = ""
packages/coding-agent/src/coding_agent/workflow/utils.py
-26
View File
@@ -1,26 +0,0 @@
import aiosqlite
from coding_agent.agent import BaseCodingAgent, AgentType
from typing import Type, AsyncIterator
from .errors import UnsupportAgentError
from .constants import SUPPORTED_AGENTS
from contextlib import asynccontextmanager
def agent_type_to_agent_class(agent: AgentType) -> Type[BaseCodingAgent]:
if agent in SUPPORTED_AGENTS:
return SUPPORTED_AGENTS[agent]
else:
raise UnsupportAgentError(
f"Agent {agent} is not supported. Supported agents are: {', '.join(list(SUPPORTED_AGENTS.keys()))}"
)
def check_if_yes(s: str) -> bool:
return s.strip().lower() in ("y", "yes", "ys", "yse")
@asynccontextmanager
async def sqlite_connection() -> AsyncIterator[aiosqlite.Connection]:
async with aiosqlite.connect("sessions.db") as db:
yield db
packages/coding-agent/src/coding_agent/workflow/workflow.py
-227
View File
@@ -1,227 +0,0 @@
import json
from workflows import Workflow, step, Context
from typing import cast
from .models import CodingWorkflowState, ChatMessage
from .events import (
InputConfig,
Plan,
Build,
Finalize,
OutputSummary,
HumanDecision,
QuestionToHuman,
MessageEvent,
)
from .utils import agent_type_to_agent_class, check_if_yes, sqlite_connection
from .errors import UnrecognizedMode
from .constants import CREATE_TABLE_STATEMENT
from ..persistence.query import create_message
from ..agent import BaseCodingAgent
class CodingAgentWorkflow(Workflow):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._agent: BaseCodingAgent | None = None
self._is_db_inited = False
async def _init_db(self) -> None:
async with sqlite_connection() as db:
await db.execute("PRAGMA journal_mode=WAL;")
await db.execute(CREATE_TABLE_STATEMENT)
await db.commit()
self._is_db_inited = True
@step
async def configure_agent(
self, ctx: Context[CodingWorkflowState], ev: InputConfig
) -> QuestionToHuman | Build | Plan:
agent_cls = agent_type_to_agent_class(ev.agent_type)
self._agent = agent_cls(
tools=ev.tools,
system_prompt=ev.system_prompt,
mcp_servers_file=ev.mcp_file,
**ev.agent_specific_options,
) # type: ignore
async with ctx.store.edit_state() as state:
state.enable_persistance = ev.enable_persistence
if ev.provided_prompt:
state.current_prompt = ev.provided_prompt
if ev.provided_mode:
state.current_mode = ev.provided_mode
return await self.resolve_agent(ctx)
async def resolve_agent(
self, ctx: Context[CodingWorkflowState]
) -> QuestionToHuman | Build | Plan:
state = await ctx.store.get_state()
if not state.current_prompt:
return QuestionToHuman(
question="What would you like me to do today?",
type="prompt",
)
elif not state.current_mode:
return QuestionToHuman(
question="Please choose a mode:",
type="mode_choice",
)
else:
if state.current_mode == "plan":
return Plan()
elif state.current_mode == "build":
return Build()
else:
raise UnrecognizedMode(f"Unrecognized mode: {state.current_mode}")
@step
async def handle_human_response(
self, ctx: Context[CodingWorkflowState], ev: HumanDecision
) -> Build | Plan | QuestionToHuman | Finalize:
if ev.response == "/exit":
return Finalize(event="build")
elif ev.type == "prompt":
async with ctx.store.edit_state() as state:
state.chat_history.append(ChatMessage(content=ev.response, role="user"))
state.current_prompt = ev.response
return await self.resolve_agent(ctx)
elif ev.type == "mode_choice":
if ev.response.strip().lower() not in ["build", "plan"]:
return await self.resolve_agent(ctx)
async with ctx.store.edit_state() as state:
state.current_mode = ev.response.strip().lower()
return await self.resolve_agent(ctx)
elif ev.type == "approve_final_result":
state = await ctx.store.get_state()
if check_if_yes(ev.response):
return Finalize(event=state.current_mode)
else:
return QuestionToHuman(
question="Since the above result did not satisfy you, what would you like to do different?",
type="prompt",
)
elif ev.type == "approve_mode_change":
if check_if_yes(ev.response):
async with ctx.store.edit_state() as state:
other = {"build", "plan"} - {state.current_mode}
state.current_mode = other.pop()
return await self.resolve_agent(ctx)
else:
async with ctx.store.edit_state() as state:
state.current_prompt = ev.response
return await self.resolve_agent(ctx)
else:
raise UnrecognizedMode(f"Unrecognized mode: {ev.type}")
@step
async def handle_finalize(
self, ev: Finalize, ctx: Context[CodingWorkflowState]
) -> QuestionToHuman | OutputSummary:
state = await ctx.store.get_state()
if ev.event == "plan":
return QuestionToHuman(
type="approve_mode_change",
question="Should we now change the mode of the coding agent to 'build'?",
)
else:
summary = f"Session {state.session_id} leveraged {self._agent.agent_type} as agent and consisted of {len(state.chat_history)} messages. The final result of this session was:\n{state.current_result}\n" # type: ignore
if state.enable_persistance:
if not self._is_db_inited:
await self._init_db()
async with sqlite_connection() as db:
try:
for message in state.chat_history:
await create_message(
conn=db,
session_id=state.session_id,
message_role=message.role,
content=message.content,
)
await db.commit()
except Exception as e:
summary += f"There was an error while saving the session to the database: {e}."
else:
summary += " The session history was correctly exported to the local database."
return OutputSummary(summary=summary)
else:
return OutputSummary(summary=summary)
@step
async def handle_build(
self, ev: Build, ctx: Context[CodingWorkflowState]
) -> QuestionToHuman:
state = await ctx.store.get_state()
agent = cast(BaseCodingAgent, self._agent)
prompt = state.current_prompt
if state.current_plan != "":
prompt += f"\n\nNOTE:\nBe aware of the plan you just made for this:\n{state.current_plan}\n"
messages = await agent.generate(prompt=prompt)
async for message in messages:
is_result = False
if message["type"] == "final_result":
state.current_result = message["result"] or "No result was produced"
content = message["result"] or "No result was produced"
is_result = True
elif message["type"] == "text":
content = message["text"]
elif message["type"] == "thinking":
content = message["thinking"]
elif message["type"] == "tool_call" or message["type"] == "tool_result":
try:
content = json.dumps(
{k: v for k, v in message.items() if k != "type"}
)
except Exception:
content = str({k: v for k, v in message.items() if k != "type"})
else:
content = ""
async with ctx.store.edit_state() as state:
state.chat_history.append(
ChatMessage(
content=content, role="assistant" if not is_result else "result"
)
)
ctx.write_event_to_stream(MessageEvent(message=message))
return QuestionToHuman(
question="Do you approve this result as the final result?",
type="approve_final_result",
)
@step
async def handle_plan(
self, ev: Plan, ctx: Context[CodingWorkflowState]
) -> QuestionToHuman:
state = await ctx.store.get_state()
agent = cast(BaseCodingAgent, self._agent)
prompt = state.current_prompt
if state.current_plan != "":
prompt += f"\n\nNOTE:\nBe aware of the previous plan you made:\n{state.current_plan}\n"
messages = await agent.plan(prompt=prompt)
async for message in messages:
is_result = False
if message["type"] == "final_result":
state.current_result = message["result"] or "No result was produced"
content = message["result"] or "No result was produced"
is_result = True
elif message["type"] == "text":
content = message["text"]
elif message["type"] == "thinking":
content = message["thinking"]
elif message["type"] == "tool_call" or message["type"] == "tool_result":
try:
content = json.dumps(
{k: v for k, v in message.items() if k != "type"}
)
except Exception:
content = str({k: v for k, v in message.items() if k != "type"})
else:
content = ""
async with ctx.store.edit_state() as state:
state.chat_history.append(
ChatMessage(
content=content, role="assistant" if not is_result else "result"
)
)
ctx.write_event_to_stream(MessageEvent(message=message))
return QuestionToHuman(
question="Do you approve the plan?", type="approve_final_result"
)
pyproject.toml
-67
View File
@@ -1,67 +0,0 @@
[project]
name = "tmpl"
version = "0.1.0"
description = "Add your description here"
readme = "README.md"
authors = [{ name = "Adrian Lyjak", email = "adrianlyjak@gmail.com" }]
requires-python = ">=3.13"
dependencies = [
"click>=8.3.0",
"copier>=9.10.2",
"jinja2>=3.1.6",
"pytest>=8.4.2",
"pyyaml>=6.0.3",
"rich>=14.1.0",
"tomlkit>=0.13.3",
"requests>=2.32.3",
"posthog>=3.7.4",
"packaging>=24.1",
"fastmcp>=2.12.5",
]
[project.scripts]
tmpl = "tmpl.cli:cli"
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[dependency-groups]
dev = [
"hatch>=1.14.2",
"mypy>=1.18.2",
"pytest>=8.4.2",
"ruff>=0.13.2",
"ty>=0.0.1a21",
]
[tool.hatch.build.targets.wheel]
packages = ["src/tmpl"]
[tool.hatch.build.targets.wheel.force-include]
".github/templates-remotes.yml" = "tmpl/.github/templates-remotes.yml"
[tool.hatch.envs.default.scripts]
format = "ruff format ."
format-check = "ruff format --check ."
lint = "ruff check --fix ."
lint-check = ["ruff check ."]
typecheck = "ty check src"
test = "pytest"
all-check = ["format-check", "lint-check", "test", "typecheck"]
all-fix = ["format", "lint", "test", "typecheck"]
tmpl-all-fix = [
"tmpl all check-javascript --fix",
"tmpl all check-python --fix",
"tmpl all check-workflows",
]
[tool.pytest.ini_options]
testpaths = ["tests"]
[tool.uv.sources]
coding-agent = {workspace = true}
[tool.uv.workspace]
members = ["packages/*"]
scripts/ensure-repos.py
-78
View File
@@ -1,78 +0,0 @@
#!/usr/bin/env -S uv run --script
# /// script
# dependencies=[
# "requests",
# ]
# ///
import os
import sys
import requests
# ===== Config =====
ORG = "run-llama"
# Token from env
TOKEN = os.getenv("GITHUB_PAT") or os.getenv("GITHUB_TOKEN")
if not TOKEN:
print(
"ERROR: Set GITHUB_PAT or GITHUB_TOKEN env var to your GitHub Personal Access Token.",
file=sys.stderr,
)
sys.exit(1)
# Build repo names (include org in each repo name)
repos = [
"template-workflow-classify-extract-sec",
]
API = "https://api.github.com"
def repo_exists(org: str, name: str) -> bool:
url = f"{API}/repos/{org}/{name}"
r = requests.get(url, headers={"Authorization": f"token {TOKEN}"})
return r.status_code == 200
def create_repo(org: str, name: str):
if repo_exists(org, name):
print(f"SKIP: {org}/{name} already exists.")
return
url = f"{API}/orgs/{org}/repos"
payload = {
"name": name,
"description": "Llama Index Workflow Template",
"private": False,
"auto_init": True,
}
r = requests.post(
url,
headers={
"Authorization": f"token {TOKEN}",
"Accept": "application/vnd.github+json",
},
json=payload,
)
if r.status_code >= 300:
print(
f"ERROR: Failed to create {org}/{name}: {r.status_code} {r.text}",
file=sys.stderr,
)
else:
print(f"Created: https://github.com/{org}/{name}")
def main():
for repo in repos:
create_repo(ORG, repo)
print("\nAll repo names:")
for r in repos:
print(f" - {r}")
if __name__ == "__main__":
main()
src/tmpl/__init__.py
-5
View File
@@ -1,5 +0,0 @@
__all__ = []
def main() -> None:
print("Hello from tmpl!")
src/tmpl/checks/__init__.py
-7
View File
@@ -1,7 +0,0 @@
"""Validation checks for tmpl."""
from .python import run_python_checks
from .javascript import run_javascript_checks
from .workflows import validate_workflows
__all__ = ["run_python_checks", "run_javascript_checks", "validate_workflows"]
src/tmpl/checks/javascript.py
-21
View File
@@ -1,21 +0,0 @@
"""JavaScript/TypeScript validation checks."""
from __future__ import annotations
from pathlib import Path
from ..utils import console, run_git_command
def run_javascript_checks(test_proj_dir: Path, fix: bool) -> None:
"""Run TypeScript and format validation checks on test/ui using npm."""
ui_dir: Path = test_proj_dir / "ui"
if not ui_dir.exists():
console.print(
"test/ui directory does not exist. Ignoring JavaScript checks.",
style="yellow",
)
return
console.print("Running TypeScript validation checks...")
run_git_command(["npm", "run", "all-fix" if fix else "all-check"], cwd=ui_dir)
console.print("✓ TypeScript checks passed")
src/tmpl/checks/python.py
-17
View File
@@ -1,17 +0,0 @@
"""Python validation checks."""
from __future__ import annotations
from pathlib import Path
from ..utils import console, run_git_command
def run_python_checks(test_proj_dir: Path, fix: bool) -> None:
"""Run Python validation checks on test directory using hatch."""
console.print("Running Python validation checks...")
run_git_command(
["uv", "run", "hatch", "run", "all-fix" if fix else "all-check"],
cwd=test_proj_dir,
)
console.print("✓ Python checks passed")
src/tmpl/checks/workflows.py
-120
View File
@@ -1,120 +0,0 @@
"""Workflow validation checks."""
from __future__ import annotations
import os
import shlex
import shutil
import subprocess
from pathlib import Path
from typing import Dict, List, Tuple
import tomllib
from ..utils import console
def _read_pyproject(path: Path) -> Dict:
with path.open("rb") as fp:
return tomllib.load(fp)
def _extract_workflows(doc: Dict) -> Dict[str, str]:
tool = doc.get("tool", {})
if "llamactl" in tool and isinstance(tool["llamactl"], dict):
wf = tool["llamactl"].get("workflows", {})
if isinstance(wf, dict):
return wf # type: ignore[return-value]
if "llamadeploy" in tool and isinstance(tool["llamadeploy"], dict):
wf = tool["llamadeploy"].get("workflows", {})
if isinstance(wf, dict):
return wf # type: ignore[return-value]
return {}
def _collect_workflows_for_project(project_dir: Path) -> List[Tuple[str, str]]:
pyproject_path = project_dir / "pyproject.toml"
try:
doc = _read_pyproject(pyproject_path)
except Exception as e:
raise SystemExit(f"Failed to parse {pyproject_path}: {e}")
workflows = _extract_workflows(doc)
results: List[Tuple[str, str]] = []
for name, import_path in workflows.items():
if not isinstance(import_path, str):
continue
results.append((name, import_path))
return results
def validate_workflows(template_dir: Path) -> None:
"""Validate all workflows for a single project.
- Discovers workflows in the project's pyproject under llamactl/llamadeploy
- Imports the object and invokes its private _validate() method
- Exits with error if any workflow fails
"""
uv_path = shutil.which("uv")
if uv_path is None:
raise SystemExit(
"uv not available; required to run within project dependencies"
)
workflows = _collect_workflows_for_project(template_dir)
assert workflows is not None, (
f"Every project should have at least one workflow. None found for project {template_dir.name}"
)
failures: List[str] = []
# If more are needed, perhaps derive them from the project's .env.template by default
env = os.environ.copy()
env["LLAMA_CLOUD_API_KEY"] = os.getenv("LLAMA_CLOUD_API_KEY", "sk-fake***fake")
for wf_name, import_path in workflows:
code = f"""
import importlib, inspect
mod, attr = {import_path!r}.split(":", 1)
module = importlib.import_module(mod)
obj = getattr(module, attr)
if inspect.isclass(obj):
try:
wf = obj(timeout=None)
except TypeError:
wf = obj()
else:
wf = obj
validate = getattr(wf, "_validate", None)
if not callable(validate):
raise SystemExit("Workflow has no callable _validate() method")
validate()
"""
cmd = [uv_path, "run", "python", "-c", code]
proc = subprocess.run(
cmd,
env=env,
cwd=str(template_dir),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
timeout=300,
check=False,
)
if proc.returncode != 0:
stdout = proc.stdout or ""
failures.append(
"\n".join(
[
"********************************************************************************",
f"Validation failed for {template_dir.name}:{wf_name} -> {import_path}",
f"Command: {' '.join(map(shlex.quote, cmd))}",
"Output:",
stdout,
]
)
)
if failures:
for msg in failures:
console.print(msg, highlight=False)
raise SystemExit(1)
console.print("✓ Workflow validation passed", style="green")
src/tmpl/cli.py
-514
View File
@@ -1,514 +0,0 @@
"""CLI interface for tmpl - Template monorepo manager."""
from __future__ import annotations
import asyncio
import json
import os
import shutil
import sys
from pathlib import Path
from typing import Optional
import click
from .config import MAPPING_DATA
from .config import get_mapping_data, get_mapping_write_path
from .git.versioning import (
detect_changed_templates,
apply_version_bumps,
save_mapping_versions,
)
from .git.tagging import run_tag_versions
from .git import (
clone_templates,
detect_base_ref,
list_changed_files,
merge_template,
mirror_template,
templates_from_files,
)
from .templates import (
get_template_dir,
)
from .checks import run_javascript_checks, run_python_checks, validate_workflows
from .init.scripts import init_python_scripts, init_package_json_scripts
from .metrics.exporter import send_posthog_event, GitHubAuth, get_all_events_for_export
from packaging.version import Version
from .utils import console
from .mcp.mcp import (
run_stdio as mcp_run_stdio,
search_templates_impl,
format_results_pretty,
)
@click.group()
def cli() -> None:
"""Template monorepo manager."""
pass
@cli.command("list")
@click.option("--detail", "detail", is_flag=True, default=False)
def list_cmd(detail: bool) -> None:
"""
List configured templates and their remote URL and branch.
"""
for name, cfg in MAPPING_DATA.items():
print(name)
if detail:
print(f" remote: {cfg.get('remote')}")
print(f" url: {cfg.get('url')}")
print(f" branch: {cfg.get('branch')}")
if cfg.get("version"):
print(f" version: {cfg.get('version')}")
@cli.command("version")
@click.option("--base", "base_ref", default=None, help="Base ref/sha to diff against")
@click.option("--head", "head_ref", default="HEAD", help="Head ref/sha to diff from")
@click.option(
"--committed-only/--include-uncommitted",
default=False,
help="Only consider committed changes (default includes uncommitted)",
)
def version_cmd(base_ref: Optional[str], head_ref: str, committed_only: bool) -> None:
"""
Bump versions for templates that have changed.
Detects changed templates (like 'changed'), then interactively prompts for each
whether the change is major, minor, patch, or ignore. Updates the version
in .github/templates-remotes.yml accordingly (defaulting to 0.0.0 when missing).
"""
changed = detect_changed_templates(
head_ref=head_ref,
base_ref=base_ref,
include_uncommitted=not committed_only,
detect_base_ref_func=detect_base_ref,
list_changed_files_func=list_changed_files,
templates_from_files_func=templates_from_files,
mapping_keys=list(MAPPING_DATA.keys()),
)
if not changed:
console.print("No template changes detected.", style="yellow")
return
mapping = get_mapping_data()
def parse_version(v: Optional[str]) -> Version:
try:
return Version(v) if v else Version("0.0.0")
except Exception:
return Version("0.0.0")
decisions: dict[str, str] = {}
for name in changed:
choice = click.prompt(
f"Version change for {name}",
type=click.Choice(
["major", "minor", "patch", "ignore"], case_sensitive=False
),
default="ignore",
show_choices=True,
)
decisions[name] = choice.lower()
new_mapping = apply_version_bumps(mapping, decisions)
if mapping == new_mapping:
console.print("No versions changed.", style="yellow")
return
save_mapping_versions(new_mapping, get_mapping_write_path())
console.print("Updated versions:", style="green")
for name, action in decisions.items():
if action != "ignore" and name in new_mapping:
console.print(f" - {name} -> {new_mapping[name].get('version')}")
@cli.command("tag-versions")
@click.option("--dry-run", is_flag=True, help="Print actions without fetching/pushing")
def tag_versions_cmd(dry_run: bool) -> None:
"""
Ensure remote tags exist for all templates with a configured version.
For each template with a `version` in the mapping, checks the remote for tag
`vX.Y.Z`. If missing, computes the subtree split commit for the template and pushes
the tag pointing to that commit to the remote.
"""
tagged = run_tag_versions(dry_run=dry_run)
if tagged:
console.print("Created/updated tags:", style="green")
for t in tagged:
name, tag = t.split(":", 1)
console.print(f" - {name} -> {tag}")
@cli.command("changed")
@click.option("--base", "base_ref", default=None, help="Base ref/sha to diff against")
@click.option("--head", "head_ref", default="HEAD", help="Head ref/sha to diff from")
@click.option("--format", "fmt", type=click.Choice(["json", "lines"]), default="json")
@click.option("--github-output", is_flag=True, help="Write GitHub Actions outputs")
@click.option(
"--committed-only/--include-uncommitted",
default=False,
help="Only consider committed changes (default includes uncommitted)",
)
def changed_cmd(
base_ref: Optional[str],
head_ref: str,
fmt: str,
github_output: bool,
committed_only: bool,
) -> None:
"""
Show which templates changed between two git refs.
- --base: base ref/sha to diff against (auto-detected if omitted)
- --head: head ref/sha to diff from (default: HEAD)
- --format: output format (json|lines)
- --github-output: also write outputs to $GITHUB_OUTPUT (json, has_changes)
"""
head = head_ref
base = base_ref or detect_base_ref(head)
files = list_changed_files(base, head, include_uncommitted=not committed_only)
changed = templates_from_files(files, list(MAPPING_DATA.keys()))
if fmt == "json":
print(json.dumps(changed))
else:
for t in changed:
print(t)
if github_output:
gh_out = os.getenv("GITHUB_OUTPUT")
if gh_out:
with open(gh_out, "a", encoding="utf-8") as f:
f.write(f"json={json.dumps(changed)}\n")
f.write(f"has_changes={'true' if changed else 'false'}\n")
@cli.command("mirror")
@click.argument("template_name", type=click.Choice(MAPPING_DATA.keys()))
def mirror_cmd(template_name: str) -> None:
"""
Push the contents of templates/<name> to its upstream repository.
Ensures the configured remote exists, then runs
`git subtree push --prefix templates/<name> <remote> <branch>` to mirror
the template subtree to the external repo/branch.
"""
cfg = MAPPING_DATA.get(template_name)
if not cfg:
raise SystemExit(f"No mapping found for template: {template_name}")
mirror_template(template_name, cfg)
@cli.command("merge")
@click.argument("template_name", type=click.Choice(MAPPING_DATA.keys()))
def merge_cmd(template_name: str) -> None:
"""
Merge upstream changes into templates/<name> from its configured remote.
Runs `git subtree pull --prefix templates/<name> <remote> <branch> --squash`.
"""
cfg = MAPPING_DATA.get(template_name)
if not cfg:
raise SystemExit(f"No mapping found for template: {template_name}")
merge_template(template_name, cfg)
@cli.command("clone")
@click.argument("template_name", type=click.Choice(MAPPING_DATA.keys()))
def clone_cmd(template_name: str) -> None:
"""
Add all configured templates under templates/ via git subtree (initial import).
For each template not already present locally, runs
`git subtree add --prefix templates/<name> <remote> <branch> --squash`.
"""
clone_templates({template_name: MAPPING_DATA[template_name]})
@cli.command("check-python")
@click.argument("template_name", type=click.Choice(MAPPING_DATA.keys()))
@click.option("--fix", is_flag=True, help="Fix formatting issues automatically.")
def template_check_python_cmd(template_name: str, fix: bool) -> None:
"""Run Python validation checks on test directory."""
template_dir = get_template_dir(template_name)
run_python_checks(template_dir, fix)
@cli.command("check-javascript")
@click.argument("template_name", type=click.Choice(MAPPING_DATA.keys()))
@click.option("--fix", is_flag=True, help="Fix formatting issues automatically.")
def template_check_javascript_cmd(template_name: str, fix: bool) -> None:
"""Run JavaScript/TypeScript validation checks on test directory."""
template_dir = get_template_dir(template_name)
run_javascript_checks(template_dir, fix)
@cli.command("init-scripts")
@click.argument("template_name", type=click.Choice(MAPPING_DATA.keys()))
def init_python_scripts_cmd(template_name: str) -> None:
"""Initialize development scripts for a template (Python and JavaScript/TypeScript)."""
template_dir = get_template_dir(template_name)
console.print(f"Working directory: {template_dir}")
init_python_scripts(template_name)
init_package_json_scripts(template_name)
@cli.command("export-metrics")
@click.option(
"--dry-run", is_flag=True, help="Do not send to PostHog; just print JSON."
)
@click.option(
"--print", "print_json", is_flag=True, help="Print JSON results to stdout."
)
@click.option(
"--backfill",
is_flag=True,
help="Send daily events for all days in the 14-day window.",
)
def export_metrics_cmd(dry_run: bool, print_json: bool, backfill: bool) -> None:
"""Export GitHub metrics for all configured template repositories.
Requires GITHUB_TOKEN (or GITHUB_PAT). If not --dry-run, also requires
POSTHOG_API_KEY and optionally POSTHOG_HOST.
"""
gh_auth = GitHubAuth.from_env()
metrics, events = get_all_events_for_export(
MAPPING_DATA, github_auth=gh_auth, backfill=backfill
)
if print_json or dry_run:
print(json.dumps(metrics))
# Always print all events that will be/were sent
if events:
print(
f"\n{'DRY RUN - Would send' if dry_run else 'Sending'} {len(events)} events:"
)
for event in events:
template = event.properties.get("template", "unknown")
print(
f" - {event.event_name} for {template}"
+ (f" at {event.timestamp}" if event.timestamp else "")
)
if not dry_run:
for event in events:
send_posthog_event(
event.event_name,
event.properties,
timestamp=event.timestamp,
)
@cli.command("check-workflows")
@click.argument("template_name", type=click.Choice(MAPPING_DATA.keys()))
def check_workflows_cmd(template_name: str) -> None:
"""Validate workflows for a single template's project."""
template_dir = get_template_dir(template_name)
validate_workflows(template_dir)
@cli.command(
"all", context_settings={"ignore_unknown_options": True, "allow_extra_args": True}
)
@click.argument("command_name")
@click.option(
"--continue-on-error",
is_flag=True,
help="Continue executing on other templates if one fails",
)
@click.pass_context
def all_cmd(ctx: click.Context, command_name: str, continue_on_error: bool) -> None:
"""Execute a command on all templates.
Run any tmpl command across all configured templates. The template name
will be automatically passed as the first argument to the command.
Examples:
tmpl all check-python --fix
tmpl all mirror --continue-on-error
"""
# Get any extra arguments passed to the command
extra_args = ctx.args
template_names = list(MAPPING_DATA.keys())
if not template_names:
console.print("No templates configured.", style="yellow")
return
extras_str = " ".join(extra_args) if extra_args else ""
console.print(
f"Running 'tmpl {command_name}{(' ' + extras_str) if extras_str else ''}' on {len(template_names)} templates..."
)
failed_templates = []
for template_name in template_names:
console.print(f"\n📁 Processing template: {template_name}", style="bold blue")
# Invoke the target subcommand within the same Click app to preserve flag parsing
args_for_cmd = [command_name, template_name, *extra_args]
try:
# Run the subcommand in-process; do not call sys.exit on errors
cli.main(args=args_for_cmd, standalone_mode=False)
console.print(f"✓ {template_name} completed successfully", style="green")
except SystemExit as e:
exit_code = int(e.code) if isinstance(e.code, int) else 1
console.print(
f"❌ {template_name} failed with exit code {exit_code}", style="red"
)
failed_templates.append(template_name)
if not continue_on_error:
console.print(
f"Stopping execution due to failure in {template_name}", style="red"
)
console.print(
"Use --continue-on-error to continue processing other templates",
style="yellow",
)
sys.exit(exit_code)
except KeyboardInterrupt:
console.print(
f"\n🛑 Interrupted while processing {template_name}", style="yellow"
)
sys.exit(130)
# Summary
console.print("\n📊 Summary:", style="bold")
successful_count = len(template_names) - len(failed_templates)
console.print(
f"✓ Successful: {successful_count}/{len(template_names)}", style="green"
)
if failed_templates:
console.print(
f"❌ Failed: {len(failed_templates)}/{len(template_names)}", style="red"
)
console.print("Failed templates:", style="red")
for template in failed_templates:
console.print(f" - {template}", style="red")
sys.exit(1)
else:
console.print("🎉 All templates processed successfully!", style="bold green")
@cli.command("mcp-stdio")
def mcp_stdio_cmd() -> None:
"""Start the MCP server for tmpl tools."""
asyncio.run(mcp_run_stdio())
@cli.command("init-agents-mcp")
@click.option(
"--target",
"target_dir",
type=click.Path(),
default=".",
help="Target directory (default: current directory)",
)
def init_agents_mcp_cmd(target_dir: str) -> None:
"""Copy meta-template files for local MCP development to target directory.
Copies files from meta-templates/agents-mcp/ to the target directory
and creates a symlink from CLAUDE.md to AGENTS.md.
"""
# Get paths
repo_root = Path(__file__).parent.parent.parent
source_dir = repo_root / "meta-templates" / "agents-mcp"
target_path = Path(target_dir).resolve()
# Validate source directory exists
if not source_dir.exists():
console.print(
f"Source directory not found: {source_dir}",
style="red",
)
sys.exit(1)
# Ensure target directory exists
target_path.mkdir(parents=True, exist_ok=True)
console.print(f"Merging {source_dir} into {target_path}...", style="blue")
# Recursively copy, merging into the target directory
shutil.copytree(source_dir, target_path, dirs_exist_ok=True)
console.print(" ✓ Merge complete", style="green")
# Create symlink from CLAUDE.md to AGENTS.md
claude_md = target_path / "CLAUDE.md"
# Remove existing CLAUDE.md if it exists (whether file or symlink)
if claude_md.exists() or claude_md.is_symlink():
claude_md.unlink()
# Create the symlink
claude_md.symlink_to("AGENTS.md")
console.print(" ✓ Created symlink: CLAUDE.md -> AGENTS.md", style="green")
# If a ui directory exists, optionally merge the UI meta-templates
ui_target = target_path / "ui"
ui_source = repo_root / "meta-templates" / "agents-mcp-ui"
if ui_target.exists() and ui_target.is_dir():
if ui_source.exists() and ui_source.is_dir():
console.print(
f"Merging {ui_source} into {ui_target}...",
style="blue",
)
shutil.copytree(ui_source, ui_target, dirs_exist_ok=True)
console.print(" ✓ UI merge complete", style="green")
# Create symlink from ui/CLAUDE.md to ui/AGENTS.md
ui_claude_md = ui_target / "CLAUDE.md"
if ui_claude_md.exists() or ui_claude_md.is_symlink():
ui_claude_md.unlink()
ui_claude_md.symlink_to("AGENTS.md")
console.print(
" ✓ Created UI symlink: ui/CLAUDE.md -> AGENTS.md", style="green"
)
else:
console.print(
f"Skipping UI merge: source not found at {ui_source}",
style="yellow",
)
else:
console.print(
"No ui directory found in target; skipping UI merge", style="yellow"
)
console.print(
f"\n✓ Successfully initialized agents-mcp files in {target_path}",
style="bold green",
)
@cli.command("search-templates")
@click.argument("query", type=str)
@click.option("--limit", "limit", type=int, default=10, help="Max results to return")
@click.option(
"--context", "context", type=int, default=10, help="Lines of context around matches"
)
def search_templates_cmd(query: str, limit: int, context: int) -> None:
"""Search the templates directory for files relevant to QUERY.
This runs the local search directly without starting the MCP server.
"""
results = search_templates_impl(query, limit=limit, context_lines=max(0, context))
# pretty format
if not results:
console.print("No results.", style="yellow")
return
console.print(f"Query: '{query}' (top {len(results)})", style="bold")
# Use shared pretty formatter so CLI and MCP agree
pretty = format_results_pretty(results)
for line in pretty.splitlines():
console.print(line, markup=False)
src/tmpl/config/__init__.py
-17
View File
@@ -1,17 +0,0 @@
"""Configuration management for tmpl."""
from .mapping import (
MAPPING_DATA,
TemplatesMapping,
_load_mapping,
get_mapping_data,
get_mapping_write_path,
)
__all__ = [
"MAPPING_DATA",
"TemplatesMapping",
"_load_mapping",
"get_mapping_data",
"get_mapping_write_path",
]
src/tmpl/config/mapping.py
-134
View File
@@ -1,134 +0,0 @@
"""Template mapping configuration management.
Non-configurable mapping discovery:
- Discover a workspace mapping file by walking up from this module's path,
looking for `.github/templates-remotes.yml`.
- If not found (e.g., installed non-editable), fall back to an embedded
default mapping bundled with the package.
- Expose a memoized getter so callers can treat it like a constant.
"""
from __future__ import annotations
from pathlib import Path
from typing import Any, Dict, TypedDict, Optional, cast
from functools import lru_cache
from importlib.resources import files
import yaml
class TemplatesMapping(TypedDict):
"""Configuration for a template mapping."""
remote: str # tpl-wf-document-qa
url: str # git@github.com:run-llama/template-workflow-document-qa.git
branch: str # main
version: Optional[str] # semantic version string like 1.2.3
def _parse_mapping_dict(data: dict[str, object]) -> dict[str, TemplatesMapping]:
templates_section = data.get("templates", {})
assert isinstance(templates_section, dict)
out: dict[str, TemplatesMapping] = {}
for k, v in templates_section.items():
v = cast(dict[str, Any], v)
remote = v["remote"]
assert isinstance(remote, str)
url = v["url"]
assert isinstance(url, str)
branch = v["branch"]
assert isinstance(branch, str)
version = v.get("version")
assert isinstance(version, str) or version is None
assert isinstance(k, str), "Key of mapping must be a string"
out[k] = TemplatesMapping(
remote=remote,
url=url,
branch=branch,
version=version,
)
return out
def _load_mapping(path: Path) -> Dict[str, TemplatesMapping]:
"""Load template mappings from a YAML file path."""
with path.open("r", encoding="utf-8") as f:
data = yaml.safe_load(f) or {}
assert isinstance(data, dict)
return _parse_mapping_dict(data)
def save_mapping(path: Path, mapping: Dict[str, TemplatesMapping]) -> None:
"""Persist template mappings to the YAML file.
Writes the `templates` section with keys: remote, url, branch, and optional version.
"""
data: Dict[str, Dict[str, Dict[str, str]]] = {"templates": {}}
for name, cfg in mapping.items():
entry: Dict[str, str] = {
"remote": cfg.get("remote", ""),
"url": cfg.get("url", ""),
"branch": cfg.get("branch", "main"),
}
version = cfg.get("version")
if version:
entry["version"] = version
data["templates"][name] = entry
with path.open("w", encoding="utf-8") as f:
yaml.safe_dump(data, f, sort_keys=False)
def load_bundled_mapping() -> Dict[str, TemplatesMapping]:
"""Load the bundled default mapping from the package resources."""
try:
mapping_file = files("tmpl.config").joinpath("templates-remotes.yml")
content = mapping_file.read_text(encoding="utf-8")
data = yaml.safe_load(content) or {}
assert isinstance(data, dict)
return _parse_mapping_dict(data)
except Exception:
# If bundled file is not available, return empty mapping
return {}
def discover_repo_mapping_path() -> Optional[Path]:
"""Discover the repository mapping file path by walking parents.
Returns a Path if `.github/templates-remotes.yml` exists relative to any
parent of this module; otherwise returns None.
"""
here = Path(__file__).resolve()
for parent in here.parents:
candidate = parent / ".github" / "templates-remotes.yml"
if candidate.exists():
return candidate
return None
@lru_cache(maxsize=1)
def get_mapping_data() -> Dict[str, TemplatesMapping]:
"""Return template mapping data, discovered or default (memoized)."""
repo_path = discover_repo_mapping_path()
if repo_path is not None:
return _load_mapping(repo_path)
# Fallback to bundled mapping file
return load_bundled_mapping()
def get_mapping_write_path() -> Path:
"""Return the path to write the mapping file.
Raises if no repository mapping file can be discovered.
"""
repo_path = discover_repo_mapping_path()
if repo_path is None:
raise RuntimeError(
"Unable to locate repository mapping file for write: .github/templates-remotes.yml"
)
return repo_path
# Load the global mapping data via the memoized getter
MAPPING_DATA = get_mapping_data()
src/tmpl/config/settings.py
-4
View File
@@ -1,4 +0,0 @@
"""Global settings and constants for tmpl.
Deprecated: mapping path is now discovered automatically.
"""
src/tmpl/git/__init__.py
-15
View File
@@ -1,15 +0,0 @@
"""Git operations for tmpl."""
from .changes import detect_base_ref, list_changed_files, templates_from_files
from .operations import ensure_remote
from .subtree import clone_templates, merge_template, mirror_template
__all__ = [
"detect_base_ref",
"list_changed_files",
"templates_from_files",
"ensure_remote",
"clone_templates",
"merge_template",
"mirror_template",
]
src/tmpl/git/changes.py
-69
View File
@@ -1,69 +0,0 @@
"""Git change detection utilities."""
from __future__ import annotations
import os
import subprocess
from typing import Iterable, List, Set
from ..utils import git_output, run
def detect_base_ref(head: str) -> str:
"""Detect the base reference for comparing changes."""
base_ref = os.getenv("GITHUB_BASE_REF")
if base_ref:
try:
git_output(["rev-parse", f"origin/{base_ref}"])
except subprocess.CalledProcessError:
run(["git", "fetch", "--no-tags", "--depth", "1", "origin", base_ref])
return git_output(["merge-base", f"origin/{base_ref}", head])
for candidate in ("main", "master"):
try:
git_output(["rev-parse", f"origin/{candidate}"])
except subprocess.CalledProcessError:
continue
return git_output(["merge-base", f"origin/{candidate}", head])
return git_output(["rev-parse", f"{head}~1"]) # returns stdout as str
def list_changed_files(
base: str, head: str, include_uncommitted: bool = True
) -> List[str]:
"""List files changed between two git references.
When include_uncommitted is True (default), also include staged and unstaged
working tree changes relative to HEAD.
"""
committed_out = git_output(["diff", "--name-only", f"{base}...{head}"])
changed: Set[str] = {line for line in committed_out.splitlines() if line}
if include_uncommitted:
# Staged changes
staged_out = git_output(["diff", "--name-only", "--cached"])
for line in staged_out.splitlines():
if line:
changed.add(line)
# Unstaged changes
unstaged_out = git_output(["diff", "--name-only"])
for line in unstaged_out.splitlines():
if line:
changed.add(line)
return sorted(changed)
def templates_from_files(files: Iterable[str], template_names: list[str]) -> List[str]:
"""Extract template names from a list of file paths."""
templates: Set[str] = set()
start_paths = []
for name in template_names:
start_paths.append("templates/" + name)
for path in files:
if path.startswith(tuple(start_paths)):
parts = path.split("/")
if len(parts) >= 2 and parts[1]:
templates.add(parts[1])
return sorted(templates)
src/tmpl/git/operations.py
-24
View File
@@ -1,24 +0,0 @@
"""Basic git operations."""
from __future__ import annotations
import subprocess
from ..utils import git_output, run
def ensure_remote(name: str, url: str) -> None:
"""Ensure a git remote exists with the correct URL."""
try:
current = git_output(["remote", "get-url", name])
except subprocess.CalledProcessError:
current = None
if current is None:
print(f"Adding missing remote {name} -> {url}")
run(["git", "remote", "add", name, url])
elif current != url:
print(f"Updating remote {name} url to {url}")
run(["git", "remote", "set-url", name, url])
else:
print(f"Remote {name} already configured correctly")
src/tmpl/git/subtree.py
-65
View File
@@ -1,65 +0,0 @@
"""Git subtree operations for template management."""
from __future__ import annotations
from pathlib import Path
from typing import Dict
from ..config import TemplatesMapping
from ..utils import run
from .operations import ensure_remote
def mirror_template(template_name: str, config: TemplatesMapping) -> None:
"""Push the contents of templates/<name> to its upstream repository."""
remote = config.get("remote")
url = config.get("url")
branch = config.get("branch", "main")
if not remote or not url:
raise SystemExit(f"Template {template_name} missing 'remote' or 'url'")
# Ensure the git remote exists and is pointed at the correct URL
ensure_remote(remote, url)
prefix = f"templates/{template_name}"
# Push subtree to remote branch
cmd = ["git", "subtree", "push", "--prefix", prefix, remote, branch]
run(cmd)
def merge_template(template_name: str, config: TemplatesMapping) -> None:
"""Merge upstream changes into templates/<name> from its configured remote."""
remote = config.get("remote")
url = config.get("url")
branch = config.get("branch", "main")
if not remote or not url:
raise SystemExit(f"Template {template_name} missing 'remote' or 'url'")
ensure_remote(remote, url)
prefix = f"templates/{template_name}"
run(["git", "subtree", "pull", "--prefix", prefix, remote, branch, "--squash"])
def clone_templates(mapping_data: Dict[str, TemplatesMapping]) -> None:
"""Add all configured templates under templates/ via git subtree (initial import)."""
root = Path.cwd()
templates_dir = root / "templates"
templates_dir.mkdir(parents=True, exist_ok=True)
for name, cfg in mapping_data.items():
url = cfg.get("url")
remote = cfg.get("remote")
branch = cfg.get("branch", "main")
if not url or not remote:
print(f"Skipping {name}: missing remote or url")
continue
# Ensure the git remote exists and is pointed at the correct URL
ensure_remote(remote, url)
prefix = f"templates/{name}"
dest = templates_dir / name
if dest.exists():
print(f"Exists: {dest}")
continue
# Add the upstream template as a subtree under templates/<name>
run(["git", "subtree", "add", "--prefix", prefix, remote, branch, "--squash"])
src/tmpl/git/tagging.py
-155
View File
@@ -1,155 +0,0 @@
from __future__ import annotations
from typing import Dict, List, Optional
from ..config import TemplatesMapping, get_mapping_data
from ..utils import git_output, run
from ..utils.console import console
def ensure_remote(remote: str, url: str) -> None:
"""Ensure the remote exists without emitting error noise if missing."""
try:
git_output(["remote", "get-url", remote])
except Exception:
run(["git", "remote", "add", remote, url])
def compute_subtree_split(prefix: str, branch: str) -> str:
return git_output(["subtree", "split", "--prefix", prefix, branch])
def get_remote_tag_hash(remote: str, tag_name: str) -> Optional[str]:
"""Return the hash of a tag on a remote, if present."""
try:
out = git_output(["ls-remote", "--tags", remote, tag_name])
if not out:
return None
lines = out.splitlines()
# Prefer peeled entry (annotated tags): refs/tags/<tag>^{}
for line in lines:
if line.endswith(f"refs/tags/{tag_name}^{{}}"):
return line.split()[0]
# Fallback to the first line (lightweight tags or annotated without peeled)
first = lines[0]
return first.split()[0] if first else None
except Exception:
return None
def remote_has_tag(remote: str, tag_name: str) -> bool:
return get_remote_tag_hash(remote, tag_name) is not None
def push_tag_to_remote(remote: str, commit: str, tag_name: str) -> None:
run(["git", "push", remote, f"{commit}:refs/tags/{tag_name}", "-f"])
def tag_all_versions(
mapping: Dict[str, TemplatesMapping], dry_run: bool = False
) -> List[str]:
"""Create missing tags on remotes for all templates.
Returns a list of "name:tag" strings created. Aggregates errors per-template
and continues; callers should check for errors via exceptions from run_tag_versions.
"""
tagged: List[str] = []
errors: List[str] = []
for name, cfg in mapping.items():
try:
version = cfg.get("version")
if not version:
continue
remote = cfg.get("remote")
url = cfg.get("url")
branch = cfg.get("branch", "main")
if not remote or not url:
console.print(
f"Skipping {name}: missing remote or url in mapping", style="yellow"
)
continue
console.print(
f"\n🔖 Tagging {name}: version=v{version} remote={remote} url={url} branch={branch}",
style="bold",
)
ensure_remote(remote, url)
console.print("Fetching remote branch...")
# Ensure objects (including the configured branch and tags) from the subtree
# remote are present locally.
run(["git", "fetch", remote, branch])
tag_name = f"v{version}"
prefix = f"templates/{name}"
commit = compute_subtree_split(prefix, branch)
console.print(
f"Computed subtree split for {prefix} on {branch}: {commit}",
style="dim",
)
existing_hash = get_remote_tag_hash(remote, tag_name)
if existing_hash:
# Detailed diagnostics when tag already exists on remote
console.print(
f"Remote {remote} tag {tag_name} = {existing_hash}", style="dim"
)
console.print(f"Computed subtree commit = {commit}", style="dim")
if existing_hash != commit:
console.print(
(
f"⚠️ {name}: remote tag {tag_name} points to a different commit than the current subtree split.\n"
f" This likely indicates unversioned changes or a missing version bump.\n"
f" Skipping push to avoid clobbering an existing tag."
),
style="yellow",
)
else:
console.print(
f"✓ {name}: remote tag matches computed subtree commit",
style="green",
)
# Skip pushing if tag exists; behavior unchanged.
continue
if dry_run:
console.print(
f"DRY-RUN: would push tag {tag_name} -> {commit} to {remote}",
style="cyan",
)
tagged.append(f"{name}:{tag_name}")
else:
console.print(
f"Pushing tag {tag_name} -> {commit} to {remote}", style="cyan"
)
push_tag_to_remote(remote, commit, tag_name)
tagged.append(f"{name}:{tag_name}")
console.print(
f"✓ Created {tag_name} on {remote} at {commit}", style="green"
)
except SystemExit as e:
code = int(e.code) if isinstance(e.code, int) else 1
errors.append(
f"{name}: failed with exit code {code} while tagging v{cfg.get('version')} on {cfg.get('remote') or 'unknown remote'}"
)
# Continue to next template
except Exception as e: # safeguard
errors.append(f"{name}: unexpected error: {e}")
if errors:
console.print(
"\n⚠️ One or more tagging operations encountered errors:",
style="bold yellow",
)
for err in errors:
console.print(f" - {err}", style="yellow")
# Do not fail the overall command; provide diagnostics only.
return tagged
def run_tag_versions(dry_run: bool = False) -> List[str]:
mapping = get_mapping_data()
return tag_all_versions(mapping, dry_run=dry_run)
src/tmpl/git/versioning.py
-61
View File
@@ -1,61 +0,0 @@
from __future__ import annotations
from pathlib import Path
from typing import Dict, List, Optional
from packaging.version import Version
from ..config import TemplatesMapping
def detect_changed_templates(
*,
head_ref: str,
base_ref: Optional[str],
include_uncommitted: bool = True,
detect_base_ref_func,
list_changed_files_func,
templates_from_files_func,
mapping_keys: List[str],
) -> List[str]:
head = head_ref
base = base_ref or detect_base_ref_func(head)
files = list_changed_files_func(base, head, include_uncommitted=include_uncommitted)
return templates_from_files_func(files, mapping_keys)
def bump_version_string(current: Optional[str], bump: str) -> str:
ver = Version(current) if current else Version("0.0.0")
major, minor, patch = ver.major, ver.minor, ver.micro
if bump == "major":
major += 1
minor = 0
patch = 0
elif bump == "minor":
minor += 1
patch = 0
elif bump == "patch":
patch += 1
return f"{major}.{minor}.{patch}"
def apply_version_bumps(
mapping: Dict[str, TemplatesMapping],
decisions: Dict[str, str],
) -> Dict[str, TemplatesMapping]:
# Deep-copy mapping entries so original mapping remains unchanged
updated: Dict[str, TemplatesMapping] = {k: dict(v) for k, v in mapping.items()} # type: ignore[dict-item]
for name, action in decisions.items():
if action == "ignore":
continue
if name not in updated:
continue
new_version = bump_version_string(updated[name].get("version"), action)
updated[name]["version"] = new_version # type: ignore[index]
return updated
def save_mapping_versions(mapping: Dict[str, TemplatesMapping], path: Path) -> None:
from ..config.mapping import save_mapping # avoid cycle on import
save_mapping(path, mapping)
src/tmpl/init/scripts.py
-251
View File
@@ -1,251 +0,0 @@
"""Initialize Python development scripts and tooling for templates."""
from __future__ import annotations
import json
import subprocess
from pathlib import Path
from typing import cast
import tomlkit
from tmpl.templates.manager import get_template_dir
from ..utils import console
def init_python_scripts(template_name: str) -> None:
"""Initialize Python scripts for a template.
This function:
1. Adds development dependencies (ty, pytest, ruff, hatch) using uv
2. Configures hatch commands in pyproject.toml for common development tasks
"""
template_dir = get_template_dir(template_name)
pyproject_path = template_dir / "pyproject.toml"
if not pyproject_path.exists():
console.print(f"❌ No pyproject.toml found in {template_dir}", style="bold red")
raise SystemExit(1)
console.print(f"📦 Adding development dependencies to {template_name}")
# Add development dependencies using uv
try:
subprocess.run(
["uv", "add", "--dev", "ty", "pytest", "ruff", "hatch"],
cwd=template_dir,
check=True,
capture_output=True,
text=True,
)
console.print("✓ Development dependencies added", style="green")
except subprocess.CalledProcessError as e:
console.print(f"❌ Failed to add dependencies: {e.stderr}", style="bold red")
return
# Modify pyproject.toml to add hatch commands
console.print("⚙️ Configuring hatch commands in pyproject.toml")
with open(pyproject_path, "r", encoding="utf-8") as f:
doc = tomlkit.load(f)
# Ensure tool.hatch.envs.default.scripts section exists
print("getting table")
scripts = get_table(["tool", "hatch", "envs", "default", "scripts"], doc)
hatch_commands = {
"format": "ruff format .",
"format-check": "ruff format --check .",
"lint": "ruff check --fix .",
"lint-check": ["ruff check ."],
"typecheck": "ty check src",
"test": "pytest",
"all-check": ["format-check", "lint-check", "test"],
"all-fix": ["format", "lint", "test"],
}
for cmd, value in hatch_commands.items():
scripts[cmd] = value
set_table(["tool", "hatch", "envs", "default", "scripts"], doc, scripts)
# Write back to file
with open(pyproject_path, "w", encoding="utf-8") as f:
tomlkit.dump(doc, f)
console.print("✓ Hatch commands configured", style="green")
# Ensure test folder exists with placeholder test if needed
_ensure_test_folder(template_dir)
# Ensure gitignore contains required items
_ensure_gitignore_items(template_dir)
def get_table(path: list[str], doc: tomlkit.TOMLDocument) -> tomlkit.TOMLDocument:
"""Get a table from a path in a tomlkit document."""
for p in path:
doc = cast(tomlkit.TOMLDocument, doc.get(p, tomlkit.table()))
return doc
def set_table(
path: list[str], doc: tomlkit.TOMLDocument, value: tomlkit.TOMLDocument
) -> None:
"""Set a table in a path in a tomlkit document."""
for p in path[:-1]:
if p not in doc or not isinstance(doc[p], tomlkit.TOMLDocument):
doc[p] = tomlkit.table()
next_doc = cast(tomlkit.TOMLDocument, doc[p])
doc = next_doc
doc[path[-1]] = value
def init_package_json_scripts(template_name: str) -> None:
"""Initialize JavaScript/TypeScript scripts for a template.
This function:
1. Adds development dependencies (prettier, typescript) using pnpm
2. Configures npm scripts in package.json for common development tasks
"""
template_dir = get_template_dir(template_name)
package_json_path = template_dir / "ui" / "package.json"
if not package_json_path.exists():
console.print(
f"⚠️ No package.json found in {template_dir}. Ignoring...", style="yellow"
)
return
console.print(f"📦 Adding development dependencies to {template_name}")
# Add development dependencies using pnpm
try:
subprocess.run(
["pnpm", "add", "-D", "prettier", "typescript"],
cwd=template_dir,
check=True,
capture_output=True,
text=True,
)
console.print("✓ Development dependencies added", style="green")
except subprocess.CalledProcessError as e:
console.print(f"❌ Failed to add dependencies: {e.stderr}", style="bold red")
return
# Modify package.json to add npm scripts
console.print("⚙️ Configuring npm scripts in package.json")
try:
with open(package_json_path, "r", encoding="utf-8") as f:
package_data = json.load(f)
# Ensure scripts section exists
if "scripts" not in package_data:
package_data["scripts"] = {}
scripts = package_data["scripts"]
# Add the npm scripts
npm_scripts = {
"lint": "tsc --noEmit",
"format": "prettier --write src",
"format-check": "prettier --check src",
"all-check": "pnpm i && pnpm run lint && pnpm run format-check && pnpm run build",
"all-fix": "pnpm i && pnpm run lint && pnpm run format && pnpm run build",
}
for cmd, value in npm_scripts.items():
scripts[cmd] = value
# Write back to file with proper formatting
with open(package_json_path, "w", encoding="utf-8") as f:
json.dump(package_data, f, indent=2, ensure_ascii=False)
f.write("\n") # Add trailing newline
console.print("✓ npm scripts configured", style="green")
except Exception as e:
console.print(f"❌ Failed to configure npm scripts: {e}", style="bold red")
def _ensure_test_folder(template_dir: Path) -> None:
"""Ensure test folder exists with placeholder test if no tests are present."""
test_dir = template_dir / "tests"
# Create tests directory if it doesn't exist
if not test_dir.exists():
test_dir.mkdir(parents=True, exist_ok=True)
console.print("📁 Created tests directory", style="green")
# Check if there are any Python test files
test_files = list(test_dir.glob("**/*.py"))
if not test_files:
# Create a placeholder test file
placeholder_test = test_dir / "test_placeholder.py"
placeholder_content = '''"""Placeholder test file.
Replace this with actual tests for your project.
"""
import pytest
def test_placeholder() -> None:
"""Placeholder test that always passes.
Remove this test once you add real tests to your project.
"""
assert True
'''
with open(placeholder_test, "w", encoding="utf-8") as f:
f.write(placeholder_content)
console.print("✓ Created placeholder test file", style="green")
def _ensure_gitignore_items(template_dir: Path) -> None:
"""Ensure .gitignore contains required items."""
gitignore_path = template_dir / ".gitignore"
required_items = [
"workflows.db",
".venv",
".env",
"package-lock.json",
"node_modules",
]
# Read existing gitignore content if it exists
existing_content = ""
if gitignore_path.exists():
with open(gitignore_path, "r", encoding="utf-8") as f:
existing_content = f.read()
# Check which items are missing
existing_lines = [
line.strip() for line in existing_content.split("\n") if line.strip()
]
missing_items = [item for item in required_items if item not in existing_lines]
if missing_items:
# Add missing items to gitignore
if existing_content and not existing_content.endswith("\n"):
existing_content += "\n"
if existing_content:
existing_content += "\n"
for item in missing_items:
existing_content += f"{item}\n"
with open(gitignore_path, "w", encoding="utf-8") as f:
f.write(existing_content)
console.print(
f"✓ Added {len(missing_items)} items to .gitignore: {', '.join(missing_items)}",
style="green",
)
else:
console.print("✓ .gitignore already contains all required items", style="green")
src/tmpl/mcp/mcp.py
-49
View File
@@ -1,49 +0,0 @@
from __future__ import annotations
from fastmcp import FastMCP
from tmpl.mcp.search import (
format_results_pretty,
search_templates_impl,
get_template as get_template_impl,
)
# Initialize MCP server
mcp = FastMCP("tmpl MCP Server")
async def run_stdio() -> None:
"""Start the MCP server."""
await mcp.run_stdio_async(show_banner=False)
async def run_server() -> None:
"""Start the MCP server."""
await mcp.run_async()
@mcp.tool
def search_templates(query: str, context: int = 10) -> str:
"""Search the templates directory for files relevant to a query. This tool is useful when developing LlamaAgent applications, in order to
identify relevant best-practice code snippets to re-use or adapt to a new use case."""
results = search_templates_impl(query, context_lines=max(0, int(context)))
return format_results_pretty(results)
@mcp.tool
def get_template(
path: str, start_line: int | None = None, end_line: int | None = None
) -> str:
"""Read the full content of a template file by path.
Args:
path: Path relative to templates directory (e.g. 'basic/src/basic/workflow.py')
start_line: Optional starting line number (1-indexed, inclusive)
end_line: Optional ending line number (1-indexed, inclusive)
Returns:
File contents with line numbers. Limited to 1000 lines by default.
"""
return get_template_impl(path, start_line=start_line, end_line=end_line)
src/tmpl/mcp/search.py
-389
View File
@@ -1,389 +0,0 @@
from __future__ import annotations
import math
import re
import os
from dataclasses import dataclass
from pathlib import Path
from typing import Iterable, List, Dict, Tuple, Optional
import fnmatch
# -----------------------------
# Search implementation (local)
# -----------------------------
TOKEN_RE = re.compile(r"\w+", re.UNICODE)
_include_ext = {
".py",
".ts",
".tsx",
".js",
".jsx",
".json",
".md",
".txt",
".yml",
".yaml",
".toml",
".css",
".html",
}
_exclude_dirs = {
"node_modules",
"dist",
"build",
".git",
".venv",
"venv",
"env",
".mypy_cache",
".pytest_cache",
".ruff_cache",
".cache",
".next",
"out",
}
_exclude_patterns = {
"*lock.json",
"*lock.yaml",
"*.lock",
}
def _iter_template_files(templates_dir: Path) -> Iterable[Path]:
"""Yield code/content files under templates directory, skipping heavy dirs."""
max_size_bytes = 1_000_000 # 1 MB cap per file
for root, dirnames, filenames in os.walk(templates_dir):
# Prune heavy/irrelevant directories in-place
dirnames[:] = [d for d in dirnames if d not in _exclude_dirs]
# In the function:
filenames[:] = [
f
for f in filenames
if not any(fnmatch.fnmatch(f, pattern) for pattern in _exclude_patterns)
]
for fname in filenames:
path = Path(root) / fname
if path.suffix.lower() not in _include_ext:
continue
try:
if path.stat().st_size > max_size_bytes:
continue
except OSError:
continue
yield path
def _tokenize(text: str) -> List[str]:
return [t.lower() for t in TOKEN_RE.findall(text)]
@dataclass
class Match:
line: int
text: str
before: List[str]
after: List[str]
@dataclass
class SearchResult:
path: str
score: float
matches: List[Match]
def _bm25_score(
query_tokens: List[str],
doc_tf: Dict[str, int],
avgdl: float,
doc_len: int,
N: int,
df: Dict[str, int],
k1: float = 1.5,
b: float = 0.75,
) -> float:
score = 0.0
for q in query_tokens:
if q not in df or df[q] == 0:
continue
n_q = df[q]
idf = math.log(1 + (N - n_q + 0.5) / (n_q + 0.5))
f_q = doc_tf.get(q, 0)
if f_q == 0:
continue
denom = f_q + k1 * (1 - b + b * (doc_len / (avgdl if avgdl > 0 else 1.0)))
score += idf * (f_q * (k1 + 1)) / denom
return score
def _find_snippets(
lines: List[str],
query_tokens: List[str],
max_matches: int = 3,
context_lines: int = 3,
) -> List[Match]:
matches: List[Match] = []
lowered_tokens = set(query_tokens)
for idx, line in enumerate(lines, start=1):
lower_line = line.lower()
# Check if ANY query token appears as a substring in the line
if any(tok in lower_line for tok in lowered_tokens):
cleaned = line.rstrip("\n")
# Skip if the cleaned line is empty or whitespace-only
if not cleaned or not cleaned.strip():
continue
start = max(1, idx - context_lines)
end = min(len(lines), idx + context_lines)
before = [
line_text.rstrip("\n") for line_text in lines[start - 1 : idx - 1]
]
after = [line_text.rstrip("\n") for line_text in lines[idx:end]]
matches.append(Match(line=idx, text=cleaned, before=before, after=after))
if len(matches) >= max_matches:
break
return matches
def _find_repo_root(start: Path) -> Path:
"""Find the repo root by looking for a templates dir with actual template subdirs.
To distinguish between src/tmpl/templates (just code) and the actual templates directory,
we check that the templates directory contains subdirectories with pyproject.toml files.
"""
for parent in [start, *start.parents]:
templates_dir = parent / "templates"
if not templates_dir.exists() or not templates_dir.is_dir():
continue
# Check if this templates dir has template subdirectories with pyproject.toml
# (i.e., it's the actual templates dir, not just a code directory)
has_template_projects = False
try:
for entry in templates_dir.iterdir():
if entry.is_dir() and (entry / "pyproject.toml").exists():
has_template_projects = True
break
except (OSError, PermissionError):
continue
if has_template_projects:
return parent
return start
def _detect_repo_root(explicit_root: Optional[Path]) -> Path:
# Prefer explicit root if provided
if explicit_root is not None:
root = _find_repo_root(explicit_root)
if (root / "templates").exists():
return root
# Next try current working directory
cwd_root = _find_repo_root(Path.cwd())
if (cwd_root / "templates").exists():
return cwd_root
# Fallback to module location
return _find_repo_root(Path(__file__).resolve())
def get_template(
relative_path: str,
start_line: Optional[int] = None,
end_line: Optional[int] = None,
max_lines: int = 1000,
) -> str:
"""Get a template file with line numbers.
Args:
relative_path: Path relative to templates directory
start_line: Optional starting line number (1-indexed, inclusive)
end_line: Optional ending line number (1-indexed, inclusive)
max_lines: Maximum number of lines to return (default 1000)
Returns:
File contents with line numbers prefixed to each line
"""
repo_root = _detect_repo_root(Path.cwd())
templates_dir = repo_root / "templates"
if not templates_dir.exists():
raise ValueError(f"Templates directory not found at {templates_dir}")
path = templates_dir / relative_path
if not path.exists():
raise ValueError(f"Template file not found at {path}")
content = path.read_text(encoding="utf-8", errors="ignore")
lines = content.splitlines()
total_lines = len(lines)
# Apply range filtering if specified
if start_line is not None or end_line is not None:
# Convert to 0-indexed
start_idx = (start_line - 1) if start_line is not None else 0
end_idx = end_line if end_line is not None else total_lines
# Clamp to valid range
start_idx = max(0, min(start_idx, total_lines))
end_idx = max(start_idx, min(end_idx, total_lines))
lines = lines[start_idx:end_idx]
line_offset = start_idx
else:
line_offset = 0
# Apply max_lines cap
lines_to_show = lines[:max_lines]
omitted_count = len(lines) - len(lines_to_show)
# Format with line numbers
result_lines = []
for i, line in enumerate(lines_to_show, start=line_offset + 1):
# Right-align line numbers to 6 characters for consistency
result_lines.append(f"{i:6}| {line}")
result = "\n".join(result_lines)
# Add omission notice if needed
if omitted_count > 0:
result += f"\n// {omitted_count} more line(s) omitted"
return result
def search_templates_impl(
query: str,
limit: int = 10,
root: Optional[Path] = None,
context_lines: int = 3,
) -> List[SearchResult]:
"""Search the repository templates directory for code matching the query.
Uses a lightweight BM25-like ranking across files. Returns top results with
a few matching line snippets.
"""
repo_root = _detect_repo_root(root)
templates_dir = repo_root / "templates"
if not templates_dir.exists():
return []
query_tokens = _tokenize(query)
if not query_tokens:
return []
# First pass to gather term frequencies and document stats
doc_tfs: Dict[Path, Dict[str, int]] = {}
doc_lens: Dict[Path, int] = {}
df: Dict[str, int] = {}
files: List[Path] = []
path_token_sets: Dict[Path, set[str]] = {}
for path in _iter_template_files(templates_dir):
try:
text = path.read_text(encoding="utf-8", errors="ignore")
except Exception:
continue
# Combine file content tokens with relative path tokens for better recall
rel_path = path.relative_to(repo_root).as_posix()
content_tokens = _tokenize(text)
path_tokens = _tokenize(rel_path)
tokens = content_tokens + path_tokens
if not tokens:
continue
files.append(path)
tf: Dict[str, int] = {}
for tok in tokens:
tf[tok] = tf.get(tok, 0) + 1
doc_tfs[path] = tf
doc_lens[path] = len(tokens)
path_token_sets[path] = set(path_tokens)
# Update document frequency only once per doc
seen = set(tf.keys())
for tok in seen:
df[tok] = df.get(tok, 0) + 1
if not files:
return []
N = len(files)
avgdl = sum(doc_lens.values()) / float(N)
# Score documents
scored: List[Tuple[Path, float]] = []
path_bonus = 2.0
for path in files:
score = _bm25_score(query_tokens, doc_tfs[path], avgdl, doc_lens[path], N, df)
# Add bonus for matches in file path tokens
if path in path_token_sets:
bonus_hits = sum(1 for q in query_tokens if q in path_token_sets[path])
if bonus_hits:
score += path_bonus * bonus_hits
if score > 0:
scored.append((path, score))
scored.sort(key=lambda x: x[1], reverse=True)
top = scored[: max(1, limit)]
results: List[SearchResult] = []
for path, score in top:
try:
lines = path.read_text(encoding="utf-8", errors="ignore").splitlines()
except Exception:
lines = []
matches = _find_snippets(lines, query_tokens, context_lines=context_lines)
results.append(
SearchResult(
path=str(path.relative_to(repo_root)),
score=round(score, 4),
matches=matches,
)
)
return results
# -----------------------------
# MCP tool wrappers
# -----------------------------
def _merge_line_entries(matches: List[Match]) -> Dict[int, Tuple[str, bool]]:
"""Merge overlapping contexts across matches.
Returns mapping of line_number -> (text, is_match).
"""
line_entries: Dict[int, Tuple[str, bool]] = {}
for m in matches:
# Before context
start_before = m.line - len(m.before)
for i, b in enumerate(m.before):
ln = start_before + i
if ln not in line_entries:
line_entries[ln] = (b, False)
# Matched line
existing = line_entries.get(m.line)
if existing is None or existing[1] is False:
line_entries[m.line] = (m.text, True)
# After context
for i, a in enumerate(m.after):
ln = m.line + 1 + i
if ln not in line_entries:
line_entries[ln] = (a, False)
return line_entries
def format_results_pretty(results: List[SearchResult]) -> str:
"""Render search results into a pretty textual format.
Uses '*' to mark matching lines and a blank space for context lines.
"""
parts: List[str] = []
for r in results:
parts.append(f"{r.path} (score {r.score})")
line_entries = _merge_line_entries(r.matches)
for ln in sorted(line_entries.keys()):
text, is_match = line_entries[ln]
indicator = "*" if is_match else " "
parts.append(f" {ln:>5}{indicator} {text}")
return "\n".join(parts)
src/tmpl/metrics/exporter.py
-326
View File
@@ -1,326 +0,0 @@
"""GitHub traffic metrics fetcher (clones) and PostHog exporter.
Collects repository traffic metrics (clones and unique cloners) for the last
14 days from the GitHub REST API and emits a single PostHog event per
repository. Designed to be used by the CLI command and from CI.
"""
from __future__ import annotations
import os
import re
from dataclasses import dataclass
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple, TypedDict
import requests
from posthog import Posthog
GITHUB_API_BASE = "https://api.github.com"
class CloneItem(TypedDict):
timestamp: str
count: int
uniques: int
class ClonesResponse(TypedDict):
count: int
uniques: int
clones: List[CloneItem]
@dataclass
class GitHubAuth:
token: Optional[str]
@classmethod
def from_env(cls) -> "GitHubAuth":
return cls(token=os.getenv("GITHUB_TOKEN") or os.getenv("GITHUB_PAT"))
@dataclass
class PostHogEvent:
"""Represents a PostHog event with all necessary data."""
event_name: str
properties: Dict[str, Any]
timestamp: Optional[datetime] = None
def to_dict(self) -> Dict[str, Any]:
"""Convert to dictionary format for backwards compatibility."""
result = {
"event_name": self.event_name,
"properties": self.properties,
}
if self.timestamp is not None:
result["timestamp"] = self.timestamp
return result
def owner_repo_from_url(url: str) -> Tuple[str, str]:
"""Parse an owner/repo from a GitHub URL or SSH git URL.
Examples:
- https://github.com/run-llama/template-workflow-basic.git -> (run-llama, template-workflow-basic)
- git@github.com:run-llama/template-workflow-basic.git -> (run-llama, template-workflow-basic)
"""
# SSH form
m = re.match(r"^git@github.com:(?P<owner>[^/]+)/(?P<repo>[^/]+?)(?:\.git)?$", url)
if m:
return m.group("owner"), m.group("repo")
# HTTPS form
m = re.match(
r"^https?://github.com/(?P<owner>[^/]+)/(?P<repo>[^/]+?)(?:\.git)?$", url
)
if m:
return m.group("owner"), m.group("repo")
raise ValueError(f"Unsupported GitHub URL format: {url}")
def _gh_get(
path: str, auth: GitHubAuth, params: Optional[Dict[str, Any]] = None
) -> requests.Response:
url = f"{GITHUB_API_BASE}{path}"
headers = {
"Accept": "application/vnd.github+json",
"User-Agent": "tmpl-metrics-exporter",
"X-GitHub-Api-Version": "2022-11-28",
}
if auth.token:
headers["Authorization"] = f"token {auth.token}"
resp = requests.get(url, headers=headers, params=params, timeout=30)
return resp
def _gh_get_json(
path: str, auth: GitHubAuth, params: Optional[Dict[str, Any]] = None
) -> Dict[str, Any]:
resp = _gh_get(path, auth, params)
if resp.status_code is None or resp.status_code >= 300:
raise RuntimeError(
f"GitHub API error {resp.status_code} for {path}: {resp.text[:200]}"
)
return resp.json()
def get_repo_clones(
owner: str, repo: str, auth: GitHubAuth, per: str = "day"
) -> ClonesResponse:
data = _gh_get_json(
f"/repos/{owner}/{repo}/traffic/clones", auth, params={"per": per}
)
# Schema is stable per GitHub docs
return ClonesResponse(
count=int(data["count"]),
uniques=int(data["uniques"]),
clones=[
CloneItem(
timestamp=str(item["timestamp"]),
count=int(item["count"]),
uniques=int(item["uniques"]),
)
for item in data["clones"]
],
)
def fetch_repo_metrics(
owner: str, repo: str, auth: Optional[GitHubAuth] = None
) -> Dict[str, Any]:
"""Fetch traffic metrics for a repository: clones and unique cloners.
Returns keys: owner, repo, clones_count, clones_uniques, clones (per day for last 14 days).
"""
auth = auth or GitHubAuth.from_env()
clones = get_repo_clones(owner, repo, auth, per="day")
return {
"owner": owner,
"repo": repo,
"clones_count": clones["count"],
"clones_uniques": clones["uniques"],
"clones": clones["clones"],
}
def send_posthog_event(
event: str,
properties: Dict[str, Any],
*,
timestamp: Optional[datetime] = None,
api_key: Optional[str] = None,
host: Optional[str] = None,
distinct_id: str = "tmpl-metrics",
) -> None:
"""Send a single event to PostHog, optionally with a specific timestamp.
Reads POSTHOG_API_KEY and POSTHOG_HOST if not provided.
"""
api_key = (
api_key or os.getenv("POSTHOG_API_KEY") or os.getenv("POSTHOG_PROJECT_API_KEY")
)
host = host or os.getenv("POSTHOG_HOST") or "https://us.posthog.com"
if api_key is None:
raise RuntimeError("POSTHOG_API_KEY is required to send metrics")
Posthog(project_api_key=api_key, host=host).capture(
distinct_id=distinct_id, event=event, properties=properties, timestamp=timestamp
)
def _parse_iso_utc(ts: str) -> datetime:
return datetime.fromisoformat(ts.replace("Z", "+00:00"))
def build_clone_events(
template: str, owner: str, repo: str, clones: ClonesResponse, *, backfill: bool
) -> List[PostHogEvent]:
"""Build PostHog events for total and daily clones metrics.
Returns a list of PostHogEvent objects.
- Total event includes: clones_total, clones_unique_total, days_count, window_start, window_end.
- Daily events include: day_timestamp, clones_day, uniques_day. When backfill is False, only last day is included.
The timestamp will be set to the day's timestamp for daily events and to window_end for the total event.
"""
items = clones["clones"]
days_count = len(items)
window_start = items[0]["timestamp"] if items else None
window_end = items[-1]["timestamp"] if items else None
events: List[PostHogEvent] = []
# Total window event
total_props: Dict[str, Any] = {
"template": template,
"owner": owner,
"repo": repo,
"clones_total": clones["count"],
"clones_unique_total": clones["uniques"],
"days_count": days_count,
"window_start": window_start,
"window_end": window_end,
"dedupe_ts": window_end,
}
events.append(
PostHogEvent(
event_name="template_repo_clones_total",
properties=total_props,
timestamp=_parse_iso_utc(window_end) if window_end else None,
)
)
# Daily events
daily_items: List[CloneItem]
if backfill:
daily_items = items
else:
daily_items = items[-1:] if items else []
for it in daily_items:
ts = it["timestamp"]
daily_props: Dict[str, Any] = {
"template": template,
"owner": owner,
"repo": repo,
"day": ts,
"clones_day": it["count"],
"clones_uniques_day": it["uniques"],
"dedupe_ts": ts,
}
events.append(
PostHogEvent(
event_name="template_repo_clones_daily",
properties=daily_props,
timestamp=_parse_iso_utc(ts),
)
)
return events
def export_all_from_mapping(
mapping: Dict[str, Dict[str, Any]], *, github_auth: Optional[GitHubAuth] = None
) -> Dict[str, Dict[str, Any]]:
"""Fetch metrics for all templates defined in the mapping file.
Returns a dict keyed by template name with metrics dict values.
"""
github_auth = github_auth or GitHubAuth.from_env()
results: Dict[str, Dict[str, Any]] = {}
for template_name, cfg in mapping.items():
try:
owner, repo = owner_repo_from_url(
cfg["url"]
) # cfg keys validated by loader
metrics = fetch_repo_metrics(owner, repo, github_auth)
results[template_name] = metrics
except Exception as exc: # capture error info per template
results[template_name] = {
"error": str(exc),
}
return results
def get_all_events_for_export(
mapping: Dict[str, Dict[str, Any]],
*,
github_auth: Optional[GitHubAuth] = None,
backfill: bool = False,
) -> Tuple[Dict[str, Dict[str, Any]], List[PostHogEvent]]:
"""Get metrics and PostHog events for all templates.
Returns a tuple of (metrics_dict, events_list) where:
- metrics_dict: keyed by template name with summary metrics
- events_list: list of PostHogEvent objects ready for sending
Each PostHogEvent contains: event_name, properties, timestamp (optional)
"""
github_auth = github_auth or GitHubAuth.from_env()
metrics: Dict[str, Dict[str, Any]] = {}
all_events: List[PostHogEvent] = []
for template_name, cfg in mapping.items():
try:
owner, repo = owner_repo_from_url(cfg["url"])
clones = get_repo_clones(owner, repo, github_auth, per="day")
# Build metrics summary
metrics[template_name] = {
"owner": owner,
"repo": repo,
"clones_count": clones["count"],
"clones_uniques": clones["uniques"],
"days_count": len(clones["clones"]),
"window_start": clones["clones"][0]["timestamp"]
if clones["clones"]
else None,
"window_end": clones["clones"][-1]["timestamp"]
if clones["clones"]
else None,
}
# Build events
events = build_clone_events(
template_name, owner, repo, clones, backfill=backfill
)
all_events.extend(events)
except Exception as exc:
metrics[template_name] = {"error": str(exc)}
return metrics, all_events
__all__ = [
"GitHubAuth",
"PostHogEvent",
"fetch_repo_metrics",
"send_posthog_event",
"export_all_from_mapping",
"get_all_events_for_export",
"owner_repo_from_url",
"get_repo_clones",
"build_clone_events",
]
src/tmpl/templates/__init__.py
-7
View File
@@ -1,7 +0,0 @@
"""Template management for tmpl."""
from .manager import get_template_dir
__all__ = [
"get_template_dir",
]
src/tmpl/templates/manager.py
-12
View File
@@ -1,12 +0,0 @@
"""High-level template management operations."""
from __future__ import annotations
from pathlib import Path
def get_template_dir(template_name: str) -> Path:
"""Get the directory path for a template."""
root = Path(__file__).parent.parent.parent.parent
return root / "templates" / template_name
src/tmpl/utils/__init__.py
-6
View File
@@ -1,6 +0,0 @@
"""Utility modules for tmpl."""
from .console import console
from .subprocess_utils import git_output, run, run_git_command
__all__ = ["console", "git_output", "run", "run_git_command"]
src/tmpl/utils/console.py
-5
View File
@@ -1,5 +0,0 @@
"""Rich console utilities."""
from rich.console import Console
console = Console()
src/tmpl/utils/filesystem.py
-39
View File
@@ -1,39 +0,0 @@
"""File system utilities."""
from __future__ import annotations
import subprocess
from pathlib import Path
from typing import Set
def get_git_tracked_files(directory: Path, respect_gitignore: bool = True) -> Set[Path]:
"""Get set of files that would be tracked by git (optionally respecting gitignore)."""
ignored_files = {".copier-answers.yml"}
if not respect_gitignore:
tracked_files: Set[Path] = set()
for file_path in directory.rglob("*"):
if file_path.is_file():
relative_path = file_path.relative_to(directory)
if relative_path.name not in ignored_files:
tracked_files.add(relative_path)
return tracked_files
result = subprocess.run(
["git", "ls-files", "--others", "--cached", "--exclude-standard"],
cwd=directory,
capture_output=True,
text=True,
check=True,
)
tracked_files: Set[Path] = set()
for line in result.stdout.strip().split("\n"):
if line.strip():
file_path = directory / line.strip()
relative_path = Path(line.strip())
if file_path.is_file() and relative_path.name not in ignored_files:
tracked_files.add(relative_path)
return tracked_files
src/tmpl/utils/subprocess_utils.py
-49
View File
@@ -1,49 +0,0 @@
"""Subprocess utilities for running commands."""
from __future__ import annotations
import subprocess
import sys
from pathlib import Path
from typing import List, Optional
from .console import console
def run(command: List[str], cwd: Optional[Path] = None) -> None:
"""Run a command and stream output to stdout."""
process = subprocess.Popen(
command,
cwd=str(cwd) if cwd else None,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
)
assert process.stdout is not None
for line in process.stdout:
sys.stdout.write(line)
code = process.wait()
if code:
raise SystemExit(code)
def git_output(args: List[str]) -> str:
"""Run a git command and return its output."""
return subprocess.check_output(["git", *args], text=True).strip()
def run_git_command(
cmd: List[str], cwd: Optional[Path] = None
) -> subprocess.CompletedProcess[str]:
"""Run a git command and return the result."""
console.print(f"Running: {' '.join(cmd)}")
try:
result = subprocess.run(
cmd, cwd=cwd, capture_output=True, text=True, check=True
)
return result
except subprocess.CalledProcessError as e:
console.print(f"Command failed with exit code {e.returncode}", style="bold red")
console.print(f"stdout: {e.stdout}", style="bold yellow")
console.print(f"stderr: {e.stderr}", style="bold yellow")
raise SystemExit(1)
template-gen-specs/classify-extract-sec/config.json
-3
View File
@@ -1,3 +0,0 @@
{
"base_template": "extraction-review"
}
template-gen-specs/classify-extract-sec/expectations.md
-7
View File
@@ -1,7 +0,0 @@
- A classify client should be added to the project
- The frontend should have minimal changes that load all of the 4 schemas rather than just one schema
- The item details page should select the schema based on the classification
- There are no lint ignore errors
- There are no getattr/setattr/hasattr calls in the code
- There should be a new step dedicated to classify
- The generated schema is appropriate given the use case of "financial analysis & investment research"
template-gen-specs/classify-extract-sec/task.md
-17
View File
@@ -1,17 +0,0 @@
We are creating a database of SEC filings, along with extracted structured information from the filings.
The goal of the database is for financial analysis & investment research.
Using LlamaClassify and LlamaExtract, create a workflow that will extract meaningful details about SEC filing documents.
Consider how the data may be queried and structured in a database.
Specifically, classify the document types into the following categories:
- 10-K
- 10-Q
- 8-K
- other
Each document type should have a distinct schema. Define the schema as a pydantic model using
structured fields. Make data optional if it may not be present or difficult to extract.
The different types should be displayed together in the UI as a table of documents, and clicking into the item should show the extracted data, according to the documents extracted schema.
templates/.gitignore
-10
View File
@@ -1,10 +0,0 @@
*/uv.lock
**/pnpm-lock.yaml
# bootstrap agents docs separately
**/AGENTS.md
**/CLAUDE.md
**/.cursor
**/.mcp.json
**/.claude
**/.python-version
templates/basic-ui/.github/workflows/ci-check.yml
Vendored
-50
View File
@@ -1,50 +0,0 @@
name: CI Lint/Format/Test Check
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
jobs:
check-python:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.13'
- name: Install uv
uses: astral-sh/setup-uv@v3
- name: Run Python checks
run: uv run hatch run all-check
check-ui:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up Node.js
uses: actions/setup-node@v4
with:
node-version: '24'
- name: Enable Corepack
run: corepack enable
- name: Activate pnpm version
working-directory: ui
run: corepack prepare --activate
- name: Run UI checks
run: pnpm run all-check
working-directory: ui
templates/basic-ui/.gitignore
-7
View File
@@ -1,7 +0,0 @@
.venv
__pycache__
workflows.db
.env
package-lock.json
node_modules
templates/basic-ui/.python-version
-1
View File
@@ -1 +0,0 @@
3.13
templates/basic-ui/README.md
-15
View File
@@ -1,15 +0,0 @@
# Llama Deploy
This application uses LlamaDeploy. For more information see [the docs](https://developers.llamaindex.ai/python/llamaagents/llamactl/getting-started/)
# Getting Started
1. install `uv` if you haven't `brew install uv`
2. run `uvx llamactl serve`
3. Visit http://localhost:4501/
# Organization
- `src` contains python workflow sources. The name of the deployment here is defined as `app`
- `ui` contains a vite app powered by `@llamaindex/ui`. It calls the local workflow server via api requests. See http://localhost:4501/deployments/app/docs for openAPI docs
templates/basic-ui/copier.yaml
-4
View File
@@ -1,4 +0,0 @@
_exclude:
- ".git"
- ".github"
- "copier.yaml"
templates/basic-ui/pyproject.toml
-43
View File
@@ -1,43 +0,0 @@
[project]
name = "app"
version = "0.1.0"
description = "Add your description here"
readme = "README.md"
authors = []
requires-python = ">=3.12"
dependencies = [
"llama-index-workflows[server]>=2.2.0",
"ruff>=0.13.0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[dependency-groups]
dev = [
"ruff>=0.13.0",
"ty>=0.0.1a16",
"pytest>=8.4.1",
"hatch>=1.14.1",
"click>=8.0.0,!=8.3.0",
]
[tool.hatch.envs.default.scripts]
"format" = "ruff format ."
"format-check" = "ruff format --check ."
"lint" = "ruff check --fix ."
"lint-check" = ["ruff check ."]
typecheck = "ty check src"
test = "pytest"
"all-check" = ["format-check", "lint-check", "test"]
"all-fix" = ["format", "lint", "test"]
[tool.llamadeploy.ui]
directory = "./ui"
[tool.llamadeploy]
env_files = [".env"]
[tool.llamadeploy.workflows]
default = "app.workflow:workflow"
templates/basic-ui/src/app/__init__.py
View File
templates/basic-ui/src/app/workflow.py
-19
View File
@@ -1,19 +0,0 @@
from workflows import Context, Workflow, step
from workflows.events import StartEvent, StopEvent
class Start(StartEvent):
pass
class BasicWorkflow(Workflow):
@step
async def hello(self, event: Start, context: Context) -> StopEvent:
return StopEvent(
result=(
"Hello from the basic-ui backend. Edit src/app/workflow.py to get started."
)
)
workflow = BasicWorkflow(timeout=None)
templates/basic-ui/tests/test_placeholder.py
-2
View File
@@ -1,2 +0,0 @@
def test_placeholder():
pass
templates/basic-ui/ui/.gitignore
-5
View File
@@ -1,5 +0,0 @@
node_modules
dist
# uses pnpm
package-lock.json
yarn.lock
templates/basic-ui/ui/index.html
-14
View File
@@ -1,14 +0,0 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Quick Start UI</title>
</head>
<body>
<div id="root"></div>
<script type="module" src="/src/main.tsx"></script>
</body>
</html>

Some files were not shown because too many files have changed in this diff Show More