10 Commits
Author SHA1 Message Date
BillyOutlast d5fc14bbb4 ci(ocr): point review job at org reusable workflow (pinned)
Fix 'failed to fetch workflow' parse error: ocr.yml referenced a local reusable workflow path (./.github/workflows/ocr-review.yml) that does not exist in this repo. Point at Heretek-AI/.github org reusable workflow pinned to main HEAD (a440b485).
2026-10-19 16:34:51 -04:00
BillyOutlast 7950eca1e7 ci: add reusable OpenCodeReview PR workflow 2026-10-19 15:49:31 -04:00
John Smith 3ef64642a7 Merge release-v1.0.0: SwarmOS v1.0.0 Unified Heretek Platform 2026-10-15 13:15:47 -04:00
John Smith b2020bc206 feat: release SwarmOS v1.0.0 Unified Heretek Platform 2026-10-15 13:15:44 -04:00
BillyOutlast 265361fc4c Merge pull request #1 from Heretek-AI/ccr-b267996d-jqeyf5
docs: document v0.5.0 reliability engines in README
2026-10-15 10:17:07 -04:00
John Smith e6d52d00ad feat: complete v0.9.0 roadmap (task budgets, session replay, warm parallel LSP & pinned V2 host contract) 2026-10-15 10:16:54 -04:00
Claude 39b00bb57b feat: SwarmOS v0.8.0 - context budget, self-maintaining memory, multi-language RepoMap & tool profiles
- Replace the artifact and memory context hooks with one budgeted injector that
  ranks blocks, caps any single artifact at 40% of the budget, and reports usage
  in /doctor.
- Score lessons by scope hits on touched files, keyword overlap, track record
  and age; never inject lessons flagged needsReview.
- Draft lesson proposals when a circuit-breaker loop recovers; track lesson
  successes/failures from verify results and breaker fires; add proposals,
  accept/reject and forget tools.
- RepoMap parses TS/JS, Python, Go and Rust after stripping comments/strings,
  records per-edge symbols for symbol-level blast radius through re-exports,
  runs PageRank to convergence, and caches parsed files.
- Pin only configured Code Mode namespaces and add a per-role toolbox hint.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_0171AqkGJuWMw9gY25tb1Q7T
2026-10-15 02:31:14 +00:00
Claude c6f1622fa9 feat: SwarmOS v0.7.0 - shell parser guard, real worktree tournaments, edit transactions & taint tracking
- Replace the regex shell blocklist with a parser that follows pipes, $(...),
  backticks, heredocs, bash -c, eval and wrappers, classifies each command, and
  routes every shell write target through the contract/ownership/tournament guard.
  Per-role policy can deny, require approval, or auto-approve read-only commands.
- plan_tournament creates a git worktree per candidate; contested files are
  locked in the main tree; score_tournament ranks candidates with tools.verify;
  resolve_tournament merges the winner (--no-ff) and removes all worktrees.
- Snapshot files before every edit/write/patch and restore all of them when the
  call fails; multi-file calls also leave an auto checkpoint.
- Sessions that read web, browser or third-party MCP content need approval
  again for network commands and edits to harness, git, CI or credential files
  until /untaint.
- /doctor reports how often the host invoked each hook.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_0171AqkGJuWMw9gY25tb1Q7T
2026-10-15 02:24:52 +00:00
Claude 795839066d feat: SwarmOS v0.6.0 - host contract check, /doctor, tools.verify, telemetry & benchmark harness
- Probe every OpenCode V2 capability SwarmOS uses at startup and run each setup
  module in isolation; /doctor and tools.swarmos.doctor report registered,
  skipped (with the missing capability) and failed modules plus caught hook errors.
- scripts/check-host-contract.ts scans an OpenCode V2 checkout for the hook
  names and APIs SwarmOS relies on (also runnable via OPENCODE_V2_SRC in tests).
- tools.verify.run discovers the repo's own checks, scopes tests to files that
  import the changed code, and returns structured counts and file:line failures.
  QA PASS receipts and the release gate now require a passing run on the
  current working tree (opt out with requireVerifiedReceipts: false).
- Per-session telemetry and an observe-only mode used as the benchmark baseline.
- bench/: 12 tasks with hidden tests and reference solutions, reference/noop/
  agent drivers, and a SwarmOS-vs-baseline report.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_0171AqkGJuWMw9gY25tb1Q7T
2026-10-15 02:16:10 +00:00
Claude 4b40b181f1 docs: document v0.5.0 reliability engines in README
Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_0171AqkGJuWMw9gY25tb1Q7T
2026-10-15 01:58:37 +00:00
141 changed files with 17347 additions and 969 deletions

No files matched your search

+41
View File
@@ -0,0 +1,41 @@
name: OpenCodeReview
concurrency:
group: >-
${{
(
github.event_name == 'pull_request_target'
|| (
github.event_name == 'issue_comment'
&& github.event.issue.pull_request
&& github.event.comment.user.type != 'Bot'
&& (
github.event.comment.author_association == 'MEMBER'
|| github.event.comment.author_association == 'OWNER'
|| github.event.comment.author_association == 'COLLABORATOR'
)
&& (
startsWith(github.event.comment.body, '/open-code-review')
|| startsWith(github.event.comment.body, '@open-code-review')
)
)
)
&& format('ocr-{0}', github.event.pull_request.number || github.event.issue.number)
|| format('noop-{0}', github.run_id)
}}
cancel-in-progress: true
on:
pull_request_target:
types: [opened, synchronize, reopened]
issue_comment:
types: [created]
permissions:
contents: read
pull-requests: write
jobs:
review:
uses: Heretek-AI/.github/.github/workflows/ocr-review.yml@a440b485d779b9177b1602c4d4f7f58cff0483a2 # main
secrets: inherit
+3
View File
@@ -40,3 +40,6 @@ Thumbs.db
# Coverage / Test
coverage/
.turbo/
# Benchmark output
bench/results/
+82
View File
@@ -181,6 +181,85 @@ SwarmOS v0.4.0 expands OpenCode V2 beyond software engineering into a **Universa
- **Oh-My-Pi Precision Hashline Editing (`tools.fs.hash_read` & `tools.fs.hash_edit`)**: Tags every line with a 2-hex FNV-1a content hash (`42#a7: const x = 1;`) and applies multi-hunk line-range replacements with automatic $\pm 3$-line drift recovery.
- **Antigravity Persistent Artifacts (`/artifacts` & `tools.artifacts.*`)**: Automatically syncs `.opencode/artifacts/task.md` and `plan.md` into the live session context window across turns.
### Agent Reliability Engines (v0.5.0+): `/map`, Circuit Breaker, Checkpoints, Memory & PTY Tap
SwarmOS v0.5.0 adds five engines that keep long-running agents oriented, recoverable, and out of retry loops:
- **PageRank RepoMap & Blast-Radius Analyzer (`/map`, `tools.repo.map`, `tools.repo.blast_radius`)**: Parses `import`/`export` edges and top-level symbols across the repository, ranks files by PageRank centrality (`tools.repo.map({ topN })`), and lists the direct and transitive reverse-import dependents of a file (`tools.repo.blast_radius({ path })`) before you change its interface.
- **Doom-Loop Circuit Breaker (automatic)**: Tracks tool attempts per session. After **2 consecutive `edit`/`patch` failures** on the same file it injects a `<circuit-breaker-intervention>` steering the agent to `tools.fs.hash_read` / `tools.fs.hash_edit`; after **3 consecutive identical `shell` failures** it halts blind retries and requires root-cause notes in `notes.md`.
- **Zero-Git-Noise Checkpoints (`tools.checkpoints.create` / `rollback` / `list`)**: Snapshots the given `paths` into `.opencode/checkpoints/<id>.json` before a risky refactor, then restores them (and removes files that did not exist yet) in one call, without commits or touching `HEAD`.
- **Cross-Session Episodic Memory (`tools.memory.record_lesson` / `recall`)**: Stores repository lessons (`id`, `trigger`, `lesson`, optional `scopePaths`) in `.opencode/memory/lessons.json` and injects them as `<episodic-repository-memory>` into every non-compaction turn.
- **Live PTY & Dev-Server Log Tap (`tools.pty.read_logs`)**: Reads a persistent terminal (`ptyID`, via `ctx.experimental.terminal.read`) or a log file (`logFile`, default `.opencode/artifacts/devserver.log`), with `lines` (10–500, default 80) and `filterErrorsOnly` so agents can check HMR/SSR/runtime errors right after an edit.
### Measurement & Verification (v0.6.0+): `/doctor`, `tools.verify.*`, telemetry & benchmarks
SwarmOS v0.6.0 makes it possible to check that SwarmOS is actually running and actually helping:
- **Host contract check & `/doctor`**: At startup SwarmOS probes every OpenCode V2 capability it uses (`src/host/contract.ts`) and runs each module separately. `/doctor` (or `tools.swarmos.doctor`) lists which capabilities exist, which modules registered, which were skipped and why (for example `permission-guard: skipped (host lacks permission.hook)`), and any exceptions thrown inside SwarmOS hooks, which are caught instead of breaking the host's tool call. `bun run contract:check <opencode-checkout>` scans an OpenCode V2 source tree for the hook names and APIs SwarmOS relies on; `OPENCODE_V2_SRC=<checkout> bun test` runs the same check in the test suite.
- **`tools.verify.run` / `tools.verify.checks`**: Finds the repository's own typecheck, lint and test commands (from a `verifier` field in `.opencode/swarmos.json` or `opencode.json`, `AGENTS.md` lines like ``- test: `bun test` ``, `package.json` scripts, or Cargo/Go/pytest defaults). With `scope: "changed"` (default) it runs typecheck and lint plus only the tests that import changed files (via the RepoMap), strips colour codes, and returns pass/fail counts and `file:line` failures. Runs are saved in `.opencode/verify/` along with a fingerprint of the working tree.
- **Verified QA receipts**: For projects created with `tools.org.init_project`, a `PASS` receipt must cite the `verifyRunId` of a passing run taken on the task's current code, and `verify_release_ready` requires a passing `scope: "full"` run on the current tree. Editing anything afterwards makes both stale. Pass `requireVerifiedReceipts: false` to `init_project` to opt out.
- **Telemetry & observe mode**: Each session's tool calls, failures, circuit-breaker fires, turns and (when the host reports it) token usage are written to `.opencode/telemetry/<sessionID>.jsonl`. `{"mode": "observe"}` in `.opencode/swarmos.json` (or `SWARMOS_MODE=observe`) loads only telemetry and `/doctor`.
- **Benchmark harness (`bench/`)**: 12 tasks (bugs, multi-file refactors, constraint-heavy features, UI fixes, one Python task), each with hidden tests and a reference solution. `bun run bench:check` proves every task fails untouched and passes when solved; `bun bench/run.ts --driver agent` runs a real agent on each task with SwarmOS on and in observe mode, and reports success rate, turns, tool failure rate, breaker fires, tokens and cache hits. See `bench/README.md`.
### Safety (v0.7.0+): shell parsing, real worktree tournaments, edit transactions & taint tracking
- **Shell commands are parsed, not pattern-matched**: `src/agents/shell-parse.ts` splits commands on `;`, `&&`, `||`, pipes and newlines, follows `$(...)`, backticks, process substitution, heredocs, `bash -c`/`sh -c`, `eval`, and wrappers like `sudo`, `env`, `timeout` and `xargs`, records redirections, and classifies every command as read, verify, write, network, destructive or exec. `bash -c "echo x > src/a.ts"`, `tee`, `cp` and `$(rm …)` no longer get past the guard. Every file a shell command writes goes through the same contract, ownership and tournament checks as `edit`.
- **Per-role policy**: destructive commands (`git reset --hard`, `git push -f`, `git checkout -- …`, `rm -rf /`) are denied for everyone, but `--force-with-lease` is fine. `director` and `manager` cannot write, and need approval to run code or reach the network. `qa-adversary` may write only test files, and inline code (`python -c`, `node -e`) needs approval. Read-only commands from `director`, `manager`, `explore`, `plan` and `qa-adversary` run without a prompt. `explore` and `plan` need approval for anything else. Commands SwarmOS cannot parse always need approval.
- **Real worktree tournaments**: `tools.org.plan_tournament` creates a git worktree per candidate at `.worktrees/<taskId>` on branch `swarmos/<taskId>`. While a tournament is active, its files are locked in the main tree and each candidate may edit only its own worktree. `tools.verify.run({ dir })` verifies a candidate, `tools.org.score_tournament` ranks all candidates (passing first, then fewest failures, then smallest diff), and `tools.org.resolve_tournament` merges the winner with `--no-ff`, removes every candidate worktree and branch, and aborts cleanly on conflicts.
- **All-or-nothing edits**: before every `edit`, `write` or `patch`, SwarmOS snapshots the files the call will touch (patch envelopes and unified diffs are parsed for targets). If the call fails, every file is restored, files the call created are removed, and the model is told what was rolled back. Multi-file calls also leave an `auto-*` checkpoint, and only the last 20 are kept.
- **Untrusted-content tracking**: after a session reads `webfetch`, `websearch`, browser, or third-party MCP output (including Code Mode scripts that call them), saved approvals no longer cover risky actions. Network commands, writes outside the project, edits to `.opencode/`, `.git/`, CI workflows, `package.json`, `.env` or shell profiles, and inline or downloaded code need approval again until the user runs `/untaint`.
- `/doctor` now also shows how many times the host invoked each hook, so a hook stuck at 0 after real use (for example `tool.execute.before`) shows the host doesn't fire it.
### Context Efficiency (v0.8.0+): budgeted context, self-maintaining memory, multi-language RepoMap & tool profiles
- **One context budget**: a single injector replaces the separate artifact and memory hooks. Each turn it collects candidate blocks (an untrusted-content notice when relevant, a short per-role toolbox hint, `task.md`, `plan.md`, and lessons), ranks them, and packs them into `contextBudgetTokens` (default 6000). One artifact may take at most 40% of the budget and is truncated beyond that, so it can't crowd out lessons. `/doctor` shows how much of the budget each session used and what was dropped.
- **Relevant lessons, not all lessons**: lessons are scored by `scopePaths` matches against the files this session has read or edited, by keyword overlap with those paths and the agent's role, by their track record, and by age (half-life of 90 days since last use). Lessons flagged `needsReview` are never injected.
- **Memory that maintains itself**: when the circuit breaker fires and a later call on the same file or command succeeds, SwarmOS saves a lesson proposal with the evidence (`tools.memory.proposals`). Proposals are never injected until a human or agent accepts them, ideally with the real root cause written in (`tools.memory.accept_proposal` / `reject_proposal`). Injected lessons gain a success when `tools.verify.run` passes in that session and a failure when it fails or the breaker fires on a file in their scope. Two or more failures that outnumber the successes set `needsReview`; `tools.memory.forget` deletes a lesson.
- **Multi-language RepoMap**: the repo map now parses TypeScript/JavaScript, Python, Go (module path from `go.mod`, used `pkg.Symbol` selectors) and Rust (`mod`, `use crate::`/`super::`/`self::`) after stripping comments and strings, so commented-out imports no longer count. Parsing is lexer-based, not tree-sitter, so the drop-in bundle stays a single file with no WASM grammars. Edges record which symbols each importer takes, so `tools.repo.blast_radius({ path, symbol })` lists only the files that use that symbol, following re-exports and renames. PageRank now iterates to convergence, and parsed files are cached by mtime.
- **Smaller always-on tool list**: only `pinnedNamespaces` (default `org`, `fs`, `repo`, `verify`, `artifacts`) stay pinned in Code Mode; other namespaces stay registered and discoverable via search, and each role gets a one-line hint naming the namespaces it is most likely to need.
### Unified Heretek Platform (v1.0.0): 7 Repositories Consolidated into 1 Omni-Harness
`SwarmOS v1.0.0` consolidates the entire Heretek-AI OpenCode ecosystem (`SwarmOS`, `The-Agency`, `claude-4-opencode`, `IUMBTEMS`, `opencode-rea`, `godot-opencode`, and `GDevelop-Unlocked`) into a single zero-fork architecture:
1. **Pillar 1 — IUMBTEMS Epistemic & Dialectic Engine (`tools.epistemic.*`, `/grill`)**:
- SHA-256 content-addressed source cache (`.opencode/epistemic/sources/<shortHash>.md`) with SPDX license contamination classification (`permissive`, `weak-copyleft`, `copyleft-contaminating`, `unknown`).
- Deterministic quote verifier (`tools.epistemic.verify_claims`) that parses `[VERIFIED: <shortHash> | "exact verbatim quote"]` tags and confirms every quote exists verbatim in the cached source.
- Socratic stress-tester (`tools.epistemic.grill_spec` & `/grill`) that challenges architecture specs before locking contracts.
2. **Pillar 2 — `claude-4-opencode` Zero-Config Compatibility Bridge (`tools.claude.*`)**:
- Automatically discovers `.claude/commands/*.md`, `.claude/agents/*.md`, and `.claude/skills/*/SKILL.md` in the workspace or `~/.claude` and registers them as native OpenCode V2 slash commands, subagents, and skills.
3. **Pillar 3 — Quantitative 100-Point Visual Quality & UI/UX Judge (`tools.ui.score_ux`, `tools.ui.get_ux_scorecard_script`)**:
- Deterministic 6-axis UI/UX scorecard across **Spatial & Viewport Integrity (25 pts)**, **Alignment & Grid Harmony (20 pts)**, **Typography & WCAG Contrast (20 pts)**, **Interactive & Accessibility Completeness (15 pts)**, **Design Token Discipline (10 pts)**, and **Runtime Health (10 pts)**.
- Produces a `0..100` score, letter grade (`S`/`A`/`B`/`C`/`F`), and exact remediation checklist saved to `.opencode/artifacts/ux-scorecard.json` (passing requires `score >= 85` and `0` horizontal overflow).
4. **Pillar 4 — On-Demand Domain Packs (`tools.packs.*`, `/pack`)**:
- Zero-overhead domain packs that stay dormant on standard software repos and auto-activate when domain markers are detected:
- **Godot 4.x Pack (`godot-opencode`)**: `tools.packs.godot_inspect_scene` parses `.tscn`/`.tres` scene trees, node hierarchies, attached `.gd` scripts, and signal connections; registers `specialist-godot`.
- **GDevelop 5 Pack (`GDevelop-Unlocked`)**: `tools.packs.gdevelop_inspect_game` and `tools.packs.gdevelop_unlock_ai_config` inspect `game.json` layouts/events and generate local custom AI endpoint configuration; registers `specialist-gdevelop`.
- **Reverse Engineering Pack (`opencode-rea`)**: `tools.packs.re_inspect_binary` performs safe static triage on ELF/PE/Mach-O/WASM binaries (magic header, SHA-256, byte entropy, suspicious URLs/paths/crypto constants); registers `specialist-re`.
5. **Pillar 5 — `The-Agency` Live Swarm Command Center WebUI (`tools.org.render_dashboard`, `/dashboard`)**:
- Generates a self-contained High-Craft Technical Dark HTML5 dashboard (`.opencode/artifacts/swarm-dashboard.html`) displaying the 5-Gate Kanban Board, Locked Contracts, Worktree Tournaments, 100-Point UI/UX Scorecards, Epistemic Cache, and Active Domain Packs.
### Complete Roadmap Edition (v0.9.0+): Task Spend Budgets, Session Replay -> Benchmarks, Warm Parallel LSP & Pinned Host Fixture
- **Per-Task Spend & Step Budgets (`tools.org.claim_task`, `tools.org.set_task_budget`)**: Tasks on `.opencode/org/board.json` now accept `budget: { maxTokens?, maxTurns?, maxToolCalls? }`. When a subagent claims a task via `tools.org.claim_task`, SwarmOS accumulates token, turn, and tool-call usage on `task.usage`. If a subagent exceeds any budget ceiling, `createTaskBudgetEvaluator` locks further file mutations, shell commands, and subagent spawns, appends a `<task-budget-exceeded>` notice, and returns control to `manager` to either raise the budget (`tools.org.set_task_budget`) or re-scope the task.
- **Session Replay -> Benchmark Regression Exporter (`/replay`, `tools.replay.*`)**: `tools.replay.list_sessions` scans `.opencode/telemetry/*.jsonl` and ranks sessions with tool failures or circuit-breaker fires first. `tools.replay.export_benchmark({ sessionID })` (or `/replay <sessionID>`) packages a recorded failure session into a runnable benchmark regression task under `bench/tasks/<taskId>/` (`task.json`, `repo/`, `replay-trace.json`, `solution/`), and `tools.replay.replay_session` replays the recorded tool timeline step-by-step.
- **Parallel & Warm-Cached Diagnostics + Symbol Intelligence (`tools.lsp.*`)**: `runParallelDiagnostics` caches file diagnostics by `(absPath, mtimeMs, size)` so unchanged files in multi-file edits return in `0ms`, runs multi-file checks in bounded parallel batches, and exposes `tools.lsp.diagnostics`, `tools.lsp.definitions`, `tools.lsp.references`, and `tools.lsp.status` inside Code Mode.
- **Pinned OpenCode V2 (`v2.0.26`) Host Contract Fixture (`test/fixtures/opencode-v2-ref/`)**: Includes a pinned reference snapshot of OpenCode V2's plugin host contract so `bun run contract:check` and `bun test` verify every required capability and hook literal out-of-the-box without requiring an external monorepo checkout.
#### Configuration (`.opencode/swarmos.json`)
```jsonc
{
"mode": "full", // or "observe"; env: SWARMOS_MODE
"telemetry": true, // env: SWARMOS_TELEMETRY
"autoTransactions": true, // env: SWARMOS_AUTO_TRANSACTIONS
"taintTracking": true, // env: SWARMOS_TAINT_TRACKING
"contextBudgetTokens": 6000, // env: SWARMOS_CONTEXT_BUDGET
"pinnedNamespaces": ["org", "fs", "repo", "verify", "artifacts"], // env: SWARMOS_PINNED=org,fs
"verifier": { "typecheck": "bun run typecheck", "test": "bun test" }
}
```
---
## 8. Development & Testing
@@ -192,6 +271,9 @@ bun install
# Run automated test suite
bun test
# Check every benchmark task is solvable and not already passing
bun run bench:check
# Typecheck source
bun run typecheck
+37
View File
@@ -0,0 +1,37 @@
# SwarmOS Benchmark
Measures whether SwarmOS actually helps by running the same tasks with SwarmOS fully on
(`swarmos`) and with SwarmOS loaded in observe-only mode (`baseline`: telemetry and `/doctor`, nothing else).
## Tasks
Each task in `tasks/<id>/` has:
| Path | Purpose |
| :--- | :--- |
| `task.json` | `id`, `category` (`bug`, `refactor`, `constraints`, `ui`), the user-style `prompt`, the `verify` command and a timeout |
| `repo/` | Starting repository the agent works in |
| `hidden/` | Tests copied to `__hidden__/` only after the agent finishes |
| `solution/` | Reference fix, used to prove the task is solvable |
## Running
```bash
bun run bench:check # reference driver must pass 12/12, noop must pass 0/12
bun bench/run.ts --driver agent --repeat 3 # real comparison, both variants
bun bench/run.ts --driver agent --tasks bug-paginate,ui-spacing-tokens --variants swarmos
```
The `agent` driver runs `BENCH_AGENT_CMD` (default `opencode run "$BENCH_PROMPT"`) inside a fresh git
workspace with an `opencode.json` that loads `swarmos-plugin.ts` (override with `--plugin <path>`).
Run `bun run build` first so the bundle is current.
Reports land in `bench/results/` as JSON plus a Markdown table: success rate (overall and per category),
mean turns, mean tool calls, tool failure rate, circuit-breaker fires, tokens and cache-hit ratio,
and wall time. Turn, tool and breaker numbers come from SwarmOS telemetry; token numbers appear only
when the host reports usage on `session.generate` events.
## Adding a task
Create the four parts above, then run `bun run bench:check`. `test/bench.test.ts` runs the same
check in CI, so a task whose hidden tests pass untouched or fail on the reference solution breaks the build.
+332
View File
@@ -0,0 +1,332 @@
import { exec, execFileSync } from "node:child_process"
import * as fs from "node:fs/promises"
import * as os from "node:os"
import * as path from "node:path"
import {
readTelemetryFile,
summarizeTelemetry,
type TelemetryRecord,
} from "../src/host/telemetry.js"
export const BENCH_DIR = path.dirname(new URL(import.meta.url).pathname)
export const TASKS_DIR = path.join(BENCH_DIR, "tasks")
export const REPO_ROOT = path.resolve(BENCH_DIR, "..")
export type Variant = "swarmos" | "baseline"
export interface BenchTask {
id: string
category: "bug" | "refactor" | "constraints" | "ui" | string
title: string
prompt: string
/** Command run from the workspace root after hidden tests are copied into `__hidden__/`. */
verify: string
timeoutSec: number
dir: string
}
export interface DriverResult {
exitCode: number
durationMs: number
output: string
}
export interface Driver {
name: string
run(task: BenchTask, workdir: string, variant: Variant): Promise<DriverResult>
}
export interface AggregatedTelemetry {
sessions: number
turns: number
toolCalls: number
toolFailures: number
breakerFires: number
tokens: { input: number; output: number; cacheRead: number } | null
cacheHitRatio: number | null
}
export interface BenchResult {
taskId: string
category: string
variant: Variant
driver: string
repeat: number
success: boolean
verifyExitCode: number
agentExitCode: number
durationMs: number
telemetry: AggregatedTelemetry
verifyTail: string
}
export async function loadTasks(filter?: string[], tasksDir = TASKS_DIR): Promise<BenchTask[]> {
const entries = await fs.readdir(tasksDir, { withFileTypes: true })
const tasks: BenchTask[] = []
for (const e of entries) {
if (!e.isDirectory()) continue
const dir = path.join(tasksDir, e.name)
const raw = JSON.parse(await fs.readFile(path.join(dir, "task.json"), "utf8"))
if (filter?.length && !filter.includes(raw.id)) continue
tasks.push({ timeoutSec: 600, ...raw, dir })
}
return tasks.sort((a, b) => a.id.localeCompare(b.id))
}
async function copyDir(src: string, dest: string): Promise<void> {
await fs.cp(src, dest, { recursive: true })
}
async function exists(p: string): Promise<boolean> {
try {
await fs.access(p)
return true
} catch {
return false
}
}
function sh(command: string, cwd: string, timeoutMs: number, env: Record<string, string> = {}): Promise<DriverResult> {
const started = Date.now()
return new Promise((resolve) => {
exec(command, { cwd, timeout: timeoutMs, maxBuffer: 64 * 1024 * 1024, env: { ...process.env, ...env } }, (err: any, stdout, stderr) => {
resolve({
exitCode: err ? (typeof err.code === "number" ? err.code : 1) : 0,
durationMs: Date.now() - started,
output: `${stdout ?? ""}${stderr ?? ""}`,
})
})
})
}
/**
* Copies a task's starting repo into a fresh git workspace and writes the variant's SwarmOS
* config. The baseline still loads SwarmOS, in observe mode, so both variants produce telemetry.
*/
export async function prepareWorkspace(task: BenchTask, variant: Variant, pluginPath?: string): Promise<string> {
const workdir = await fs.mkdtemp(path.join(os.tmpdir(), `swarmos-bench-${task.id}-`))
await copyDir(path.join(task.dir, "repo"), workdir)
await fs.writeFile(path.join(workdir, ".gitignore"), ".opencode/telemetry/\n.opencode/verify/\nnode_modules/\n__pycache__/\n")
await fs.mkdir(path.join(workdir, ".opencode"), { recursive: true })
await fs.writeFile(
path.join(workdir, ".opencode", "swarmos.json"),
JSON.stringify({ mode: variant === "swarmos" ? "full" : "observe" }, null, 2),
)
if (pluginPath) {
await fs.writeFile(
path.join(workdir, "opencode.json"),
JSON.stringify({ plugins: [pluginPath], experimental: { subagent_depth: 4 } }, null, 2),
)
}
const git = (...args: string[]) => execFileSync("git", args, { cwd: workdir, stdio: "pipe" })
git("init", "-q")
git("config", "user.email", "[email protected]")
git("config", "user.name", "SwarmOS Bench")
git("add", "-A")
git("commit", "-qm", "task start")
return workdir
}
/** Copies hidden tests in after the agent finishes and runs the task's verifier. */
export async function verifyWorkspace(task: BenchTask, workdir: string): Promise<DriverResult> {
const hidden = path.join(task.dir, "hidden")
const target = path.join(workdir, "__hidden__")
await fs.rm(target, { recursive: true, force: true })
if (await exists(hidden)) await copyDir(hidden, target)
return sh(task.verify, workdir, 120_000, { CI: "1", NO_COLOR: "1" })
}
export async function aggregateTelemetry(workdir: string): Promise<AggregatedTelemetry> {
const dir = path.join(workdir, ".opencode", "telemetry")
let files: string[] = []
try {
files = (await fs.readdir(dir)).filter((f) => f.endsWith(".jsonl")).map((f) => path.join(dir, f))
} catch {
// no telemetry
}
const agg: AggregatedTelemetry = { sessions: 0, turns: 0, toolCalls: 0, toolFailures: 0, breakerFires: 0, tokens: null, cacheHitRatio: null }
for (const file of files) {
const records: TelemetryRecord[] = await readTelemetryFile(file)
const s = summarizeTelemetry(records)
agg.sessions++
agg.turns += s.turns
agg.toolCalls += s.toolCalls
agg.toolFailures += s.toolFailures
agg.breakerFires += s.breakerFires
if (s.tokens) {
agg.tokens ??= { input: 0, output: 0, cacheRead: 0 }
agg.tokens.input += s.tokens.input
agg.tokens.output += s.tokens.output
agg.tokens.cacheRead += s.tokens.cacheRead
}
}
if (agg.tokens && agg.tokens.input + agg.tokens.cacheRead > 0) {
agg.cacheHitRatio = +(agg.tokens.cacheRead / (agg.tokens.input + agg.tokens.cacheRead)).toFixed(4)
}
return agg
}
/** Applies the task's reference solution: proves the task is solvable and its verifier accepts a fix. */
export const referenceDriver: Driver = {
name: "reference",
async run(task, workdir) {
const started = Date.now()
await copyDir(path.join(task.dir, "solution"), workdir)
return { exitCode: 0, durationMs: Date.now() - started, output: "applied reference solution" }
},
}
/** Does nothing: proves the task's verifier fails on the untouched starting repo. */
export const noopDriver: Driver = {
name: "noop",
async run() {
return { exitCode: 0, durationMs: 0, output: "" }
},
}
/**
* Runs a real agent. The command template comes from BENCH_AGENT_CMD (default
* `opencode run "$BENCH_PROMPT"`); the prompt is passed through the BENCH_PROMPT env var.
*/
export function agentDriver(command = process.env.BENCH_AGENT_CMD ?? 'opencode run "$BENCH_PROMPT"'): Driver {
return {
name: "agent",
run(task, workdir, variant) {
return sh(command, workdir, task.timeoutSec * 1000, {
BENCH_PROMPT: task.prompt,
BENCH_TASK_ID: task.id,
BENCH_VARIANT: variant,
SWARMOS_MODE: variant === "swarmos" ? "full" : "observe",
})
},
}
}
export interface RunBenchOptions {
tasks: BenchTask[]
variants: Variant[]
driver: Driver
repeat?: number
pluginPath?: string
keepWorkspaces?: boolean
onResult?: (r: BenchResult) => void
}
export async function runBench(opts: RunBenchOptions): Promise<BenchResult[]> {
const results: BenchResult[] = []
for (const task of opts.tasks) {
for (const variant of opts.variants) {
for (let repeat = 1; repeat <= (opts.repeat ?? 1); repeat++) {
const workdir = await prepareWorkspace(task, variant, opts.pluginPath)
try {
const agent = await opts.driver.run(task, workdir, variant)
const verify = await verifyWorkspace(task, workdir)
const result: BenchResult = {
taskId: task.id,
category: task.category,
variant,
driver: opts.driver.name,
repeat,
success: verify.exitCode === 0,
verifyExitCode: verify.exitCode,
agentExitCode: agent.exitCode,
durationMs: agent.durationMs,
telemetry: await aggregateTelemetry(workdir),
verifyTail: verify.output.split("\n").filter((l) => l.trim()).slice(-15).join("\n"),
}
results.push(result)
opts.onResult?.(result)
} finally {
if (!opts.keepWorkspaces) await fs.rm(workdir, { recursive: true, force: true })
}
}
}
}
return results
}
export interface VariantSummary {
variant: Variant
runs: number
successRate: number
meanTurns: number | null
meanToolCalls: number | null
toolFailureRate: number | null
breakerFires: number
meanTokens: number | null
cacheHitRatio: number | null
meanDurationMs: number
byCategory: Record<string, { runs: number; successRate: number }>
}
const mean = (xs: number[]) => (xs.length ? +(xs.reduce((a, b) => a + b, 0) / xs.length).toFixed(2) : null)
export function summarize(results: BenchResult[]): VariantSummary[] {
const variants = [...new Set(results.map((r) => r.variant))]
return variants.map((variant) => {
const rs = results.filter((r) => r.variant === variant)
const withTelemetry = rs.filter((r) => r.telemetry.sessions > 0)
const calls = withTelemetry.reduce((a, r) => a + r.telemetry.toolCalls, 0)
const failures = withTelemetry.reduce((a, r) => a + r.telemetry.toolFailures, 0)
const withTokens = rs.filter((r) => r.telemetry.tokens)
const tokIn = withTokens.reduce((a, r) => a + r.telemetry.tokens!.input, 0)
const tokCache = withTokens.reduce((a, r) => a + r.telemetry.tokens!.cacheRead, 0)
const byCategory: VariantSummary["byCategory"] = {}
for (const cat of [...new Set(rs.map((r) => r.category))]) {
const c = rs.filter((r) => r.category === cat)
byCategory[cat] = { runs: c.length, successRate: +(c.filter((r) => r.success).length / c.length).toFixed(4) }
}
return {
variant,
runs: rs.length,
successRate: rs.length ? +(rs.filter((r) => r.success).length / rs.length).toFixed(4) : 0,
meanTurns: mean(withTelemetry.map((r) => r.telemetry.turns)),
meanToolCalls: mean(withTelemetry.map((r) => r.telemetry.toolCalls)),
toolFailureRate: calls > 0 ? +(failures / calls).toFixed(4) : null,
breakerFires: rs.reduce((a, r) => a + r.telemetry.breakerFires, 0),
meanTokens: mean(withTokens.map((r) => r.telemetry.tokens!.input + r.telemetry.tokens!.output + r.telemetry.tokens!.cacheRead)),
cacheHitRatio: tokIn + tokCache > 0 ? +(tokCache / (tokIn + tokCache)).toFixed(4) : null,
meanDurationMs: mean(rs.map((r) => r.durationMs)) ?? 0,
byCategory,
}
})
}
const pct = (x: number | null) => (x === null ? "n/a" : `${(x * 100).toFixed(1)}%`)
const num = (x: number | null) => (x === null ? "n/a" : String(x))
export function formatReport(results: BenchResult[], meta: { driver: string; startedAt: string }): string {
const summaries = summarize(results)
const categories = [...new Set(results.map((r) => r.category))].sort()
const taskIds = [...new Set(results.map((r) => r.taskId))].sort()
const variants = summaries.map((s) => s.variant)
return [
`# SwarmOS Benchmark (${meta.driver} driver, ${meta.startedAt})`,
"",
"| Variant | Runs | Success | Mean turns | Mean tool calls | Tool failure rate | Breaker fires | Mean tokens | Cache hit | Mean time |",
"| :--- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
...summaries.map(
(s) =>
`| ${s.variant} | ${s.runs} | ${pct(s.successRate)} | ${num(s.meanTurns)} | ${num(s.meanToolCalls)} | ${pct(s.toolFailureRate)} | ${s.breakerFires} | ${num(s.meanTokens)} | ${pct(s.cacheHitRatio)} | ${(s.meanDurationMs / 1000).toFixed(1)}s |`,
),
"",
"## Success by category",
`| Category | ${variants.join(" | ")} |`,
`| :--- | ${variants.map(() => "---:").join(" | ")} |`,
...categories.map((c) => `| ${c} | ${summaries.map((s) => (s.byCategory[c] ? pct(s.byCategory[c].successRate) : "-")).join(" | ")} |`),
"",
"## Per task",
`| Task | ${variants.join(" | ")} |`,
`| :--- | ${variants.map(() => ":---:").join(" | ")} |`,
...taskIds.map((id) => {
const cells = variants.map((v) => {
const rs = results.filter((r) => r.taskId === id && r.variant === v)
return rs.length ? `${rs.filter((r) => r.success).length}/${rs.length}` : "-"
})
return `| ${id} | ${cells.join(" | ")} |`
}),
"",
"_Token and cache metrics appear only when the host reports usage on `session.generate` events._",
].join("\n")
}
+68
View File
@@ -0,0 +1,68 @@
#!/usr/bin/env bun
/**
* SwarmOS benchmark runner.
*
* bun bench/run.ts --driver reference # check every task is solvable (expects 100%)
* bun bench/run.ts --driver noop # check no task passes untouched (expects 0%)
* bun bench/run.ts --driver agent --repeat 3 # real runs: SwarmOS vs observe-only baseline
*
* Options: --tasks id1,id2 --variants swarmos,baseline --repeat N --plugin <path> --keep --out <dir>
* The agent driver runs BENCH_AGENT_CMD (default: opencode run "$BENCH_PROMPT") in each workspace.
*/
import * as fs from "node:fs/promises"
import * as path from "node:path"
import {
BENCH_DIR,
REPO_ROOT,
agentDriver,
formatReport,
loadTasks,
noopDriver,
referenceDriver,
runBench,
type Driver,
type Variant,
} from "./lib.js"
function arg(name: string): string | undefined {
const idx = process.argv.indexOf(`--${name}`)
return idx >= 0 ? process.argv[idx + 1] : undefined
}
const driverName = arg("driver") ?? "reference"
const drivers: Record<string, () => Driver> = {
reference: () => referenceDriver,
noop: () => noopDriver,
agent: () => agentDriver(),
}
if (!drivers[driverName]) {
console.error(`Unknown driver "${driverName}". Use one of: ${Object.keys(drivers).join(", ")}`)
process.exit(2)
}
const variants = (arg("variants") ?? (driverName === "agent" ? "swarmos,baseline" : "swarmos"))
.split(",")
.map((v) => v.trim()) as Variant[]
const tasks = await loadTasks(arg("tasks")?.split(","))
const pluginPath = driverName === "agent" ? path.resolve(arg("plugin") ?? path.join(REPO_ROOT, "swarmos-plugin.ts")) : undefined
const startedAt = new Date().toISOString()
console.log(`Running ${tasks.length} task(s) x ${variants.length} variant(s) with the ${driverName} driver`)
const results = await runBench({
tasks,
variants,
driver: drivers[driverName](),
repeat: Number(arg("repeat") ?? 1),
pluginPath,
keepWorkspaces: process.argv.includes("--keep"),
onResult: (r) =>
console.log(`${r.success ? "PASS" : "FAIL"} ${r.taskId} [${r.variant}#${r.repeat}] ${(r.durationMs / 1000).toFixed(1)}s`),
})
const report = formatReport(results, { driver: driverName, startedAt })
const outDir = path.resolve(arg("out") ?? path.join(BENCH_DIR, "results"))
await fs.mkdir(outDir, { recursive: true })
const stamp = startedAt.replace(/[:.]/g, "-")
await fs.writeFile(path.join(outDir, `${stamp}-${driverName}.json`), JSON.stringify({ startedAt, driver: driverName, results }, null, 2))
await fs.writeFile(path.join(outDir, `${stamp}-${driverName}.md`), report)
console.log(`\n${report}\n\nSaved to ${outDir}`)
@@ -0,0 +1,25 @@
import { expect, it } from "bun:test"
import { importBatch } from "../src/importer.ts"
import { Store } from "../src/store.ts"
it("saves every valid record in order before resolving", async () => {
const store = new Store()
const records = Array.from({ length: 8 }, (_, i) => ({ id: `r${i}`, value: i }))
const res = await importBatch(store, records)
expect(res).toEqual({ saved: 8, failed: [] })
expect(store.rows.map((r) => r.id)).toEqual(records.map((r) => r.id))
})
it("reports invalid records with their index", async () => {
const store = new Store()
const res = await importBatch(store, [
{ id: "a", value: 1 },
{ id: "", value: 2 },
{ id: "c", value: Number.NaN },
])
expect(res.saved).toBe(1)
expect(res.failed).toEqual([
{ index: 1, reason: "missing id" },
{ index: 2, reason: "value must be a finite number" },
])
})
@@ -0,0 +1 @@
{ "name": "bench-async", "type": "module", "scripts": { "test": "bun test" } }
@@ -0,0 +1,26 @@
import type { Record, Store } from "./store.js"
export interface ImportResult {
saved: number
failed: Array<{ index: number; reason: string }>
}
function validate(r: Record): string | undefined {
if (!r.id) return "missing id"
if (!Number.isFinite(r.value)) return "value must be a finite number"
return undefined
}
export async function importBatch(store: Store, records: Record[]): Promise<ImportResult> {
const result: ImportResult = { saved: 0, failed: [] }
records.forEach(async (r, index) => {
const reason = validate(r)
if (reason) {
result.failed.push({ index, reason })
return
}
await store.save(r)
result.saved++
})
return result
}
@@ -0,0 +1,13 @@
export interface Record {
id: string
value: number
}
export class Store {
readonly rows: Record[] = []
async save(row: Record): Promise<void> {
await new Promise((r) => setTimeout(r, Math.random() * 5))
this.rows.push(row)
}
}
@@ -0,0 +1,26 @@
import type { Record, Store } from "./store.js"
export interface ImportResult {
saved: number
failed: Array<{ index: number; reason: string }>
}
function validate(r: Record): string | undefined {
if (!r.id) return "missing id"
if (!Number.isFinite(r.value)) return "value must be a finite number"
return undefined
}
export async function importBatch(store: Store, records: Record[]): Promise<ImportResult> {
const result: ImportResult = { saved: 0, failed: [] }
for (const [index, r] of records.entries()) {
const reason = validate(r)
if (reason) {
result.failed.push({ index, reason })
continue
}
await store.save(r)
result.saved++
}
return result
}
+8
View File
@@ -0,0 +1,8 @@
{
"id": "bug-async-batch",
"category": "bug",
"title": "Batch import reports success before records are saved",
"prompt": "importBatch() in src/importer.ts returns { saved: 0 } and the caller sees an empty store, even though every record is valid. Records that fail validation should be reported in `failed` with their index, and saving must keep the original record order. Fix it.",
"verify": "bun test ./__hidden__",
"timeoutSec": 600
}
@@ -0,0 +1,18 @@
import { expect, it } from "bun:test"
import { parseDuration } from "../src/duration.ts"
import { requestTimeout } from "../src/timeout.ts"
it("parses combined units", () => {
expect(parseDuration("1h30m")).toBe(5_400_000)
expect(parseDuration("1h5m10s")).toBe(3_910_000)
expect(parseDuration("2h")).toBe(7_200_000)
expect(parseDuration("90m")).toBe(5_400_000)
expect(parseDuration("45s")).toBe(45_000)
expect(requestTimeout({})).toBe(30_000)
})
it("rejects invalid input", () => {
for (const bad of ["", "5x", "h", "1h junk", "10", "-5s"]) {
expect(() => parseDuration(bad)).toThrow(RangeError)
}
})
@@ -0,0 +1 @@
{ "name": "bench-duration", "type": "module", "scripts": { "test": "bun test" } }
@@ -0,0 +1,12 @@
const UNIT_MS: Record<string, number> = { h: 3_600_000, m: 60_000, s: 1000 }
export function parseDuration(input: string): number {
let total = 0
const re = /(\d+)([hms])/g
let m: RegExpExecArray | null
while ((m = re.exec(input)) !== null) {
total += Number(m[1]) * (UNIT_MS[m[2]] ?? 0)
if (m[2] === "h") break
}
return total === 0 ? Number(input) * 1000 : total
}
@@ -0,0 +1,5 @@
import { parseDuration } from "./duration.js"
export function requestTimeout(config: { timeout?: string }): number {
return parseDuration(config.timeout ?? "30s")
}
@@ -0,0 +1,8 @@
const UNIT_MS: Record<string, number> = { h: 3_600_000, m: 60_000, s: 1000 }
export function parseDuration(input: string): number {
if (!/^(?:\d+[hms])+$/.test(input)) throw new RangeError(`Invalid duration: "${input}"`)
let total = 0
for (const m of input.matchAll(/(\d+)([hms])/g)) total += Number(m[1]) * UNIT_MS[m[2]]
return total
}
+8
View File
@@ -0,0 +1,8 @@
{
"id": "bug-duration-parse",
"category": "bug",
"title": "Duration parser mis-reads minutes and seconds",
"prompt": "parseDuration in src/duration.ts should turn strings like \"1h30m\", \"45s\", \"2h\", \"1h5m10s\" and \"90m\" into milliseconds. Right now \"45s\" and \"90m\" come back wrong and garbage like \"5x\" or \"\" is silently accepted. Fix it so valid inputs work and invalid input throws a RangeError.",
"verify": "bun test ./__hidden__",
"timeoutSec": 600
}
@@ -0,0 +1,36 @@
import { expect, it } from "bun:test"
import { LRUCache } from "../src/cache/lru.ts"
import { SessionService } from "../src/session-service.ts"
it("evicts the least recently used entry", () => {
const c = new LRUCache<string, number>(3)
c.set("a", 1)
c.set("b", 2)
c.set("c", 3)
c.set("d", 4)
expect([..."abcd"].filter((k) => c.has(k))).toEqual(["b", "c", "d"])
expect(c.size).toBe(3)
})
it("treats get() as a use", () => {
const c = new LRUCache<string, number>(3)
c.set("a", 1)
c.set("b", 2)
c.set("c", 3)
expect(c.get("a")).toBe(1)
c.set("d", 4)
expect([..."abcd"].filter((k) => c.has(k))).toEqual(["a", "c", "d"])
})
it("keeps active sessions logged in", () => {
const svc = new SessionService(3)
svc.login("alice")
svc.login("bob")
svc.login("dave")
svc.touch("alice")
svc.login("carol")
expect(svc.isLoggedIn("alice")).toBe(true)
expect(svc.isLoggedIn("bob")).toBe(false)
expect(svc.isLoggedIn("dave")).toBe(true)
expect(svc.isLoggedIn("carol")).toBe(true)
})
@@ -0,0 +1 @@
{ "name": "bench-lru", "type": "module", "scripts": { "test": "bun test" } }
+26
View File
@@ -0,0 +1,26 @@
export class LRUCache<K, V> {
private map = new Map<K, V>()
constructor(private readonly capacity: number) {}
get(key: K): V | undefined {
return this.map.get(key)
}
set(key: K, value: V): void {
if (this.map.has(key)) this.map.delete(key)
this.map.set(key, value)
if (this.map.size > this.capacity) {
const keys = Array.from(this.map.keys())
this.map.delete(keys[keys.length - 2])
}
}
has(key: K): boolean {
return this.map.has(key)
}
get size(): number {
return this.map.size
}
}
@@ -0,0 +1,29 @@
import { LRUCache } from "./cache/lru.js"
export interface Session {
userId: string
lastSeen: number
}
export class SessionService {
private cache: LRUCache<string, Session>
constructor(capacity = 3) {
this.cache = new LRUCache(capacity)
}
login(userId: string): void {
this.cache.set(userId, { userId, lastSeen: Date.now() })
}
touch(userId: string): boolean {
const s = this.cache.get(userId)
if (!s) return false
s.lastSeen = Date.now()
return true
}
isLoggedIn(userId: string): boolean {
return this.cache.has(userId)
}
}
+29
View File
@@ -0,0 +1,29 @@
export class LRUCache<K, V> {
private map = new Map<K, V>()
constructor(private readonly capacity: number) {}
get(key: K): V | undefined {
if (!this.map.has(key)) return undefined
const value = this.map.get(key) as V
this.map.delete(key)
this.map.set(key, value)
return value
}
set(key: K, value: V): void {
if (this.map.has(key)) this.map.delete(key)
this.map.set(key, value)
if (this.map.size > this.capacity) {
this.map.delete(this.map.keys().next().value as K)
}
}
has(key: K): boolean {
return this.map.has(key)
}
get size(): number {
return this.map.size
}
}
+8
View File
@@ -0,0 +1,8 @@
{
"id": "bug-lru-eviction",
"category": "bug",
"title": "Session cache evicts the session that was just used",
"prompt": "Our session service keeps logging users out: a user who is actively making requests gets evicted from the session cache while idle sessions stay. Find the root cause and fix it. Keep the public API of src/session-service.ts unchanged.",
"verify": "bun test ./__hidden__",
"timeoutSec": 600
}
@@ -0,0 +1,18 @@
import { expect, it } from "bun:test"
import { paginate } from "../src/paginate.ts"
import { productPage } from "../src/products.ts"
it("returns non-overlapping 1-indexed pages", () => {
const ids = [1, 2, 3].flatMap((p) => productPage(p).items.map((x) => x.id))
expect(ids).toEqual(Array.from({ length: 23 }, (_, i) => i + 1))
})
it("reports totalPages exactly", () => {
expect(paginate([1, 2, 3, 4], 1, 2).totalPages).toBe(2)
expect(paginate([1, 2, 3, 4, 5], 1, 2).totalPages).toBe(3)
expect(paginate([], 1, 10).totalPages).toBe(0)
})
it("returns an empty page past the end", () => {
expect(paginate([1, 2, 3], 3, 2).items).toEqual([])
})
@@ -0,0 +1 @@
{ "name": "bench-paginate", "type": "module", "scripts": { "test": "bun test" } }
@@ -0,0 +1,11 @@
export interface Page<T> {
items: T[]
page: number
totalPages: number
}
export function paginate<T>(items: T[], page: number, size: number): Page<T> {
const start = (page - 1) * size - (page > 1 ? 1 : 0)
const totalPages = Math.floor(items.length / size) + 1
return { items: items.slice(start, start + size), page, totalPages }
}
@@ -0,0 +1,7 @@
import { paginate } from "./paginate.js"
export const PRODUCTS = Array.from({ length: 23 }, (_, i) => ({ id: i + 1, name: `Product ${i + 1}` }))
export function productPage(page: number, size = 10) {
return paginate(PRODUCTS, page, size)
}
@@ -0,0 +1,11 @@
export interface Page<T> {
items: T[]
page: number
totalPages: number
}
export function paginate<T>(items: T[], page: number, size: number): Page<T> {
const start = (page - 1) * size
const totalPages = Math.ceil(items.length / size)
return { items: items.slice(start, start + size), page, totalPages }
}
+8
View File
@@ -0,0 +1,8 @@
{
"id": "bug-paginate",
"category": "bug",
"title": "Pagination repeats an item across pages",
"prompt": "Users report that page 2 of the product list starts with the last product from page 1, and the last page sometimes comes back empty. Fix the pagination so pages are 1-indexed, never overlap, and the reported totalPages is correct. Do not change the function signatures in src/paginate.ts.",
"verify": "bun test ./__hidden__",
"timeoutSec": 600
}
@@ -0,0 +1,34 @@
import { expect, it } from "bun:test"
import { loadConfig } from "../src/config.ts"
import { parseEnv } from "../src/env.ts"
it("parses the supported forms", () => {
const env = parseEnv(
[
"# comment",
"",
"A=plain value # trailing",
"export B=exported",
"C='single # kept $literal'",
'D="line1\\nline2\\t\\"q\\" \\\\"',
"A=override",
"EMPTY=",
].join("\n"),
)
expect(env).toEqual({
A: "override",
B: "exported",
C: "single # kept $literal",
D: 'line1\nline2\t"q" \\',
EMPTY: "",
})
})
it("reports the failing line number", () => {
expect(() => parseEnv("OK=1\n9BAD=2")).toThrow(SyntaxError)
expect(() => parseEnv("OK=1\n\nnot a pair")).toThrow("line 3")
})
it("config uses the parser", () => {
expect(loadConfig('HOST="db.local"\nPORT=8080 # main\n')).toEqual({ port: 8080, host: "db.local" })
})
@@ -0,0 +1 @@
{ "name": "bench-env", "type": "module", "scripts": { "test": "bun test" } }
@@ -0,0 +1,8 @@
export function loadConfig(text: string): { port: number; host: string } {
const env: Record<string, string> = {}
for (const line of text.split("\n")) {
const [k, v] = line.split("=")
if (k) env[k.trim()] = (v ?? "").trim()
}
return { port: Number(env.PORT ?? 3000), host: env.HOST ?? "localhost" }
}
@@ -0,0 +1,3 @@
export function parseEnv(text: string): Record<string, string> {
throw new Error("not implemented")
}
@@ -0,0 +1,6 @@
import { parseEnv } from "./env.js"
export function loadConfig(text: string): { port: number; host: string } {
const env = parseEnv(text)
return { port: Number(env.PORT ?? 3000), host: env.HOST ?? "localhost" }
}
@@ -0,0 +1,25 @@
const LINE_RE = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/
function unescapeDouble(s: string): string {
return s.replace(/\\([nt"\\])/g, (_, c) => ({ n: "\n", t: "\t", '"': '"', "\\": "\\" })[c as "n"])
}
export function parseEnv(text: string): Record<string, string> {
const env: Record<string, string> = {}
text.split("\n").forEach((raw, i) => {
const line = raw.trim()
if (!line || line.startsWith("#")) return
const m = LINE_RE.exec(raw)
if (!m) throw new SyntaxError(`Invalid .env entry on line ${i + 1}`)
const [, key, rest] = m
let value: string
const dq = /^"((?:[^"\\]|\\.)*)"(?:\s+#.*)?$/.exec(rest)
const sq = /^'([^']*)'(?:\s+#.*)?$/.exec(rest)
if (dq) value = unescapeDouble(dq[1])
else if (sq) value = sq[1]
else if (/^["']/.test(rest)) throw new SyntaxError(`Unterminated quote on line ${i + 1}`)
else value = rest.replace(/\s+#.*$/, "").trim()
env[key] = value
})
return env
}
+8
View File
@@ -0,0 +1,8 @@
{
"id": "feature-env-parser",
"category": "constraints",
"title": "Implement a .env parser with strict rules",
"prompt": "Implement parseEnv(text) in src/env.ts returning Record<string, string>. Rules: blank lines and lines starting with # are ignored; an optional leading `export ` is allowed; keys match [A-Za-z_][A-Za-z0-9_]*; values may be unquoted (trimmed, with an inline ` #comment` removed), single-quoted (taken literally) or double-quoted (supporting \\n, \\t, \\\" and \\\\ escapes); duplicate keys: last one wins; any other line throws SyntaxError whose message contains `line N` (1-based). Then make src/config.ts use parseEnv instead of its naive split.",
"verify": "bun test ./__hidden__",
"timeoutSec": 900
}
@@ -0,0 +1,24 @@
import { expect, it } from "bun:test"
import { slugify } from "../src/slugify.ts"
it("lowercases, strips accents and collapses separators", () => {
expect(slugify("Hello World")).toBe("hello-world")
expect(slugify("Crème Brûlée")).toBe("creme-brulee")
expect(slugify(" --A__B!!c ")).toBe("a-b-c")
})
it("supports custom separators and ampersands", () => {
expect(slugify("Rock & Roll", { separator: "_" })).toBe("rock_and_roll")
expect(slugify("R&D")).toBe("r-and-d")
})
it("truncates on word boundaries without trailing separators", () => {
expect(slugify("the quick brown fox", { maxLength: 12 })).toBe("the-quick")
expect(slugify("the quick brown fox", { maxLength: 15 })).toBe("the-quick-brown")
expect(slugify("supercalifragilistic", { maxLength: 5 })).toBe("super")
})
it("throws on empty slugs", () => {
expect(() => slugify("!!!")).toThrow(TypeError)
expect(() => slugify("")).toThrow("empty slug")
})
@@ -0,0 +1 @@
{ "name": "bench-slugify", "type": "module", "scripts": { "test": "bun test" } }
@@ -0,0 +1,8 @@
export interface SlugifyOptions {
separator?: string
maxLength?: number
}
export function slugify(input: string, options: SlugifyOptions = {}): string {
throw new Error("not implemented")
}
@@ -0,0 +1,29 @@
export interface SlugifyOptions {
separator?: string
maxLength?: number
}
export function slugify(input: string, options: SlugifyOptions = {}): string {
const sep = options.separator ?? "-"
const words = input
.normalize("NFKD")
.replace(/[\u0300-\u036f]/g, "")
.toLowerCase()
.replace(/&/g, " and ")
.split(/[^a-z0-9]+/)
.filter(Boolean)
if (words.length === 0) throw new TypeError("empty slug")
let slug = words.join(sep)
const max = options.maxLength
if (max !== undefined && slug.length > max) {
let out = ""
for (const w of words) {
const next = out ? `${out}${sep}${w}` : w
if (next.length > max) break
out = next
}
slug = out || words[0].slice(0, max)
}
return slug
}
@@ -0,0 +1,8 @@
{
"id": "feature-slugify-constraints",
"category": "constraints",
"title": "Implement slugify with eight constraints",
"prompt": "Implement slugify(input, options?) in src/slugify.ts. Requirements, all mandatory: (1) lowercase output; (2) strip accents (\"Crème Brûlée\" -> \"creme-brulee\"); (3) any run of characters that are not a-z or 0-9 becomes a single separator; (4) no leading or trailing separator; (5) options.separator overrides the default \"-\"; (6) options.maxLength truncates the result without leaving a trailing separator and without cutting a word in half when the word boundary is within the limit; (7) \"&\" becomes the word \"and\"; (8) an input that produces an empty slug throws TypeError(\"empty slug\"). Do not add dependencies.",
"verify": "bun test ./__hidden__",
"timeoutSec": 900
}
Whitespace-only changes.
@@ -0,0 +1,27 @@
import unittest
from stats.summary import median, summarize
class SummaryTest(unittest.TestCase):
def test_even_median(self):
self.assertEqual(median([4, 1, 3, 2]), 2.5)
def test_odd_median(self):
self.assertEqual(median([5, 1, 3]), 3)
def test_does_not_mutate(self):
data = [3, 1, 2]
summarize(data)
self.assertEqual(data, [3, 1, 2])
def test_keys(self):
self.assertEqual(summarize([1, 2]), {"count": 2, "min": 1, "max": 2, "median": 1.5})
def test_empty(self):
with self.assertRaises(ValueError):
summarize([])
if __name__ == "__main__":
unittest.main()
Whitespace-only changes.
@@ -0,0 +1,15 @@
def median(values):
values.sort()
mid = len(values) // 2
return values[mid]
def summarize(values):
if not values:
raise ValueError("values must not be empty")
return {
"count": len(values),
"min": min(values),
"max": max(values),
"median": median(values),
}
@@ -0,0 +1,17 @@
def median(values):
ordered = sorted(values)
mid = len(ordered) // 2
if len(ordered) % 2 == 0:
return (ordered[mid - 1] + ordered[mid]) / 2
return ordered[mid]
def summarize(values):
if not values:
raise ValueError("values must not be empty")
return {
"count": len(values),
"min": min(values),
"max": max(values),
"median": median(values),
}
+8
View File
@@ -0,0 +1,8 @@
{
"id": "py-median-stats",
"category": "bug",
"title": "Python stats helpers return wrong medians and mutate input",
"prompt": "stats/summary.py: median() is wrong for even-length lists and the caller's list comes back reordered after calling summarize(). Fix both bugs. summarize() must return the same keys as today. Use only the standard library.",
"verify": "python3 -m unittest discover -s __hidden__ -t .",
"timeoutSec": 600
}
@@ -0,0 +1,28 @@
import { expect, it } from "bun:test"
import { readFileSync } from "node:fs"
import { join } from "node:path"
import { inviteAdmin } from "../src/admin.ts"
import { signupHandler } from "../src/handlers.ts"
import * as validation from "../src/validation.ts"
const src = (f: string) => readFileSync(join(import.meta.dir, "..", "src", f), "utf8")
it("exports helpers from validation.ts", () => {
expect(validation.isEmail("[email protected]")).toBe(true)
expect(validation.isStrongPassword("Password123")).toBe(true)
expect(validation.validateSignup({ email: "nope", password: "short" })).toEqual(["invalid email", "weak password"])
})
it("handlers and admin no longer define their own helpers", () => {
expect(src("handlers.ts")).not.toMatch(/function\s+(isEmail|isStrongPassword|validateSignup)\b/)
expect(src("admin.ts")).not.toMatch(/function\s+isEmail\b/)
expect(src("handlers.ts")).toContain("validation")
expect(src("admin.ts")).toContain("validation")
})
it("keeps behaviour", () => {
expect(signupHandler({ email: "[email protected]", password: "Sup3rStrongPw" })).toEqual({ status: 201, errors: [] })
expect(signupHandler({ email: "bad", password: "Sup3rStrongPw" })).toEqual({ status: 400, errors: ["invalid email"] })
expect(inviteAdmin("[email protected]")).toEqual({ ok: true, email: "[email protected]" })
expect(inviteAdmin("nope")).toEqual({ ok: false, error: "invalid email" })
})
@@ -0,0 +1 @@
{ "name": "bench-extract", "type": "module", "scripts": { "test": "bun test" } }
@@ -0,0 +1,8 @@
function isEmail(value: string): boolean {
return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(value)
}
export function inviteAdmin(email: string) {
if (!isEmail(email)) return { ok: false as const, error: "invalid email" }
return { ok: true as const, email: email.toLowerCase() }
}
@@ -0,0 +1,19 @@
export function isEmail(value: string): boolean {
return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(value)
}
export function isStrongPassword(value: string): boolean {
return value.length >= 10 && /[A-Z]/.test(value) && /[0-9]/.test(value)
}
export function validateSignup(input: { email: string; password: string }): string[] {
const errors: string[] = []
if (!isEmail(input.email)) errors.push("invalid email")
if (!isStrongPassword(input.password)) errors.push("weak password")
return errors
}
export function signupHandler(body: { email: string; password: string }) {
const errors = validateSignup(body)
return errors.length ? { status: 400, errors } : { status: 201, errors: [] }
}
@@ -0,0 +1,6 @@
import { isEmail } from "./validation.js"
export function inviteAdmin(email: string) {
if (!isEmail(email)) return { ok: false as const, error: "invalid email" }
return { ok: true as const, email: email.toLowerCase() }
}
@@ -0,0 +1,6 @@
import { validateSignup } from "./validation.js"
export function signupHandler(body: { email: string; password: string }) {
const errors = validateSignup(body)
return errors.length ? { status: 400, errors } : { status: 201, errors: [] }
}
@@ -0,0 +1,14 @@
export function isEmail(value: string): boolean {
return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(value)
}
export function isStrongPassword(value: string): boolean {
return value.length >= 10 && /[A-Z]/.test(value) && /[0-9]/.test(value)
}
export function validateSignup(input: { email: string; password: string }): string[] {
const errors: string[] = []
if (!isEmail(input.email)) errors.push("invalid email")
if (!isStrongPassword(input.password)) errors.push("weak password")
return errors
}
@@ -0,0 +1,8 @@
{
"id": "refactor-extract-validation",
"category": "refactor",
"title": "Extract validation helpers into their own module",
"prompt": "src/handlers.ts mixes HTTP handlers with input validation. Move isEmail, isStrongPassword and validateSignup into a new src/validation.ts (exported from there), import them in src/handlers.ts, and make src/admin.ts import them from src/validation.ts too instead of duplicating isEmail. Behaviour must not change. src/handlers.ts must no longer define any of those functions.",
"verify": "bun test ./__hidden__",
"timeoutSec": 900
}
@@ -0,0 +1,23 @@
import { expect, it } from "bun:test"
import { dbLog, query } from "../src/db.ts"
import { handle, httpLog } from "../src/http.ts"
import { jobLog, runJob } from "../src/jobs.ts"
import { createLogger } from "../src/logger.ts"
it("filters below the configured level and defaults to info", () => {
const l = (createLogger as any)({ name: "t" })
l.log("debug", "hidden")
l.log("info", "shown")
expect(l.lines).toEqual(["[INFO] t: shown"])
})
it("callers use their configured levels", () => {
query("SELECT 1")
query("DROP TABLE x")
expect(dbLog.lines).toEqual(["[WARN] db: destructive query"])
handle("/ok")
handle("/boom")
expect(httpLog.lines).toEqual(["[ERROR] http: handler crashed"])
runJob("j1")
expect(jobLog.lines).toEqual(["[DEBUG] jobs: start j1", "[INFO] jobs: done j1"])
})
@@ -0,0 +1 @@
{ "name": "bench-logger", "type": "module", "scripts": { "test": "bun test" } }
@@ -0,0 +1,9 @@
import { createLogger } from "./logger.js"
export const dbLog = createLogger("db")
export function query(sql: string): number {
dbLog.log("debug", `query ${sql}`)
if (sql.includes("DROP")) dbLog.log("warn", "destructive query")
return sql.length
}
@@ -0,0 +1,12 @@
import { createLogger } from "./logger.js"
export const httpLog = createLogger("http")
export function handle(path: string): number {
httpLog.log("info", `GET ${path}`)
if (path === "/boom") {
httpLog.log("error", "handler crashed")
return 500
}
return 200
}
@@ -0,0 +1,8 @@
import { createLogger } from "./logger.js"
export const jobLog = createLogger("jobs")
export function runJob(id: string): void {
jobLog.log("debug", `start ${id}`)
jobLog.log("info", `done ${id}`)
}
@@ -0,0 +1,16 @@
export type Level = "debug" | "info" | "warn" | "error"
export interface Logger {
lines: string[]
log(level: Level, msg: string): void
}
export function createLogger(name: string): Logger {
const lines: string[] = []
return {
lines,
log(level, msg) {
lines.push(`[${level.toUpperCase()}] ${name}: ${msg}`)
},
}
}
@@ -0,0 +1,9 @@
import { createLogger } from "./logger.js"
export const dbLog = createLogger({ name: "db", level: "warn" })
export function query(sql: string): number {
dbLog.log("debug", `query ${sql}`)
if (sql.includes("DROP")) dbLog.log("warn", "destructive query")
return sql.length
}
@@ -0,0 +1,12 @@
import { createLogger } from "./logger.js"
export const httpLog = createLogger({ name: "http", level: "warn" })
export function handle(path: string): number {
httpLog.log("info", `GET ${path}`)
if (path === "/boom") {
httpLog.log("error", "handler crashed")
return 500
}
return 200
}
@@ -0,0 +1,8 @@
import { createLogger } from "./logger.js"
export const jobLog = createLogger({ name: "jobs", level: "debug" })
export function runJob(id: string): void {
jobLog.log("debug", `start ${id}`)
jobLog.log("info", `done ${id}`)
}
@@ -0,0 +1,20 @@
export type Level = "debug" | "info" | "warn" | "error"
export interface Logger {
lines: string[]
log(level: Level, msg: string): void
}
const ORDER: Record<Level, number> = { debug: 0, info: 1, warn: 2, error: 3 }
export function createLogger(options: { name: string; level?: Level }): Logger {
const lines: string[] = []
const min = ORDER[options.level ?? "info"]
return {
lines,
log(level, msg) {
if (ORDER[level] < min) return
lines.push(`[${level.toUpperCase()}] ${options.name}: ${msg}`)
},
}
}
@@ -0,0 +1,8 @@
{
"id": "refactor-logger-options",
"category": "refactor",
"title": "Change createLogger to take an options object with a level",
"prompt": "Change createLogger(name) in src/logger.ts to createLogger({ name, level }) where level is \"debug\" | \"info\" | \"warn\" | \"error\" (default \"info\"). A logger must drop messages below its level. Update every caller in src/: the db and http loggers should use level \"warn\", the job logger \"debug\". Keep the log line format exactly as it is today.",
"verify": "bun test ./__hidden__",
"timeoutSec": 900
}
@@ -0,0 +1,40 @@
import { expect, it } from "bun:test"
import { readdirSync, readFileSync, statSync } from "node:fs"
import { join } from "node:path"
import { getUserHandler } from "../src/api/handlers.ts"
import { isAdmin } from "../src/auth.ts"
import { displayName } from "../src/profile.ts"
import * as repo from "../src/users/repo.ts"
function walk(dir: string): string[] {
return readdirSync(dir).flatMap((f) => {
const p = join(dir, f)
return statSync(p).isDirectory() ? walk(p) : [p]
})
}
it("exposes findUser returning undefined for unknown ids", () => {
expect((repo as any).findUser(1).name).toBe("Ada")
expect((repo as any).findUser(99)).toBeUndefined()
})
it("keeps a throwing deprecated alias", () => {
expect(repo.getUserById(2).name).toBe("Linus")
expect(() => repo.getUserById(99)).toThrow("User 99 not found")
})
it("keeps caller behaviour", () => {
expect(isAdmin(1)).toBe(true)
expect(isAdmin(99)).toBe(false)
expect(displayName(2)).toBe("LINUS")
expect(() => displayName(99)).toThrow("User 99 not found")
expect(getUserHandler(1).status).toBe(200)
expect(getUserHandler(99)).toEqual({ status: 404, body: { error: "not found" } })
})
it("no other file references getUserById", () => {
const offenders = walk(join(import.meta.dir, "..", "src"))
.filter((f) => !f.endsWith(join("users", "repo.ts")))
.filter((f) => readFileSync(f, "utf8").includes("getUserById"))
expect(offenders).toEqual([])
})
@@ -0,0 +1 @@
{ "name": "bench-rename", "type": "module", "scripts": { "test": "bun test" } }
@@ -0,0 +1,11 @@
import { getUserById } from "../users/repo.js"
import { displayName } from "../profile.js"
export function getUserHandler(id: number): { status: number; body: unknown } {
try {
const user = getUserById(id)
return { status: 200, body: { ...user, display: displayName(id) } }
} catch {
return { status: 404, body: { error: "not found" } }
}
}
@@ -0,0 +1,9 @@
import { getUserById } from "./users/repo.js"
export function isAdmin(id: number): boolean {
try {
return getUserById(id).admin
} catch {
return false
}
}
@@ -0,0 +1,5 @@
import { getUserById } from "./users/repo.js"
export function displayName(id: number): string {
return getUserById(id).name.toUpperCase()
}
@@ -0,0 +1,16 @@
export interface User {
id: number
name: string
admin: boolean
}
const USERS: User[] = [
{ id: 1, name: "Ada", admin: true },
{ id: 2, name: "Linus", admin: false },
]
export function getUserById(id: number): User {
const user = USERS.find((u) => u.id === id)
if (!user) throw new Error(`User ${id} not found`)
return user
}
@@ -0,0 +1,8 @@
import { findUser } from "../users/repo.js"
import { displayName } from "../profile.js"
export function getUserHandler(id: number): { status: number; body: unknown } {
const user = findUser(id)
if (!user) return { status: 404, body: { error: "not found" } }
return { status: 200, body: { ...user, display: displayName(id) } }
}
@@ -0,0 +1,5 @@
import { findUser } from "./users/repo.js"
export function isAdmin(id: number): boolean {
return findUser(id)?.admin ?? false
}
@@ -0,0 +1,7 @@
import { findUser } from "./users/repo.js"
export function displayName(id: number): string {
const user = findUser(id)
if (!user) throw new Error(`User ${id} not found`)
return user.name.toUpperCase()
}
@@ -0,0 +1,21 @@
export interface User {
id: number
name: string
admin: boolean
}
const USERS: User[] = [
{ id: 1, name: "Ada", admin: true },
{ id: 2, name: "Linus", admin: false },
]
export function findUser(id: number): User | undefined {
return USERS.find((u) => u.id === id)
}
/** @deprecated Use findUser. */
export function getUserById(id: number): User {
const user = findUser(id)
if (!user) throw new Error(`User ${id} not found`)
return user
}
@@ -0,0 +1,8 @@
{
"id": "refactor-rename-api",
"category": "refactor",
"title": "Rename getUserById to findUser across the codebase",
"prompt": "Rename the function getUserById (src/users/repo.ts) to findUser everywhere in src/. findUser must return undefined instead of throwing when the user does not exist, and every caller must keep its current behaviour (callers that relied on the throw should now throw their own Error with the same message). Keep a deprecated getUserById export in src/users/repo.ts that delegates to findUser and still throws, for external consumers. No file other than src/users/repo.ts may reference getUserById.",
"verify": "bun test ./__hidden__",
"timeoutSec": 900
}
@@ -0,0 +1,51 @@
import { expect, it } from "bun:test"
import { readFileSync } from "node:fs"
import { join } from "node:path"
const css = (f: string) => readFileSync(join(import.meta.dir, "..", "styles", f), "utf8")
function rule(text: string, selector: string): Record<string, string> {
const m = new RegExp(`${selector.replace(".", "\\.")}\\s*\\{([^}]*)\\}`).exec(text)
if (!m) throw new Error(`missing rule ${selector}`)
return Object.fromEntries(
m[1].split(";").map((d) => d.split(":").map((s) => s.trim())).filter((p) => p.length === 2 && p[0]) as [string, string][],
)
}
function luminance(hex: string): number {
const h = hex.replace("#", "")
const full = h.length === 3 ? h.split("").map((c) => c + c).join("") : h
const [r, g, b] = [0, 2, 4].map((i) => parseInt(full.slice(i, i + 2), 16) / 255).map((c) =>
c <= 0.03928 ? c / 12.92 : ((c + 0.055) / 1.055) ** 2.4,
)
return 0.2126 * r + 0.7152 * g + 0.0722 * b
}
const contrast = (a: string, b: string) => {
const [l1, l2] = [luminance(a), luminance(b)].sort((x, y) => y - x)
return (l1 + 0.05) / (l2 + 0.05)
}
it("titles no longer clip descenders", () => {
const t = rule(css("card.css"), ".card-title")
expect(Number(t["line-height"])).toBeGreaterThanOrEqual(1.2)
const clipsByHeight = t.height && t.overflow === "hidden" && parseFloat(t.height) < 22 * 1.2
expect(Boolean(clipsByHeight)).toBe(false)
})
it("all card text meets WCAG AA against the background", () => {
const text = css("card.css")
const bg = rule(text, ".card").background
expect(bg).toBe("#ffffff")
for (const sel of [".card-title", ".card-meta", ".card-body"]) {
expect(contrast(rule(text, sel).color, bg)).toBeGreaterThanOrEqual(4.5)
}
})
it("keeps font sizes and the grid", () => {
const text = css("card.css")
expect(rule(text, ".card-title")["font-size"]).toBe("22px")
expect(rule(text, ".card-meta")["font-size"]).toBe("13px")
expect(rule(text, ".card-body")["font-size"]).toBe("15px")
expect(css("layout.css")).toContain("grid-template-columns: repeat(3, minmax(0, 1fr));")
})
@@ -0,0 +1,10 @@
<!doctype html>
<link rel="stylesheet" href="styles/layout.css" />
<link rel="stylesheet" href="styles/card.css" />
<main class="grid">
<article class="card">
<h2 class="card-title">Graphite Typography Guide</h2>
<p class="card-meta">Updated yesterday · 12 pages</p>
<p class="card-body">Everything you need to typeset long-form copy.</p>
</article>
</main>
@@ -0,0 +1 @@
{ "name": "bench-ui-card", "type": "module", "scripts": { "test": "bun test" } }
@@ -0,0 +1,25 @@
.card {
background: #ffffff;
border-radius: 12px;
padding: 20px;
}
.card-title {
font-size: 22px;
line-height: 1;
height: 22px;
overflow: hidden;
color: #1a1a1a;
}
.card-meta {
font-size: 13px;
line-height: 1.4;
color: #b8b8b8;
}
.card-body {
font-size: 15px;
line-height: 1.5;
color: #444444;
}
@@ -0,0 +1,5 @@
.grid {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
gap: 24px;
}
@@ -0,0 +1,23 @@
.card {
background: #ffffff;
border-radius: 12px;
padding: 20px;
}
.card-title {
font-size: 22px;
line-height: 1.25;
color: #1a1a1a;
}
.card-meta {
font-size: 13px;
line-height: 1.4;
color: #6b6b6b;
}
.card-body {
font-size: 15px;
line-height: 1.5;
color: #444444;
}
@@ -0,0 +1,8 @@
{
"id": "ui-clipping-contrast",
"category": "ui",
"title": "Card titles clip descenders and muted text is unreadable",
"prompt": "On the product cards (styles/card.css) the titles cut off the bottom of letters like g, y and p, and the muted metadata text is very hard to read on the white card. Fix both in styles/card.css. Text colours must reach WCAG AA (4.5:1) against the card background. Do not change any font-size, the grid in styles/layout.css, or the card background colour.",
"verify": "bun test ./__hidden__",
"timeoutSec": 600
}
@@ -0,0 +1,38 @@
import { expect, it } from "bun:test"
import { readFileSync, readdirSync } from "node:fs"
import { join } from "node:path"
const dir = join(import.meta.dir, "..", "styles")
const tokensCss = readFileSync(join(dir, "tokens.css"), "utf8")
const tokens = Object.fromEntries([...tokensCss.matchAll(/--(space-\d):\s*(\d+)px/g)].map((m) => [m[1], Number(m[2])]))
function decls(): Array<{ file: string; prop: string; value: string }> {
return readdirSync(join(dir, "components")).flatMap((file) =>
[...readFileSync(join(dir, "components", file), "utf8").matchAll(/(padding[\w-]*|margin[\w-]*|gap)\s*:\s*([^;]+);/g)].map(
(m) => ({ file, prop: m[1], value: m[2].trim() }),
),
)
}
it("leaves tokens.css untouched", () => {
expect(tokens).toEqual({ "space-1": 4, "space-2": 8, "space-3": 12, "space-4": 16, "space-5": 24, "space-6": 32 })
})
it("uses only tokens (or 0) for spacing", () => {
for (const d of decls()) {
for (const part of d.value.split(/\s+/)) {
expect(part === "0" || /^var\(--space-[1-6]\)$/.test(part)).toBe(true)
}
}
})
it("picks the nearest token for each original value", () => {
// First declaration of each property per file (the component's own rule, not later sub-rules).
const byKey: Record<string, string> = {}
for (const d of decls()) byKey[`${d.file}:${d.prop}`] ??= d.value
expect(byKey["button.css:padding"]).toBe("var(--space-2) var(--space-4)")
expect(byKey["button.css:margin-left"]).toBe("var(--space-2)")
expect(byKey["panel.css:padding"].split(/\s+/)[0]).toBe("var(--space-5)")
expect(byKey["panel.css:margin-bottom"]).toBe("var(--space-6)")
expect(byKey["panel.css:gap"]).toBe("var(--space-3)")
})
@@ -0,0 +1 @@
{ "name": "bench-ui-tokens", "type": "module", "scripts": { "test": "bun test" } }
@@ -0,0 +1,9 @@
.button {
padding: 9px 15px;
margin: 0;
border-radius: 8px;
}
.button + .button {
margin-left: 7px;
}
@@ -0,0 +1,10 @@
.panel {
padding: 22px;
margin-bottom: 30px;
display: flex;
gap: 13px;
}
.panel-header {
padding: 0 0 11px;
}
@@ -0,0 +1,8 @@
:root {
--space-1: 4px;
--space-2: 8px;
--space-3: 12px;
--space-4: 16px;
--space-5: 24px;
--space-6: 32px;
}
@@ -0,0 +1,9 @@
.button {
padding: var(--space-2) var(--space-4);
margin: 0;
border-radius: 8px;
}
.button + .button {
margin-left: var(--space-2);
}
@@ -0,0 +1,10 @@
.panel {
padding: var(--space-5);
margin-bottom: var(--space-6);
display: flex;
gap: var(--space-3);
}
.panel-header {
padding: 0 0 var(--space-3);
}
+8
View File
@@ -0,0 +1,8 @@
{
"id": "ui-spacing-tokens",
"category": "ui",
"title": "Replace hard-coded spacing with design tokens",
"prompt": "Our components use ad-hoc pixel paddings, margins and gaps (p-3 next to p-5 style drift). Replace every padding, margin and gap value in styles/components/*.css with the spacing tokens defined in styles/tokens.css (var(--space-1) ... var(--space-6)), picking the nearest token for each value. Do not change tokens.css. Values of 0 may stay 0.",
"verify": "bun test ./__hidden__",
"timeoutSec": 600
}
+6 -3
View File
@@ -1,6 +1,6 @@
{
"name": "@heretek-ai/swarmos",
"version": "0.5.0",
"version": "1.0.0",
"description": "Omni-Harness & Autonomous Multi-Agent Software Company, GUI Studio, Deep Research & Office Suite for OpenCode V2",
"type": "module",
"main": "./dist/index.js",
@@ -38,8 +38,11 @@
"build": "bun run build:bundle && bun run build:types",
"build:bundle": "bun build src/index.ts --target bun --outfile dist/index.js && bun build src/index.ts --target bun --outfile dist/swarmos.standalone.js && cp dist/swarmos.standalone.js ./swarmos-plugin.ts",
"build:types": "tsc --emitDeclarationOnly",
"typecheck": "tsc --noEmit",
"test": "bun test test/"
"typecheck": "tsc --noEmit && tsc --noEmit -p tsconfig.bench.json",
"test": "bun test test/",
"bench": "bun bench/run.ts",
"bench:check": "bun bench/run.ts --driver reference && bun bench/run.ts --driver noop",
"contract:check": "bun scripts/check-host-contract.ts"
},
"repository": {
"type": "git",
+11
View File
@@ -0,0 +1,11 @@
#!/usr/bin/env bun
// Usage: bun scripts/check-host-contract.ts [path-to-opencode-v2-checkout]
import * as path from "node:path"
import { checkHostContract, formatContractCheck } from "../src/host/contract-check.js"
const defaultFixture = path.resolve(path.dirname(new URL(import.meta.url).pathname), "../test/fixtures/opencode-v2-ref")
const sourceDir = process.argv[2] ?? process.env.OPENCODE_V2_SRC ?? defaultFixture
const report = await checkHostContract(sourceDir)
console.log(formatContractCheck(report))
process.exit(report.ok ? 0 : 1)
+193 -75
View File
@@ -1,6 +1,7 @@
import * as path from "node:path"
import { loadBoard } from "../board/board.js"
import type { OrgBoard } from "../board/types.js"
import { analyzeShell, isReadOnly } from "./shell-parse.js"
export interface PermissionEvaluationEvent {
sessionID?: string
@@ -16,8 +17,15 @@ export interface PermissionEvaluationEvent {
export interface GuardDecision {
allowed: boolean
reason?: string
/** Require explicit approval even when a saved rule would allow the action. */
ask?: boolean
/** Safe to run without prompting (read-only command for a role that only reads). */
autoAllow?: boolean
}
/** Extra policy evaluated after the board guard (taint tracking, task budgets). */
export type GuardEvaluator = (event: PermissionEvaluationEvent, board: OrgBoard) => GuardDecision | undefined | Promise<GuardDecision | undefined>
/**
* Normalizes a resource path to a forward-slash path relative to rootDir.
*/
@@ -78,14 +86,161 @@ export function matchesPathPattern(relPath: string, pattern: string): boolean {
return regex.test(normPath)
}
const DESTRUCTIVE_GIT_RE =
/\bgit\s+(?:reset\s+--hard|clean\s+-[a-zA-Z]*f|push\s+[^;|\n]*--force(?!-with-lease)|checkout\s+--\s+)/i
const LEADERSHIP_AGENTS = new Set(["director", "manager"])
/** Agents whose read-only shell commands are approved without prompting. */
const AUTO_APPROVE_READ_AGENTS = new Set(["director", "manager", "explore", "plan", "qa-adversary"])
const TEST_PATH_PATTERNS = ["**/*.test.*", "**/*.spec.*", "test/**", "tests/**", "e2e/**", "**/__tests__/**", "**/test/**", "**/tests/**"]
const IMPLEMENTATION_AGENTS = new Set(["staff-backend", "staff-ui"])
const TOURNAMENT_EXEMPT_AGENTS = new Set(["architect", "release-ops"])
const LEADERSHIP_MUTATING_SHELL_RE =
/(?:^|[;&|]\s*)(?:rm|mv|cp|chmod|chown|truncate|dd)\s+|\b(?:sed|perl)\s+-[a-zA-Z]*i\b|\bgit\s+(?:commit|merge|rebase|cherry-pick|am|apply|stash\s+pop)\b|(?:^|[^2>])>>?\s*(?!\/dev\/null\b)\S+/i
function isScratchPath(raw: string): boolean {
return /^\/(?:tmp|var\/tmp|dev)\//.test(raw) || raw === "/dev/null"
}
const QA_SOURCE_MUTATING_SHELL_RE =
/\b(?:sed|perl)\s+-[a-zA-Z]*i\b|\bgit\s+(?:commit|merge|rebase|reset|checkout\s+--)\b|(?:^|[;&|]\s*)(?:rm|mv)\s+(?![^;&|\n]*(?:\.test\.|\.spec\.|\btest\/|\btests\/|\be2e\/|\/tmp\/))|(?:^|[^2>])>>?\s*(?!\/dev\/null\b)(?![^;&|\n]*(?:\.test\.|\.spec\.|\btest\/|\btests\/|\be2e\/|\/tmp\/))\S+/i
function isPlaceholder(target: string): boolean {
return target.startsWith("<") && target.endsWith(">")
}
export function isTestPath(relPath: string): boolean {
return TEST_PATH_PATTERNS.some((p) => matchesPathPattern(relPath, p))
}
const ACTIVE_TASK_STATUSES = new Set(["pending", "in_progress", "ready_for_qa", "qa_rejected"])
/**
* Decides whether `agent` may create, modify or delete `rawPath` given locked contracts,
* disjoint task ownership and active worktree tournaments. Used for edit/write/patch
* resources and for file targets of shell commands.
*/
export function evaluateFileMutation(rootDir: string, board: OrgBoard, agent: string, rawPath: string): GuardDecision {
let relPath = normalizeResourcePath(rootDir, rawPath)
if (relPath.startsWith("..") || path.isAbsolute(relPath)) return { allowed: true }
// Paths inside a tournament candidate worktree are judged relative to that worktree.
const wt = /^(\.worktrees\/[^/]+)\/(.+)$/.exec(relPath)
let worktreeTask: OrgBoard["tasks"][number] | undefined
if (wt) {
worktreeTask = board.tasks.find((t) => t.worktree === wt[1])
relPath = wt[2]
}
if (agent && agent !== "architect") {
for (const contract of board.contracts) {
if (contract.files.some((cf) => matchesPathPattern(relPath, cf))) {
return {
allowed: false,
reason: `SwarmOS Contract Guard: File "${relPath}" is locked under architectural contract [${contract.id}] ("${contract.title}"). Only the "architect" role may modify locked contract files.`,
}
}
}
}
if (worktreeTask) {
if (IMPLEMENTATION_AGENTS.has(agent) && worktreeTask.owner !== agent) {
return {
allowed: false,
reason: `SwarmOS Worktree Guard: "${wt![1]}" belongs to task [${worktreeTask.id}] owned by "${worktreeTask.owner}", not "${agent}".`,
}
}
return { allowed: true }
}
if (IMPLEMENTATION_AGENTS.has(agent)) {
const activeTasks = board.tasks.filter((t) => ACTIVE_TASK_STATUSES.has(t.status) && !t.worktree)
const ownedBySelf = activeTasks.some(
(t) => t.owner === agent && t.ownedPaths.some((pattern) => matchesPathPattern(relPath, pattern)),
)
if (!ownedBySelf) {
const conflictingTask = activeTasks.find(
(t) => t.owner !== agent && t.ownedPaths.some((pattern) => matchesPathPattern(relPath, pattern)),
)
if (conflictingTask) {
return {
allowed: false,
reason: `SwarmOS Ownership Guard: File "${relPath}" is claimed by active task [${conflictingTask.id}] assigned to "${conflictingTask.owner}". Role "${agent}" cannot modify files outside its disjoint ownedPaths boundary.`,
}
}
}
}
if (agent && !TOURNAMENT_EXEMPT_AGENTS.has(agent)) {
for (const tour of board.tournaments ?? []) {
if (tour.status !== "active") continue
if (tour.ownedPaths.some((pattern) => matchesPathPattern(relPath, pattern))) {
const where = tour.candidates.map((c) => c.worktree).join(" or ")
return {
allowed: false,
reason: `SwarmOS Tournament Guard: "${relPath}" is contested by active tournament [${tour.id}]. Edit it inside your candidate worktree (${where}), not in the main tree.`,
}
}
}
}
return { allowed: true }
}
function evaluateShellCommand(rootDir: string, board: OrgBoard, agent: string, cmd: string): GuardDecision {
const analysis = analyzeShell(cmd)
const short = cmd.slice(0, 80)
const destructive = analysis.commands.find((c) => c.effects.includes("destructive"))
if (destructive) {
const label = destructive.program === "git" ? "SwarmOS Git Safety Guard" : "SwarmOS Destructive Command Guard"
return {
allowed: false,
reason: `${label}: Destructive command "${short}" (${destructive.note ?? destructive.program}) is blocked across all SwarmOS roles.`,
}
}
if (analysis.unparseable) {
return { allowed: true, ask: true, reason: `SwarmOS Shell Guard: could not parse "${short}"; approve it manually.` }
}
const writeCommands = analysis.commands.filter((c) => c.effects.includes("write"))
const projectWrites = analysis.writes.filter((w) => !isScratchPath(w))
const unknownTargets = writeCommands.some((c) => c.writes.length === 0) || projectWrites.some(isPlaceholder)
if (LEADERSHIP_AGENTS.has(agent) && writeCommands.length > 0 && (projectWrites.length > 0 || unknownTargets)) {
return {
allowed: false,
reason: `SwarmOS Separation-of-Powers Guard: Leadership role "${agent}" is read-only and cannot execute file-mutating or git-committing shell commands ("${short}"). Delegate implementation to subagents.`,
}
}
if (agent === "qa-adversary" && writeCommands.length > 0) {
const nonTest = projectWrites.filter((w) => isPlaceholder(w) || !isTestPath(normalizeResourcePath(rootDir, w)))
if (unknownTargets || nonTest.length > 0) {
return {
allowed: false,
reason: `SwarmOS Red-Team Guard: "qa-adversary" is forbidden from mutating production source files or git history via shell ("${short}").`,
}
}
}
for (const target of projectWrites) {
if (isPlaceholder(target)) continue
const decision = evaluateFileMutation(rootDir, board, agent, target)
if (!decision.allowed) return decision
}
if (analysis.effects.includes("exec") || analysis.effects.includes("network")) {
if (LEADERSHIP_AGENTS.has(agent) || agent === "explore" || agent === "plan") {
return { allowed: true, ask: true, reason: `SwarmOS Shell Guard: "${short}" runs code or reaches the network; read-only role "${agent}" needs approval.` }
}
// QA may only touch tests; inline code could write anywhere, so a human checks it.
if (agent === "qa-adversary" && analysis.commands.some((c) => c.effects.includes("exec") && /inline/.test(c.note ?? ""))) {
return { allowed: true, ask: true, reason: `SwarmOS Red-Team Guard: "${short}" runs inline code SwarmOS cannot inspect; approve it manually.` }
}
return { allowed: true }
}
const readOnly = isReadOnly(analysis, agent !== "explore" && agent !== "plan")
if (readOnly && AUTO_APPROVE_READ_AGENTS.has(agent)) return { allowed: true, autoAllow: true }
if (!readOnly && (agent === "explore" || agent === "plan")) {
return { allowed: true, ask: true, reason: `SwarmOS Shell Guard: "${short}" is not read-only; "${agent}" needs approval.` }
}
return { allowed: true }
}
/**
* Pure synchronous evaluator against an already-loaded OrgBoard.
@@ -99,80 +254,26 @@ export function evaluateBoardGuard(
const action = (event.action ?? "").toLowerCase()
const resources = Array.isArray(event.resources) ? event.resources : []
// 1. File Mutation Guards (edit, write, patch all assert action="edit" or "write" or "patch")
if (action === "edit" || action === "write" || action === "patch") {
for (const rawResource of resources) {
if (typeof rawResource !== "string" || rawResource === "*") continue
const relPath = normalizeResourcePath(rootDir, rawResource)
// 1A. Locked Architectural Contract Immutability Guard
if (agent && agent !== "architect") {
for (const contract of board.contracts) {
const hitsContract = contract.files.some((cf) => matchesPathPattern(relPath, cf))
if (hitsContract) {
return {
allowed: false,
reason: `SwarmOS Contract Guard: File "${relPath}" is locked under architectural contract [${contract.id}] ("${contract.title}"). Only the "architect" role may modify locked contract files.`,
}
}
}
}
// 1B. Disjoint Task Ownership (`ownedPaths`) Guard for Implementation Engineers
if (agent === "staff-backend" || agent === "staff-ui") {
const activeTasks = board.tasks.filter(
(t) =>
t.status === "pending" ||
t.status === "in_progress" ||
t.status === "ready_for_qa" ||
t.status === "qa_rejected",
)
const ownedBySelf = activeTasks.some(
(t) => t.owner === agent && t.ownedPaths.some((pattern) => matchesPathPattern(relPath, pattern)),
)
if (!ownedBySelf) {
const conflictingTask = activeTasks.find(
(t) => t.owner !== agent && t.ownedPaths.some((pattern) => matchesPathPattern(relPath, pattern)),
)
if (conflictingTask) {
return {
allowed: false,
reason: `SwarmOS Ownership Guard: File "${relPath}" is claimed by active task [${conflictingTask.id}] assigned to "${conflictingTask.owner}". Role "${agent}" cannot modify files outside its disjoint ownedPaths boundary.`,
}
}
}
}
const decision = evaluateFileMutation(rootDir, board, agent, rawResource)
if (!decision.allowed) return decision
}
}
// 2. Shell Command Guards
if (action === "shell" || action === "bash") {
let ask: GuardDecision | undefined
let autoAllow = resources.length > 0
for (const cmd of resources) {
if (typeof cmd !== "string") continue
if (DESTRUCTIVE_GIT_RE.test(cmd)) {
return {
allowed: false,
reason: `SwarmOS Git Safety Guard: Destructive git command "${cmd.slice(0, 80)}" is blocked across all SwarmOS roles.`,
}
}
if ((agent === "director" || agent === "manager") && LEADERSHIP_MUTATING_SHELL_RE.test(cmd)) {
return {
allowed: false,
reason: `SwarmOS Separation-of-Powers Guard: Leadership role "${agent}" is read-only and cannot execute file-mutating or git-committing shell commands ("${cmd.slice(0, 80)}"). Delegate implementation to subagents.`,
}
}
if (agent === "qa-adversary" && QA_SOURCE_MUTATING_SHELL_RE.test(cmd)) {
return {
allowed: false,
reason: `SwarmOS Red-Team Guard: "qa-adversary" is forbidden from mutating production source files or git history via shell ("${cmd.slice(0, 80)}").`,
}
}
const decision = evaluateShellCommand(rootDir, board, agent, cmd)
if (!decision.allowed) return decision
if (decision.ask && !ask) ask = decision
if (!decision.autoAllow) autoAllow = false
}
if (ask) return ask
if (autoAllow) return { allowed: true, autoAllow: true }
}
return { allowed: true }
@@ -184,24 +285,41 @@ export function evaluateBoardGuard(
export async function evaluatePermissionGuard(
rootDir: string,
event: PermissionEvaluationEvent,
evaluators: GuardEvaluator[] = [],
): Promise<GuardDecision> {
const board = await loadBoard(rootDir)
return evaluateBoardGuard(rootDir, board, event)
const base = evaluateBoardGuard(rootDir, board, event)
if (!base.allowed) return base
let result = base
for (const evaluate of evaluators) {
const extra = await evaluate(event, board)
if (!extra) continue
if (!extra.allowed) return extra
if (extra.ask) result = { allowed: true, ask: true, reason: extra.reason }
}
return result
}
/**
* Registers the SwarmOS runtime permission guard on OpenCode V2's `permission.hook("evaluate", ...)`.
* Setting `event.effect = "deny"` and `event.message` produces a clean, typed `Permission.BlockedError`
* that OpenCode V2 tools surface directly to the model as a `ToolFailure`.
* `deny` produces a typed `Permission.BlockedError` the model sees as a `ToolFailure`; `ask`
* downgrades a saved approval to a prompt; read-only commands from read-only roles skip the prompt.
*/
export async function registerPermissionGuard(ctx: any, rootDir: string): Promise<void> {
export async function registerPermissionGuard(ctx: any, rootDir: string, evaluators: GuardEvaluator[] = []): Promise<void> {
if (ctx.permission?.hook) {
await ctx.permission.hook("evaluate", async (event: PermissionEvaluationEvent) => {
if (event.effect === "deny") return
const decision = await evaluatePermissionGuard(rootDir, event)
const decision = await evaluatePermissionGuard(rootDir, event, evaluators)
if (!decision.allowed) {
event.effect = "deny"
event.message = decision.reason
} else if (decision.ask) {
if (event.effect === "allow") {
event.effect = "ask"
event.message = decision.reason
}
} else if (decision.autoAllow && event.effect === "ask") {
event.effect = "allow"
}
})
}
+5 -5
View File
@@ -50,7 +50,7 @@ export const PROMPT_MANAGER = `You are the **Engineering Manager & Tech Lead** (
- Break milestones into granular tasks in \`tools.org.create_task\` with explicit file ownership boundaries (\`ownedPaths\`) so parallel engineers never collide or overwrite each other's work.
2. **Parallel Dispatch & Best-of-N Tournaments**:
- Launch independent \`staff-backend\` and \`staff-ui\` tasks in parallel (\`subagent\` calls in the same step, or in isolated Git worktrees via \`tools.opencode.worktree_create\`).
- On high-complexity algorithmic or UI tasks, spawn 2 competing subagents in separate worktrees (Best-of-2 Tournament) and select the implementation that passes \`qa-adversary\` with zero defects.
- On high-complexity algorithmic or UI tasks, run a Best-of-N tournament: \`tools.org.plan_tournament\` creates a real git worktree per candidate (\`.worktrees/<taskId>\`). Tell each candidate subagent its worktree path; the guard blocks edits to contested files in the main tree. Have \`qa-adversary\` verify each candidate with \`tools.verify.run({ dir: "<worktree>" })\`, rank them with \`tools.org.score_tournament\`, then \`tools.org.resolve_tournament\` merges the winner and removes the other worktrees.
3. **Zero Unverified Merges**:
- NEVER accept an engineer's self-reported success at face value.
- After \`staff-backend\` or \`staff-ui\` completes a task, you MUST spawn \`qa-adversary\` to audit the diff, run the test suite, and (for UI work) run the live browser DOM geometry audit.
@@ -88,19 +88,19 @@ export const PROMPT_QA_ADVERSARY = `You are the **Adversarial QA, E2E & Security
# Verification Checklist
1. **Contract & Diff Audit**: Run \`git status --short\` and \`git diff\` (or read modified files). Verify zero stubbed functions, zero hardcoded fake data, zero swallowed errors, and 100% adherence to locked contracts.
2. **Automated Test Execution**: Run the package typecheck, linter, and unit/integration test suite via \`shell\`. Write additional edge-case tests (empty inputs, malformed payloads, concurrent calls, boundary values) to test resilience.
2. **Automated Test Execution**: Call \`execute\` -> \`tools.verify.run({})\` (typecheck, lint, and the tests that import the changed files; use \`scope: "full"\` for cross-cutting changes). For tournament candidates, pass the candidate's worktree: \`tools.verify.run({ dir: ".worktrees/<taskId>" })\`. Write additional edge-case tests (empty inputs, malformed payloads, concurrent calls, boundary values) to test resilience.
3. **Live Browser & UI Audit (for Frontend/Full-Stack Tasks)**:
- Inspect \`tools.browser.console\` (must have 0 uncaught errors) and \`tools.browser.network.list\` (must have 0 unexpected 4xx/5xx failures).
- Run the DOM Geometry Auditor script via \`tools.browser.evaluate\` and inspect \`tools.browser.screenshot\`.
4. **Verdict**:
- Call \`execute\` -> \`tools.org.submit_qa_receipt({ taskId, verdict: "PASS" | "REJECT", defects, testCommand, summary })\`.
- Issue \`PASS\` ONLY when all checks, tests, and geometry audits succeed with zero critical defects.`
- Call \`execute\` -> \`tools.org.submit_qa_receipt({ taskId, verdict: "PASS" | "REJECT", defects, testCommand, summary, verifyRunId })\`.
- Issue \`PASS\` ONLY when all checks, tests, and geometry audits succeed with zero critical defects. A PASS must cite the \`id\` of a passing \`tools.verify.run\` taken after the last edit; the board rejects PASS receipts without one.`
export const PROMPT_RELEASE_OPS = `You are the **Staff Release & Integration Engineer** (\`release-ops\`) at OpenCode SwarmOS.
# Your Role & Release Verification
- Audit \`tools.org.get_board\` to verify that all required tasks have a verified \`PASS\` QA receipt.
- If work was performed across multiple Git worktrees (\`tools.opencode.worktree_list\`), integrate/merge them cleanly, resolve any cross-module conflicts, and run the canonical full-repository verification suite (e.g., \`bun test\`, \`bun run check\`, \`npm test\`, \`cargo test\`).
- If work was performed across multiple Git worktrees (\`tools.opencode.worktree_list\`), integrate/merge them cleanly, resolve any cross-module conflicts, and run the canonical full-repository verification via \`tools.verify.run({ scope: "full" })\`. \`tools.org.verify_release_ready\` blocks release until a full run passes on the current tree.
- Produce a clean release readiness report detailing verified features, test coverage, and startup instructions.`
+730
View File
@@ -0,0 +1,730 @@
/**
* A small POSIX-shell parser for permission decisions. It does not execute or expand anything;
* it splits a command line into simple commands (following `;`, `&&`, `||`, `|`, `&`, newlines,
* subshells, `$(...)`, backticks, process substitution, `bash -c`, `eval` and common wrappers),
* records redirections, and classifies what each command can do.
*/
export type ShellEffect = "read" | "verify" | "write" | "network" | "destructive" | "exec"
export interface Redirect {
op: string
target: string
}
export interface SimpleCommand {
argv: string[]
redirects: Redirect[]
}
export interface CommandAnalysis {
program: string
argv: string[]
effects: ShellEffect[]
/** File paths this command may create, modify or delete (when knowable). */
writes: string[]
note?: string
}
export interface ShellAnalysis {
commands: CommandAnalysis[]
effects: ShellEffect[]
writes: string[]
/** True when parsing failed (unbalanced quotes etc.); callers should treat the command as unknown. */
unparseable: boolean
}
interface ParseResult {
commands: SimpleCommand[]
nested: string[]
ok: boolean
}
const OPERATOR_CHARS = new Set([";", "&", "|", "\n", "(", ")"])
/** Finds the index of the `)` closing a `$(` / `<(` opened just before `start`, honoring quotes. */
function findClosingParen(src: string, start: number): number {
let depth = 1
for (let i = start; i < src.length; i++) {
const ch = src[i]
if (ch === "\\") {
i++
} else if (ch === "'") {
const end = src.indexOf("'", i + 1)
if (end < 0) return -1
i = end
} else if (ch === '"') {
i++
while (i < src.length && src[i] !== '"') {
if (src[i] === "\\") i++
i++
}
if (i >= src.length) return -1
} else if (ch === "(") {
depth++
} else if (ch === ")") {
depth--
if (depth === 0) return i
}
}
return -1
}
export function parseShell(src: string): ParseResult {
const commands: SimpleCommand[] = []
const nested: string[] = []
let argv: string[] = []
let redirects: Redirect[] = []
let word = ""
let inWord = false
let pendingRedirect: string | null = null
const pendingHeredocs: string[] = []
let ok = true
const endWord = () => {
if (!inWord) return
if (pendingRedirect) {
if (pendingRedirect.startsWith("<<") && !pendingRedirect.startsWith("<<<")) {
pendingHeredocs.push(word)
} else {
redirects.push({ op: pendingRedirect, target: word })
}
pendingRedirect = null
} else {
argv.push(word)
}
word = ""
inWord = false
}
const endCommand = () => {
endWord()
if (argv.length || redirects.length) commands.push({ argv, redirects })
argv = []
redirects = []
}
let i = 0
while (i < src.length) {
const ch = src[i]
if (ch === "\\") {
if (src[i + 1] === "\n") {
i += 2
continue
}
word += src[i + 1] ?? ""
inWord = true
i += 2
continue
}
if (ch === "'") {
const end = src.indexOf("'", i + 1)
if (end < 0) {
ok = false
word += src.slice(i + 1)
inWord = true
break
}
word += src.slice(i + 1, end)
inWord = true
i = end + 1
continue
}
if (ch === '"') {
i++
inWord = true
let closed = false
while (i < src.length) {
const c = src[i]
if (c === "\\" && i + 1 < src.length && '"\\$`\n'.includes(src[i + 1])) {
word += src[i + 1]
i += 2
} else if (c === '"') {
closed = true
i++
break
} else if (c === "$" && src[i + 1] === "(" && src[i + 2] !== "(") {
const end = findClosingParen(src, i + 2)
if (end < 0) {
ok = false
i = src.length
break
}
nested.push(src.slice(i + 2, end))
word += "$(...)"
i = end + 1
} else if (c === "`") {
const end = src.indexOf("`", i + 1)
if (end < 0) {
ok = false
i = src.length
break
}
nested.push(src.slice(i + 1, end))
word += "`...`"
i = end + 1
} else {
word += c
i++
}
}
if (!closed) ok = false
continue
}
if (ch === "$" && src[i + 1] === "(" && src[i + 2] === "(") {
// Arithmetic expansion: keep as an opaque word.
const end = src.indexOf("))", i + 3)
if (end < 0) {
ok = false
break
}
word += src.slice(i, end + 2)
inWord = true
i = end + 2
continue
}
if ((ch === "$" || ch === "<" || ch === ">") && src[i + 1] === "(") {
if (ch !== "$" && inWord) {
// e.g. "a>(" is unusual; fall through to operator handling below.
} else {
const end = findClosingParen(src, i + 2)
if (end < 0) {
ok = false
break
}
nested.push(src.slice(i + 2, end))
word += ch === "$" ? "$(...)" : `${ch}(...)`
inWord = true
i = end + 1
continue
}
}
if (ch === "`") {
const end = src.indexOf("`", i + 1)
if (end < 0) {
ok = false
break
}
nested.push(src.slice(i + 1, end))
word += "`...`"
inWord = true
i = end + 1
continue
}
if (ch === "#" && !inWord) {
const nl = src.indexOf("\n", i)
i = nl < 0 ? src.length : nl
continue
}
// Redirection operators, optionally prefixed by an fd number already collected in `word`.
if (ch === ">" || ch === "<" || (ch === "&" && src[i + 1] === ">")) {
let fdPrefix = ""
if (inWord && /^\d+$/.test(word)) {
fdPrefix = word
word = ""
inWord = false
} else {
endWord()
}
let op = ch
let j = i + 1
if (ch === "&") {
op = "&>"
j = i + 2
if (src[j] === ">") {
op = "&>>"
j++
}
} else if (ch === ">" && (src[j] === ">" || src[j] === "|")) {
op += src[j]
j++
} else if (ch === "<" && src[j] === "<") {
op = "<<"
j++
if (src[j] === "<") {
op = "<<<"
j++
} else if (src[j] === "-") {
op = "<<-"
j++
}
} else if (ch === "<" && src[j] === ">") {
op = "<>"
j++
}
// fd duplication: >&2, 2>&1, <&0
if ((op === ">" || op === "<") && src[j] === "&") {
let k = j + 1
while (k < src.length && /[0-9-]/.test(src[k])) k++
redirects.push({ op: `${fdPrefix}${op}&`, target: src.slice(j + 1, k) })
i = k
continue
}
pendingRedirect = `${fdPrefix}${op}`
i = j
continue
}
if (OPERATOR_CHARS.has(ch)) {
endCommand()
if (ch === "\n" && pendingHeredocs.length > 0) {
// Skip heredoc bodies so their lines are not parsed as commands.
let pos = i + 1
while (pendingHeredocs.length > 0) {
const delim = pendingHeredocs.shift()!
let found = false
while (pos < src.length) {
const nl = src.indexOf("\n", pos)
const line = src.slice(pos, nl < 0 ? src.length : nl)
pos = nl < 0 ? src.length : nl + 1
if (line.replace(/^\t+/, "") === delim) {
found = true
break
}
}
if (!found) ok = false
}
i = pos
continue
}
i++
continue
}
if (ch === " " || ch === "\t" || ch === "\r") {
endWord()
i++
continue
}
word += ch
inWord = true
i++
}
endCommand()
if (pendingRedirect) ok = false
return { commands, nested, ok }
}
const READ_ONLY = new Set([
"ls", "cat", "head", "tail", "wc", "grep", "egrep", "fgrep", "rg", "ag", "pwd", "which", "whereis", "type",
"echo", "printf", "stat", "file", "tree", "du", "df", "printenv", "cut", "tr", "diff", "cmp", "comm",
"sha1sum", "sha256sum", "sha512sum", "md5sum", "shasum", "date", "uname", "whoami", "id", "true", "false",
"test", "[", "basename", "dirname", "realpath", "readlink", "nl", "column", "jq", "less", "more", "hostname",
"ps", "lsof", "xxd", "od", "hexdump", "strings", "fd", "fdfind", "sleep", "seq", "yes", "cd", "pushd", "popd",
"uniq", "sort", "find", "sed", "awk", "gawk", "mawk", "ast-grep", "sg", "env", "tac", "rev", "fold", "expr",
"locale", "getconf", "nproc", "free", "uptime", "ldd", "man", "help",
])
const WRITE_PROGRAMS = new Set([
"rm", "rmdir", "mv", "cp", "touch", "mkdir", "ln", "chmod", "chown", "chgrp", "truncate", "dd", "tee",
"install", "unzip", "tar", "patch", "shred", "unlink", "mkfifo", "split", "csplit", "gzip", "gunzip", "zip",
])
const NETWORK_PROGRAMS = new Set(["curl", "wget", "ssh", "scp", "sftp", "nc", "ncat", "netcat", "telnet", "ftp", "gh", "rsync"])
const SHELLS = new Set(["bash", "sh", "zsh", "dash", "ksh", "fish"])
const INTERPRETERS = new Set(["python", "python3", "python2", "node", "bun", "deno", "ruby", "perl", "php", "osascript", "pwsh", "powershell"])
const INLINE_FLAGS = new Set(["-c", "-e", "--eval", "-p", "--print", "-E", "-r"])
const VERIFY_SCRIPT_RE = /^(?:test|tests|lint|typecheck|type-check|check|check-types|build|format:check|fmt:check|ci)(?::.*)?$/
/** Strips wrappers like `sudo`, `env X=1`, `timeout 5`, `nice -n 5`, `xargs -0` and leading assignments. */
function unwrap(argv: string[]): { argv: string[]; viaXargs: boolean } {
let a = [...argv]
let viaXargs = false
for (let guard = 0; guard < 10 && a.length > 0; guard++) {
while (a.length && /^[A-Za-z_][A-Za-z0-9_]*=/.test(a[0])) a.shift()
const p = a[0]
if (!p) break
if (p === "sudo" || p === "doas") {
a.shift()
while (a.length && a[0].startsWith("-")) {
const flag = a.shift()!
if (/^-[ugCh]$/.test(flag)) a.shift()
}
} else if (p === "env") {
a.shift()
while (a.length && (a[0].startsWith("-") || /^[A-Za-z_][A-Za-z0-9_]*=/.test(a[0]))) {
const flag = a.shift()!
if (flag === "-u" || flag === "-C" || flag === "-S") a.shift()
}
if (a.length === 0) return { argv: ["env"], viaXargs }
} else if (p === "timeout") {
a.shift()
while (a.length && a[0].startsWith("-")) {
const flag = a.shift()!
if (flag === "-s" || flag === "-k" || flag === "--signal" || flag === "--kill-after") a.shift()
}
a.shift() // duration
} else if (p === "nice" || p === "ionice") {
a.shift()
while (a.length && a[0].startsWith("-")) {
const flag = a.shift()!
if (/^-[ncp]$/.test(flag)) a.shift()
}
} else if (p === "nohup" || p === "command" || p === "exec" || p === "time" || p === "builtin" || p === "stdbuf" || p === "unbuffer") {
a.shift()
while (a.length && a[0].startsWith("-")) a.shift()
} else if (p === "xargs") {
viaXargs = true
a.shift()
while (a.length && a[0].startsWith("-")) {
const flag = a.shift()!
if (/^-[IdEsLnPa]$/.test(flag)) a.shift()
}
if (a.length === 0) return { argv: ["echo"], viaXargs }
} else {
break
}
}
return { argv: a, viaXargs }
}
function nonFlagArgs(args: string[]): string[] {
const out: string[] = []
let endOfFlags = false
for (const a of args) {
if (!endOfFlags && a === "--") {
endOfFlags = true
continue
}
if (!endOfFlags && a.startsWith("-") && a !== "-") continue
out.push(a)
}
return out
}
function hasFlag(args: string[], ...flags: string[]): boolean {
return args.some((a) => flags.includes(a) || flags.some((f) => /^-[a-zA-Z]$/.test(f) && /^-[a-zA-Z]+$/.test(a) && a.includes(f[1])))
}
function classifyGit(args: string[]): { effects: ShellEffect[]; writes: string[]; note?: string } {
// Skip global options like -C <dir>, -c k=v, --no-pager.
let i = 0
while (i < args.length && args[i].startsWith("-")) {
if (args[i] === "-C" || args[i] === "-c") i++
i++
}
const sub = args[i] ?? ""
const rest = args.slice(i + 1)
const read = { effects: ["read"] as ShellEffect[], writes: [] as string[] }
const write = { effects: ["write"] as ShellEffect[], writes: [] as string[] }
switch (sub) {
case "":
case "status":
case "diff":
case "log":
case "show":
case "blame":
case "rev-parse":
case "ls-files":
case "ls-tree":
case "grep":
case "describe":
case "shortlog":
case "cat-file":
case "rev-list":
case "merge-base":
case "name-rev":
case "whatchanged":
case "help":
case "version":
case "--version":
return read
case "reflog":
return rest[0] === "expire" || rest[0] === "delete" ? { effects: ["destructive"], writes: [], note: "rewrites reflog" } : read
case "config":
return hasFlag(rest, "--get", "--get-all", "--list", "-l", "--get-regexp") || nonFlagArgs(rest).length <= 1 ? read : write
case "remote":
return rest.length === 0 || hasFlag(rest, "-v", "--verbose") || rest[0] === "show" || rest[0] === "get-url" ? read : write
case "branch":
if (hasFlag(rest, "-D") || (hasFlag(rest, "-d", "--delete") && hasFlag(rest, "-f", "--force"))) {
return { effects: ["destructive"], writes: [], note: "force-deletes a branch" }
}
return nonFlagArgs(rest).length === 0 || hasFlag(rest, "--list", "-l", "--show-current", "-a", "-r", "-v", "-vv", "--contains", "--merged")
? read
: write
case "tag":
return nonFlagArgs(rest).length === 0 || hasFlag(rest, "-l", "--list", "-n") ? read : write
case "stash":
if (rest[0] === "list" || rest[0] === "show") return read
if (rest[0] === "drop" || rest[0] === "clear") return { effects: ["destructive"], writes: [], note: "discards stashed work" }
return write
case "worktree":
if (rest[0] === "list") return read
if (rest[0] === "remove" && hasFlag(rest, "-f", "--force")) return { effects: ["destructive"], writes: [], note: "force-removes a worktree" }
return write
case "reset":
return hasFlag(rest, "--hard", "--merge", "--keep") ? { effects: ["destructive"], writes: [], note: "discards working-tree changes" } : write
case "clean":
return hasFlag(rest, "-f", "--force") ? { effects: ["destructive"], writes: [], note: "deletes untracked files" } : read
case "checkout": {
const dashdash = rest.indexOf("--")
if (dashdash >= 0 || rest.includes(".") || hasFlag(rest, "-f", "--force")) {
return { effects: ["destructive"], writes: [], note: "overwrites working-tree files" }
}
return write
}
case "restore":
return hasFlag(rest, "--staged", "-S") && !hasFlag(rest, "--worktree", "-W")
? write
: { effects: ["destructive"], writes: [], note: "overwrites working-tree files" }
case "push":
if (rest.some((a) => a === "--force" || a === "-f" || /^-[a-zA-Z]*f/.test(a) || a === "--mirror" || a === "--delete" || a.startsWith("+"))) {
return { effects: ["destructive", "network"], writes: [], note: "force-push or remote deletion" }
}
return { effects: ["write", "network"], writes: [] }
case "pull":
case "fetch":
case "clone":
case "submodule":
case "ls-remote":
return { effects: sub === "ls-remote" ? ["network"] : ["write", "network"], writes: [] }
case "gc":
case "prune":
return hasFlag(rest, "--prune=now", "--aggressive") || sub === "prune"
? { effects: ["destructive"], writes: [], note: "prunes objects" }
: write
case "rm":
case "mv":
return { effects: ["write"], writes: nonFlagArgs(rest) }
default:
// add, commit, merge, rebase, cherry-pick, am, apply, revert, switch, init, notes, bisect, ...
return write
}
}
function classifyPackageManager(program: string, args: string[]): { effects: ShellEffect[]; writes: string[] } {
const sub = args[0] ?? ""
if (program === "npx" || program === "bunx" || (program === "pnpm" && sub === "dlx") || (program === "yarn" && sub === "dlx")) {
return { effects: ["exec", "network"], writes: [] }
}
if (sub === "test" || sub === "t") return { effects: ["verify"], writes: [] }
if (sub === "run" || sub === "run-script") {
return VERIFY_SCRIPT_RE.test(args[1] ?? "") ? { effects: ["verify"], writes: [] } : { effects: ["exec"], writes: [] }
}
if (program === "bun" && sub === "test") return { effects: ["verify"], writes: [] }
if (["install", "i", "add", "remove", "rm", "uninstall", "update", "upgrade", "ci", "link"].includes(sub)) {
return { effects: ["write", "network"], writes: [] }
}
if (sub === "publish" || sub === "deploy") return { effects: ["network", "write"], writes: [] }
if (["ls", "list", "outdated", "why", "view", "info", "--version", "-v", "help"].includes(sub)) {
return { effects: sub === "view" || sub === "info" || sub === "outdated" ? ["network"] : ["read"], writes: [] }
}
if (program === "bun" || program === "node" || program === "deno") return { effects: ["exec"], writes: [] }
return { effects: ["exec"], writes: [] }
}
function classify(argv: string[], depth: number): { effects: ShellEffect[]; writes: string[]; note?: string; nested?: string } {
const program = (argv[0] ?? "").split("/").pop() ?? ""
const args = argv.slice(1)
if (SHELLS.has(program)) {
const cIdx = args.findIndex((a) => a === "-c" || (/^-[a-zA-Z]*c$/.test(a) && a.startsWith("-")))
if (cIdx >= 0 && args[cIdx + 1] !== undefined) return { effects: [], writes: [], nested: args[cIdx + 1] }
return { effects: ["exec"], writes: [], note: `runs a ${program} script` }
}
if (program === "eval") return { effects: [], writes: [], nested: args.join(" ") }
if (program === "source" || program === ".") return { effects: ["exec"], writes: [], note: "sources a script" }
if (program === "git") return classifyGit(args)
if (["npm", "pnpm", "yarn", "npx", "bunx", "bun", "deno"].includes(program)) {
if (program === "bun" && INLINE_FLAGS.has(args[0] ?? "")) return { effects: ["exec"], writes: [], note: "inline bun script" }
return classifyPackageManager(program, args)
}
if (INTERPRETERS.has(program)) {
if (args.some((a) => INLINE_FLAGS.has(a) || /^-[a-zA-Z]*[ce]$/.test(a))) {
if (program === "perl" && args.some((a) => /^-[a-zA-Z]*i/.test(a))) {
return { effects: ["write", "exec"], writes: nonFlagArgs(args).slice(1), note: "perl in-place edit" }
}
return { effects: ["exec"], writes: [], note: `inline ${program} code` }
}
if ((program === "python" || program === "python3") && args[0] === "-m") {
const mod = args[1] ?? ""
if (["pytest", "unittest", "mypy", "py_compile", "compileall", "doctest", "pyflakes", "pylint"].includes(mod)) {
return { effects: ["verify"], writes: [] }
}
if (mod === "pip") return { effects: ["write", "network"], writes: [] }
if (mod === "http.server" || mod === "json.tool") return { effects: ["read"], writes: [] }
return { effects: ["exec"], writes: [] }
}
if (program === "node" && (args[0] === "--check" || args[0] === "-c")) return { effects: ["verify"], writes: [] }
if (program === "node" && args[0] === "--test") return { effects: ["verify"], writes: [] }
if (args.length === 0 || args[0] === "--version" || args[0] === "-V") return { effects: ["read"], writes: [] }
return { effects: ["exec"], writes: [], note: `runs a ${program} script` }
}
if (["tsc", "vitest", "jest", "pytest", "mypy", "oxlint", "biome", "eslint", "ruff", "prettier", "tsgo", "pyright", "go", "cargo", "make", "gradle", "mvn"].includes(program)) {
if (program === "eslint" && hasFlag(args, "--fix")) return { effects: ["write"], writes: [] }
if (program === "ruff" && (args[0] === "format" || hasFlag(args, "--fix"))) return { effects: ["write"], writes: [] }
if (program === "prettier") return hasFlag(args, "--write", "-w") ? { effects: ["write"], writes: [] } : { effects: ["verify"], writes: [] }
if (program === "biome" && hasFlag(args, "--write", "--apply")) return { effects: ["write"], writes: [] }
if (program === "go") {
const sub = args[0] ?? ""
if (["test", "vet", "build", "list", "version", "env", "doc"].includes(sub)) return { effects: ["verify"], writes: [] }
if (sub === "fmt") return { effects: ["write"], writes: [] }
return { effects: ["write", "network"], writes: [] }
}
if (program === "cargo") {
const sub = args[0] ?? ""
if (["test", "check", "clippy", "build", "doc", "tree", "metadata", "--version"].includes(sub)) return { effects: ["verify"], writes: [] }
if (sub === "fmt") return hasFlag(args, "--check") ? { effects: ["verify"], writes: [] } : { effects: ["write"], writes: [] }
return { effects: ["write", "network"], writes: [] }
}
if (program === "make" || program === "gradle" || program === "mvn") {
return /^(test|check|lint|build|verify)$/.test(args[0] ?? "") ? { effects: ["verify"], writes: [] } : { effects: ["exec"], writes: [] }
}
return { effects: ["verify"], writes: [] }
}
if (NETWORK_PROGRAMS.has(program)) {
const writes: string[] = []
if (program === "curl") {
const o = args.findIndex((a) => a === "-o" || a === "--output")
if (o >= 0 && args[o + 1]) writes.push(args[o + 1])
if (hasFlag(args, "-O", "--remote-name")) writes.push("<remote-name>")
}
if (program === "wget") {
const o = args.findIndex((a) => a === "-O" || a === "--output-document")
writes.push(o >= 0 && args[o + 1] ? args[o + 1] : "<remote-name>")
}
if (program === "rsync" || program === "scp") {
const files = nonFlagArgs(args)
const dest = files[files.length - 1]
if (dest && !dest.includes(":")) writes.push(dest)
}
return { effects: writes.length ? ["network", "write"] : ["network"], writes }
}
if (WRITE_PROGRAMS.has(program)) {
const files = nonFlagArgs(args)
let writes = files
if (program === "cp" || program === "install" || program === "ln") writes = files.slice(-1)
if (program === "dd") writes = args.filter((a) => a.startsWith("of=")).map((a) => a.slice(3))
if (program === "tar") {
const f = args.findIndex((a) => a === "-f" || /^-?[a-zA-Z]*f$/.test(a))
const extracting = args.some((a) => /^-?[a-zA-Z]*x/.test(a) || a === "--extract")
writes = extracting ? ["<archive contents>"] : f >= 0 && args[f + 1] ? [args[f + 1]] : []
if (!extracting && !args.some((a) => /^-?[a-zA-Z]*c/.test(a) || a === "--create")) return { effects: ["read"], writes: [] }
}
if (program === "unzip") writes = ["<archive contents>"]
if (program === "patch") writes = ["<patched files>"]
if (program === "rm" && hasFlag(args, "-r", "-R", "--recursive") && files.some((f) => f === "/" || f === "~" || f === "*" || f === "." || f === "..")) {
return { effects: ["destructive"], writes: files, note: "recursive delete of a broad path" }
}
return { effects: ["write"], writes }
}
if (READ_ONLY.has(program)) {
if (program === "sed" && args.some((a) => a === "--in-place" || a.startsWith("--in-place=") || /^-[a-zA-Z]*i/.test(a))) {
const operands: string[] = []
let scriptViaFlag = false
for (let k = 0; k < args.length; k++) {
const x = args[k]
if (x === "-e" || x === "-f" || x === "--expression" || x === "--file") {
scriptViaFlag = true
k++
} else if (x.startsWith("--expression=") || x.startsWith("--file=")) {
scriptViaFlag = true
} else if (!x.startsWith("-") && x !== "") {
operands.push(x)
}
}
return { effects: ["write"], writes: scriptViaFlag ? operands : operands.slice(1), note: "sed in-place edit" }
}
if (program === "sed" && args.some((a) => /(^|;|\s)w\s+\S|(^|;)e(\s|$)/.test(a))) {
return { effects: ["write", "exec"], writes: [], note: "sed script writes files or runs commands" }
}
if ((program === "awk" || program === "gawk" || program === "mawk") && (args.some((a) => /\bsystem\s*\(|print[^;]*>|\|\s*"/.test(a)) || hasFlag(args, "-i"))) {
return { effects: ["write", "exec"], writes: [], note: "awk program writes files or runs commands" }
}
if (program === "find" && args.some((a) => ["-delete", "-exec", "-execdir", "-ok", "-okdir", "-fprint", "-fprintf", "-fls", "-fprint0"].includes(a))) {
return args.includes("-delete") ? { effects: ["write"], writes: ["<find matches>"] } : { effects: ["exec"], writes: [], note: "find -exec" }
}
if (program === "sort" && args.some((a) => a === "-o" || a.startsWith("--output"))) {
const o = args.indexOf("-o")
return { effects: ["write"], writes: o >= 0 && args[o + 1] ? [args[o + 1]] : [] }
}
if ((program === "ast-grep" || program === "sg") && hasFlag(args, "-U", "--update-all", "--interactive", "-i")) {
return { effects: ["write"], writes: [] }
}
return { effects: ["read"], writes: [] }
}
if (depth > 0 && program === "") return { effects: ["read"], writes: [] }
return { effects: ["exec"], writes: [], note: program ? `unknown program "${program}"` : undefined }
}
const SAFE_REDIRECT_TARGETS = new Set(["/dev/null", "/dev/stdout", "/dev/stderr", "/dev/tty"])
export function analyzeShell(src: string, depth = 0): ShellAnalysis {
const parsed = parseShell(src)
const commands: CommandAnalysis[] = []
let unparseable = !parsed.ok
for (const cmd of parsed.commands) {
const { argv, viaXargs } = unwrap(cmd.argv)
const result = argv.length ? classify(argv, depth) : { effects: ["read"] as ShellEffect[], writes: [] as string[] }
const effects = new Set<ShellEffect>(result.effects)
const writes = [...result.writes]
if ("nested" in result && result.nested !== undefined) {
if (depth >= 4) {
effects.add("exec")
} else {
const inner = analyzeShell(result.nested, depth + 1)
commands.push(...inner.commands)
if (inner.unparseable) unparseable = true
}
}
if (viaXargs && effects.has("write") && writes.length === 0) writes.push("<xargs input>")
for (const r of cmd.redirects) {
if (r.op.endsWith("&")) continue
if (r.op.includes(">") && !SAFE_REDIRECT_TARGETS.has(r.target)) {
effects.add("write")
writes.push(r.target)
}
}
if (effects.size === 0 && !("nested" in result && result.nested !== undefined)) effects.add("read")
if (effects.size > 0) {
commands.push({
program: (argv[0] ?? "").split("/").pop() ?? "",
argv,
effects: [...effects],
writes,
...("note" in result && result.note ? { note: result.note } : {}),
})
}
}
for (const n of parsed.nested) {
if (depth >= 4) {
unparseable = true
continue
}
const inner = analyzeShell(n, depth + 1)
commands.push(...inner.commands)
if (inner.unparseable) unparseable = true
}
const effects = [...new Set(commands.flatMap((c) => c.effects))]
return { commands, effects, writes: [...new Set(commands.flatMap((c) => c.writes))], unparseable }
}
export function isReadOnly(analysis: ShellAnalysis, allowVerify = false): boolean {
if (analysis.unparseable || analysis.commands.length === 0) return false
return analysis.effects.every((e) => e === "read" || (allowVerify && e === "verify"))
}
+224 -25
View File
@@ -1,6 +1,10 @@
import * as fs from "node:fs/promises"
import * as path from "node:path"
import { computeTreeFingerprint, loadLatestVerifyRun, loadVerifyRun } from "../omni/verify.js"
import { git, isGitRepo, mergeTournamentWinner, provisionWorktree, type MergeResult } from "./worktrees.js"
import { getSessionState } from "../host/session-state.js"
import type {
ClaimTaskInput,
Contract,
CreateTaskInput,
InitProjectInput,
@@ -10,9 +14,12 @@ import type {
ProjectCharter,
ReleaseReadinessReport,
ResolveTournamentInput,
SetTaskBudgetInput,
SubmitQAReceiptInput,
Task,
TaskBudget,
TaskStatus,
TaskUsage,
Tournament,
TournamentCandidate,
} from "./types.js"
@@ -42,6 +49,7 @@ export async function loadBoard(rootDir: string): Promise<OrgBoard> {
const parsed = JSON.parse(raw) as Partial<OrgBoard>
return {
version: parsed.version ?? BOARD_VERSION,
...(parsed.policy ? { policy: parsed.policy } : {}),
project: parsed.project ?? null,
contracts: Array.isArray(parsed.contracts) ? parsed.contracts : [],
tasks: Array.isArray(parsed.tasks) ? parsed.tasks : [],
@@ -104,6 +112,9 @@ export async function initProject(rootDir: string, input: InitProjectInput): Pro
updatedAt: new Date().toISOString(),
}
board.project = charter
if (!board.policy || input.requireVerifiedReceipts !== undefined) {
board.policy = { requireVerifiedReceipts: input.requireVerifiedReceipts ?? true }
}
return charter
})
}
@@ -140,6 +151,9 @@ export async function createTask(rootDir: string, input: CreateTaskInput): Promi
ownedPaths: input.ownedPaths,
worktree: input.worktree,
...(input.tournamentId ? { tournamentId: input.tournamentId } : {}),
...(existing?.claimedBySessionID ? { claimedBySessionID: existing.claimedBySessionID } : {}),
...(input.budget ?? existing?.budget ? { budget: input.budget ?? existing?.budget } : {}),
usage: existing?.usage ?? { tokensUsed: 0, turnsUsed: 0, toolCallsUsed: 0 },
status: existing?.status ?? "pending",
qaReceipts: existing?.qaReceipts ?? [],
createdAt: existing?.createdAt ?? now,
@@ -167,7 +181,110 @@ export async function updateTaskStatus(rootDir: string, taskId: string, status:
})
}
/**
* Binds a session ID to a board task (`tools.org.claim_task`) and sets the task to `in_progress`
* if it is currently `pending`, so spend limits and `/doctor` track the active task.
*/
export async function claimTask(rootDir: string, input: ClaimTaskInput): Promise<Task> {
const sid = input.sessionID ?? "default"
const state = getSessionState(sid)
state.taskId = input.taskId
return withBoardLock(rootDir, (board) => {
const task = board.tasks.find((t) => t.id === input.taskId)
if (!task) {
throw new Error(`SwarmOS Task not found: ${input.taskId}`)
}
task.claimedBySessionID = sid
if (task.status === "pending") {
task.status = "in_progress"
}
task.usage ??= { tokensUsed: 0, turnsUsed: 0, toolCallsUsed: 0 }
task.updatedAt = new Date().toISOString()
return task
})
}
/**
* Updates or raises a task's spend/step budget (`tools.org.set_task_budget`) and clears `overBudget`
* if the new limits accommodate current usage (or if `resetUsage: true` is passed).
*/
export async function setTaskBudget(rootDir: string, input: SetTaskBudgetInput): Promise<Task> {
return withBoardLock(rootDir, (board) => {
const task = board.tasks.find((t) => t.id === input.taskId)
if (!task) {
throw new Error(`SwarmOS Task not found: ${input.taskId}`)
}
task.budget = { ...task.budget, ...input.budget }
if (input.resetUsage || !task.usage) {
task.usage = { tokensUsed: 0, turnsUsed: 0, toolCallsUsed: 0 }
} else {
const reason = evaluateTaskBudgetViolation(task.budget, task.usage)
task.usage.overBudget = Boolean(reason)
if (reason) task.usage.overBudgetReason = reason
else delete task.usage.overBudgetReason
}
task.updatedAt = new Date().toISOString()
return task
})
}
export function evaluateTaskBudgetViolation(budget: TaskBudget | undefined, usage: TaskUsage | undefined): string | undefined {
if (!budget || !usage) return undefined
if (budget.maxTokens !== undefined && usage.tokensUsed > budget.maxTokens) {
return `tokensUsed (${usage.tokensUsed}) > maxTokens (${budget.maxTokens})`
}
if (budget.maxTurns !== undefined && usage.turnsUsed > budget.maxTurns) {
return `turnsUsed (${usage.turnsUsed}) > maxTurns (${budget.maxTurns})`
}
if (budget.maxToolCalls !== undefined && usage.toolCallsUsed > budget.maxToolCalls) {
return `toolCallsUsed (${usage.toolCallsUsed}) > maxToolCalls (${budget.maxToolCalls})`
}
return undefined
}
/** Directory a task's code lives in: its worktree when it has one, else the workspace root. */
export function taskWorkdir(rootDir: string, task: Pick<Task, "worktree">): string {
return task.worktree ? path.resolve(rootDir, task.worktree) : path.resolve(rootDir)
}
/**
* Checks that a PASS verdict is backed by a passing `tools.verify.run` taken on the task's
* current working tree. Returns the fingerprint to store on the receipt.
*/
async function assertVerifiedPass(rootDir: string, task: Task, verifyRunId: string | undefined): Promise<string | null> {
if (!verifyRunId) {
throw new Error(
`SwarmOS Verify Gate: PASS for task [${task.id}] must cite a passing \`tools.verify.run\` (pass \`verifyRunId\`). Run verification instead of asserting it.`,
)
}
const workdir = taskWorkdir(rootDir, task)
const run = (await loadVerifyRun(workdir, verifyRunId)) ?? (await loadVerifyRun(rootDir, verifyRunId))
if (!run) throw new Error(`SwarmOS Verify Gate: verify run [${verifyRunId}] not found for task [${task.id}].`)
if (!run.passed) {
throw new Error(`SwarmOS Verify Gate: verify run [${verifyRunId}] FAILED; a failing run cannot back a PASS receipt.`)
}
if (path.resolve(run.rootDir) !== workdir) {
throw new Error(`SwarmOS Verify Gate: verify run [${verifyRunId}] ran in ${run.rootDir}, but task [${task.id}] lives in ${workdir}.`)
}
const current = await computeTreeFingerprint(workdir)
if (current !== null && run.fingerprint !== current) {
throw new Error(
`SwarmOS Verify Gate: code changed since verify run [${verifyRunId}]. Re-run \`tools.verify.run\` on the current tree.`,
)
}
return current
}
export async function submitQAReceipt(rootDir: string, input: SubmitQAReceiptInput): Promise<Task> {
const snapshot = await loadBoard(rootDir)
const target = snapshot.tasks.find((t) => t.id === input.taskId)
let fingerprint: string | null | undefined
if (target && input.verdict === "PASS" && snapshot.policy?.requireVerifiedReceipts) {
fingerprint = await assertVerifiedPass(rootDir, target, input.verifyRunId)
}
return withBoardLock(rootDir, (board) => {
const task = board.tasks.find((t) => t.id === input.taskId)
if (!task) {
@@ -179,6 +296,8 @@ export async function submitQAReceipt(rootDir: string, input: SubmitQAReceiptInp
summary: input.summary,
defects: input.defects,
testCommand: input.testCommand,
...(input.verifyRunId ? { verifyRunId: input.verifyRunId } : {}),
...(fingerprint !== undefined ? { fingerprint } : {}),
timestamp: new Date().toISOString(),
}
@@ -190,9 +309,42 @@ export async function submitQAReceipt(rootDir: string, input: SubmitQAReceiptInp
}
/**
* Provisions a Best-of-N Worktree Tournament with isolated candidate tasks and worktree paths.
* Provisions a Best-of-N Worktree Tournament: candidate tasks on the board plus, in a git
* repository, a real `git worktree` per candidate on branch `swarmos/<taskId>`.
*/
export async function planTournament(rootDir: string, input: PlanTournamentInput): Promise<Tournament> {
const planned = await recordTournamentPlan(rootDir, input)
if (!(await isGitRepo(rootDir))) {
return markProvisioning(rootDir, planned.id, undefined, planned.candidates.map(() => ({ provisioned: false, reason: "not a git repository" })))
}
const baseRef = (await git(rootDir, ["rev-parse", "HEAD"])).stdout.trim() || "HEAD"
const results = []
for (const c of planned.candidates) {
results.push(await provisionWorktree(rootDir, c.taskId, c.worktree, baseRef))
}
return markProvisioning(rootDir, planned.id, baseRef, results)
}
async function markProvisioning(
rootDir: string,
tournamentId: string,
baseRef: string | undefined,
results: Array<{ provisioned: boolean; reason?: string; branch?: string }>,
): Promise<Tournament> {
return withBoardLock(rootDir, (board) => {
const tournament = board.tournaments!.find((t) => t.id === tournamentId)!
if (baseRef) tournament.baseRef = baseRef
tournament.candidates.forEach((c, i) => {
c.provisioned = results[i].provisioned
if (results[i].branch) c.branch = results[i].branch
if (results[i].reason) c.provisionNote = results[i].reason
else delete c.provisionNote
})
return tournament
})
}
async function recordTournamentPlan(rootDir: string, input: PlanTournamentInput): Promise<Tournament> {
return withBoardLock(rootDir, (board) => {
if (!Array.isArray(board.tournaments)) {
board.tournaments = []
@@ -256,30 +408,15 @@ export async function planTournament(rootDir: string, input: PlanTournamentInput
* and marking all competing candidate tasks as `superseded`.
*/
export async function resolveTournament(rootDir: string, input: ResolveTournamentInput): Promise<Tournament> {
const snapshot = await loadBoard(rootDir)
const planned = assertResolvable(snapshot, input)
let merge: MergeResult | undefined
if (input.merge !== false && planned.candidates.some((c) => c.provisioned)) {
merge = await mergeTournamentWinner(rootDir, planned, input.winningTaskId)
}
return withBoardLock(rootDir, (board) => {
if (!Array.isArray(board.tournaments)) {
board.tournaments = []
}
const tournament = board.tournaments.find((t) => t.id === input.tournamentId)
if (!tournament) {
throw new Error(`SwarmOS Tournament not found: ${input.tournamentId}`)
}
const isCandidate = tournament.candidates.some((c) => c.taskId === input.winningTaskId)
if (!isCandidate) {
throw new Error(
`SwarmOS Tournament Guard: Task "${input.winningTaskId}" is not a registered candidate of tournament "${input.tournamentId}".`,
)
}
const winningTask = board.tasks.find((t) => t.id === input.winningTaskId)
const latestReceipt = winningTask?.qaReceipts[winningTask.qaReceipts.length - 1]
if (!winningTask || winningTask.status !== "verified" || latestReceipt?.verdict !== "PASS") {
throw new Error(
`SwarmOS Tournament Guard: Winning candidate task [${input.winningTaskId}] must have a verified PASS QA receipt before resolving the tournament.`,
)
}
const tournament = assertResolvable(board, input)
const now = new Date().toISOString()
for (const candidate of tournament.candidates) {
if (candidate.taskId === input.winningTaskId) continue
@@ -294,11 +431,45 @@ export async function resolveTournament(rootDir: string, input: ResolveTournamen
tournament.winningTaskId = input.winningTaskId
tournament.resolutionSummary = input.summary
tournament.updatedAt = now
if (merge?.commit) {
tournament.mergeCommit = merge.commit
// The winner's code now lives in the main tree; its worktree is gone.
const winnerTask = board.tasks.find((t) => t.id === input.winningTaskId)
if (winnerTask) delete winnerTask.worktree
for (const c of tournament.candidates) c.provisioned = false
}
return tournament
})
}
function assertResolvable(board: OrgBoard, input: ResolveTournamentInput): Tournament {
const tournament = (board.tournaments ?? []).find((t) => t.id === input.tournamentId)
if (!tournament) {
throw new Error(`SwarmOS Tournament not found: ${input.tournamentId}`)
}
const isCandidate = tournament.candidates.some((c) => c.taskId === input.winningTaskId)
if (!isCandidate) {
throw new Error(
`SwarmOS Tournament Guard: Task "${input.winningTaskId}" is not a registered candidate of tournament "${input.tournamentId}".`,
)
}
const winningTask = board.tasks.find((t) => t.id === input.winningTaskId)
const latestReceipt = winningTask?.qaReceipts[winningTask.qaReceipts.length - 1]
if (!winningTask || winningTask.status !== "verified" || latestReceipt?.verdict !== "PASS") {
throw new Error(
`SwarmOS Tournament Guard: Winning candidate task [${input.winningTaskId}] must have a verified PASS QA receipt before resolving the tournament.`,
)
}
if (tournament.status === "resolved") {
throw new Error(`SwarmOS Tournament Guard: Tournament "${input.tournamentId}" is already resolved.`)
}
return tournament
}
/**
* Evaluates whether the entire project board satisfies the SwarmOS 5-Gate Release Protocol.
*/
@@ -349,6 +520,32 @@ export async function verifyReleaseReady(rootDir: string): Promise<ReleaseReadin
}
}
let latestFullVerifyRunId: string | undefined
let verifyBlocked = false
if (board.policy?.requireVerifiedReceipts) {
for (const id of verifiedTasks) {
const task = board.tasks.find((t) => t.id === id)!
if (!task.qaReceipts[task.qaReceipts.length - 1]?.verifyRunId) {
verifyBlocked = true
blockers.push(`Task [${id}] PASS receipt does not cite a \`tools.verify.run\` result.`)
}
}
const full = await loadLatestVerifyRun(rootDir, "full")
const fingerprint = await computeTreeFingerprint(rootDir)
if (!full) {
verifyBlocked = true
blockers.push("No full verification run: call `tools.verify.run({ scope: \"full\" })` before release.")
} else if (!full.passed) {
verifyBlocked = true
blockers.push(`Latest full verification run [${full.id}] FAILED.`)
} else if (fingerprint !== null && full.fingerprint !== fingerprint) {
verifyBlocked = true
blockers.push(`Code changed since full verification run [${full.id}]: re-run \`tools.verify.run({ scope: "full" })\`.`)
} else {
latestFullVerifyRunId = full.id
}
}
const tournaments = Array.isArray(board.tournaments) ? board.tournaments : []
const activeTournaments = tournaments.filter((t) => t.status === "active").map((t) => t.id)
for (const tid of activeTournaments) {
@@ -361,7 +558,8 @@ export async function verifyReleaseReady(rootDir: string): Promise<ReleaseReadin
activeTasks.length > 0 &&
unverifiedTasks.length === 0 &&
rejectedTasks.length === 0 &&
activeTournaments.length === 0
activeTournaments.length === 0 &&
!verifyBlocked
return {
ready,
@@ -373,6 +571,7 @@ export async function verifyReleaseReady(rootDir: string): Promise<ReleaseReadin
supersededTasks,
activeTournaments,
blockers,
...(latestFullVerifyRunId ? { latestFullVerifyRunId } : {}),
checkedAt: new Date().toISOString(),
}
}
+59
View File
@@ -34,9 +34,31 @@ export interface QAReceipt {
summary: string
defects: string[]
testCommand?: string
/** `tools.verify.run` run backing a PASS verdict. */
verifyRunId?: string
/** Working-tree fingerprint the verify run was taken at. */
fingerprint?: string | null
timestamp: string
}
export interface TaskBudget {
/** Maximum total tokens (input + output + cacheRead) allowed for this task. */
maxTokens?: number
/** Maximum LLM turns allowed for this task. */
maxTurns?: number
/** Maximum tool calls allowed for this task. */
maxToolCalls?: number
}
export interface TaskUsage {
tokensUsed: number
turnsUsed: number
toolCallsUsed: number
overBudget?: boolean
overBudgetReason?: string
updatedAt?: string
}
export interface Task {
id: string
title: string
@@ -44,6 +66,9 @@ export interface Task {
ownedPaths: string[]
worktree?: string
tournamentId?: string
claimedBySessionID?: string
budget?: TaskBudget
usage?: TaskUsage
status: TaskStatus
qaReceipts: QAReceipt[]
createdAt: string
@@ -54,6 +79,11 @@ export interface TournamentCandidate {
label: string
taskId: string
worktree: string
/** Git branch backing the worktree (`swarmos/<taskId>`). */
branch?: string
/** Whether a real git worktree exists at `worktree` (false outside git repositories). */
provisioned?: boolean
provisionNote?: string
}
export interface Tournament {
@@ -65,6 +95,10 @@ export interface Tournament {
status: "active" | "resolved"
winningTaskId?: string
resolutionSummary?: string
/** Commit the candidates branched from. */
baseRef?: string
/** Merge commit created when the winner was merged into the main tree. */
mergeCommit?: string
createdAt: string
updatedAt: string
}
@@ -79,11 +113,18 @@ export interface ReleaseReadinessReport {
supersededTasks: string[]
activeTournaments: string[]
blockers: string[]
latestFullVerifyRunId?: string
checkedAt: string
}
export interface BoardPolicy {
/** PASS receipts must cite a passing `tools.verify.run` taken on the current working tree. */
requireVerifiedReceipts: boolean
}
export interface OrgBoard {
version: string
policy?: BoardPolicy
project: ProjectCharter | null
contracts: Contract[]
tasks: Task[]
@@ -94,6 +135,8 @@ export interface InitProjectInput {
name: string
vision: string
aestheticArchetype: AestheticArchetype
/** Defaults to true for new projects. */
requireVerifiedReceipts?: boolean
}
export interface LockContractInput {
@@ -111,6 +154,19 @@ export interface CreateTaskInput {
ownedPaths: string[]
worktree?: string
tournamentId?: string
budget?: TaskBudget
}
export interface ClaimTaskInput {
taskId: string
sessionID?: string
}
export interface SetTaskBudgetInput {
taskId: string
budget: TaskBudget
/** If true, resets accumulated token/turn/toolCall usage counters on the task. */
resetUsage?: boolean
}
export interface SubmitQAReceiptInput {
@@ -119,6 +175,7 @@ export interface SubmitQAReceiptInput {
summary: string
defects: string[]
testCommand?: string
verifyRunId?: string
}
export interface PlanTournamentInput {
@@ -133,4 +190,6 @@ export interface ResolveTournamentInput {
tournamentId: string
winningTaskId: string
summary: string
/** Merge the winner's branch and remove all candidate worktrees (default true when worktrees exist). */
merge?: boolean
}
Loaded 100 of 141 files, more files were not shown because too many files have changed in this diff. Show more