Heretek-AI/IUMBTEMS
1
0
Fork 0
mirror of https://github.com/Heretek-AI/IUMBTEMS.git synced 2026-10-12 02:03:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Release 1.1.1: security release (audit blockers N1–N12) (#53)

Browse Source 
Release 1.1.1 security release. Ships S1–S4, #49, #52, #32, #31, M1, S6 docs. Closes #25–#32, #36–#48, #50–#52 on merge.
This commit is contained in:
BillyOutlast
2026-10-07 16:32:33 -04:00
committed by GitHub
parent 5e9c8e7c6b
commit 6c1937a3cf
97 changed files with 3088 additions and 710 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+8
View File
@@ -16,6 +16,14 @@ jobs:
steps:
# Third-party actions are pinned to commit SHAs (bump deliberately).
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
# The agent sandbox (1.1.1): factory seats and gate runs require
# bubblewrap; Ubuntu 24.04 also restricts unprivileged user namespaces.
- name: Install bubblewrap
run: |
sudo apt-get update -q
sudo apt-get install -y -q bubblewrap
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
bwrap --ro-bind / / --dev /dev -- true
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: "1.4.2"
.github/workflows/e2e-feedback.yml
+158
View File
@@ -0,0 +1,158 @@
name: E2E feedback (opencode + plugin)
# Repeatable end-to-end test: install opencode, install the plugin the way
# users get it, exercise the seats against a real model, then leave a
# deterministic debug report plus one best-effort agent feedback note.
#
# - `scripts/pack-smoke.sh` proves the packed tarballs install and load.
# - `bun run evals` runs one real, step-capped turn per `evals/cases/*.json`
# in an isolated workspace loading this repo's plugin; a failed case keeps
# its workspace, event stream and stderr under evals/results/<stamp>/.
# - `bun scripts/e2e-report.ts` renders the deterministic report, and
# evals/results/ is uploaded as the `e2e-feedback` artifact.
# - The `agent-feedback` step asks the model to diagnose that report (one
# `timeout`-bounded turn; best-effort, never masks the eval verdict).
#
# Secrets/vars (same triple as the nightly evals job):
# secrets.ES_EVAL_API_KEY the provider key
# vars.ES_EVAL_MODEL e.g. opencode/muse-spark-1.3-contributor-free
# vars.ES_EVAL_KEY_VAR env-var name for the key (default ANTHROPIC_API_KEY;
# use OPENCODE_API_KEY for Zen/Go keys)
on:
push:
branches: [rewrite]
workflow_dispatch:
inputs:
model:
description: "Override vars.ES_EVAL_MODEL for this run (empty = use the variable)"
required: false
default: ""
max_steps:
description: "Step ceiling: each case runs under its own maxSteps, never above this"
required: false
default: "16"
max_usd:
description: "Spend budget for the whole run in USD, from step_finish cost (empty = none)"
required: false
default: ""
# Least privilege: the workflow only reads the checkout.
permissions:
contents: read
jobs:
e2e-feedback:
runs-on: ubuntu-latest
timeout-minutes: 30
env:
ES_EVAL_MODEL: ${{ inputs.model || vars.ES_EVAL_MODEL }}
ES_EVAL_KEY_VAR: ${{ vars.ES_EVAL_KEY_VAR || 'ANTHROPIC_API_KEY' }}
ES_EVAL_MAX_STEPS: ${{ inputs.max_steps || '16' }}
ES_EVAL_MAX_USD: ${{ inputs.max_usd }}
ES_EVAL_TIMEOUT_MS: "300000"
steps:
# Third-party actions are pinned to commit SHAs (bump deliberately).
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
# The agent sandbox (1.1.1): factory seats and gate runs require
# bubblewrap; Ubuntu 24.04 also restricts unprivileged user namespaces.
- name: Install bubblewrap
run: |
sudo apt-get update -q
sudo apt-get install -y -q bubblewrap
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
bwrap --ro-bind / / --dev /dev -- true
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: "1.4.2"
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: "22"
- run: bun install --frozen-lockfile --ignore-scripts
# The packages as users install them (fails on workspace:* leaks or
# missing bundled assets).
- name: Install-as-user smoke test
run: scripts/pack-smoke.sh
- name: Install opencode CLI
run: |
# Pinned to the same CLI the repo tests against; bump deliberately
# together with @opencode/sdk and @opencode/plugin in package.json.
npm install --global --ignore-scripts @opencode/[email protected]
export PATH="$(npm root --global)/@opencode/cli-linux-x64/bin:$PATH"
opencode --version
- name: Exercise seats against the real model
id: evals
env:
ES_EVAL_API_KEY: ${{ secrets.ES_EVAL_API_KEY }}
run: |
if [ -z "$ES_EVAL_API_KEY" ] || [ -z "$ES_EVAL_MODEL" ]; then
echo "e2e-feedback: secrets.ES_EVAL_API_KEY and vars.ES_EVAL_MODEL (or inputs.model) are not both set; skipping."
echo "skipped=true" >> "$GITHUB_OUTPUT"
exit 0
fi
echo "skipped=false" >> "$GITHUB_OUTPUT"
export PATH="$(npm root --global)/@opencode/cli-linux-x64/bin:$PATH"
export "$ES_EVAL_KEY_VAR=$ES_EVAL_API_KEY"
opencode --version
bun run evals
# Deterministic report: offline, secret-free, runs even when evals fail.
- name: Render debug report
if: always() && steps.evals.outputs.skipped != 'true'
run: |
export PATH="$(npm root --global)/@opencode/cli-linux-x64/bin:$PATH"
newest="$(ls -t evals/results/*.json 2>/dev/null | head -1)"
[ -n "$newest" ] || { echo "debug report: no flat evals/results/*.json (evals never wrote one); skipping."; exit 0; }
bun scripts/e2e-report.ts "$newest" "$(opencode --version 2>/dev/null || echo unknown)" "${GITHUB_SHA:0:7}"
{
echo "## E2E feedback (${ES_EVAL_MODEL})"
echo
cat "${newest%.json}.md"
} >> "$GITHUB_STEP_SUMMARY"
# One best-effort agent turn over the deterministic report, bounded by
# `timeout`. Never fails the job: the eval verdict above is what
# reports.
- name: Agent feedback on the report
if: always() && steps.evals.outputs.skipped != 'true'
continue-on-error: true
env:
ES_EVAL_API_KEY: ${{ secrets.ES_EVAL_API_KEY }}
run: |
export PATH="$(npm root --global)/@opencode/cli-linux-x64/bin:$PATH"
export "$ES_EVAL_KEY_VAR=$ES_EVAL_API_KEY"
newest="$(ls -t evals/results/*.json 2>/dev/null | head -1)"
report="${newest%.json}.md"
[ -n "$newest" ] && [ -f "$report" ] || { echo "agent-feedback: no report found; skipping."; exit 0; }
work="$(mktemp -d)"
home="$work/home"
mkdir -p "$home"
cat > "$work/opencode.json" <<EOF
{
"model": "$ES_EVAL_MODEL",
"plugins": [{ "package": "$GITHUB_WORKSPACE/packages/opencode", "options": { "pr": "off" } }]
}
EOF
prompt="You are a debug analyst. Read the E2E feedback report below and reply with: (1) the most likely root cause per FAIL case, citing the failing check or tool line; (2) the smallest repo change that would fix or further isolate it; (3) anything in the report that looks like harness flakiness rather than a product bug. Keep it under 30 lines. Report: $(head -c 4000 "$report")"
# opencode resolves its project from PWD (not the spawn cwd), so run
# inside $work with PWD, HOME and XDG pointed there; otherwise the
# repo is the project and the host's default model answers.
(cd "$work" && PWD="$work" HOME="$home" XDG_CONFIG_HOME="$home/.config" XDG_DATA_HOME="$home/.local/share" \
XDG_STATE_HOME="$home/.local/state" XDG_CACHE_HOME="$home/.cache" \
timeout 150 opencode run --standalone --agent build "$prompt") > "$work/feedback.txt" 2>"$work/feedback.err" \
|| { echo "agent-feedback: model turn failed; last of stderr:"; tail -c 800 "$work/feedback.err"; echo; echo "keeping the deterministic report only."; exit 0; }
cp "$work/feedback.txt" "${newest%.json}.agent-feedback.md"
{ echo "## Agent feedback"; echo; cat "$work/feedback.txt"; } >> "$GITHUB_STEP_SUMMARY"
echo "agent-feedback: wrote $(basename "${newest%.json}.agent-feedback.md")"
# The results JSON, the report and every failed case's kept workspace.
- name: Upload eval evidence
if: always() && steps.evals.outputs.skipped != 'true'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: e2e-feedback
path: evals/results/
if-no-files-found: ignore
retention-days: 14
.github/workflows/nightly.yml
+24 -5
View File
@@ -23,6 +23,14 @@ jobs:
steps:
# Third-party actions are pinned to commit SHAs (bump deliberately).
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
# The agent sandbox (1.1.1): factory seats and gate runs require
# bubblewrap; Ubuntu 24.04 also restricts unprivileged user namespaces.
- name: Install bubblewrap
run: |
sudo apt-get update -q
sudo apt-get install -y -q bubblewrap
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
bwrap --ro-bind / / --dev /dev -- true
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: "1.4.2"
@@ -30,15 +38,26 @@ jobs:
- run: scripts/v2-head.sh packages/opencode
# Opt-in, cost-capped behaviour evals: one real turn per case against
# vars.ES_EVAL_MODEL, killed at the per-case step cap (no cost events exist
# in the CLI stream, so steps bound the spend). Skipped unless both the
# model and secrets.ES_EVAL_API_KEY are configured; the key is exported as
# vars.ES_EVAL_KEY_VAR (default ANTHROPIC_API_KEY).
# vars.ES_EVAL_MODEL, killed at the case's own step cap (bounded by
# ES_EVAL_MAX_STEPS) or the spend budget read from the stream's step_finish
# cost. Skipped unless both the model and secrets.ES_EVAL_API_KEY are
# configured; the key is exported as vars.ES_EVAL_KEY_VAR (default
# ANTHROPIC_API_KEY). Scheduled and manual runs only: on push, the E2E
# feedback workflow runs the same evals once, with a debug report.
evals:
if: github.event_name != 'push'
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
# The agent sandbox (1.1.1): factory seats and gate runs require
# bubblewrap; Ubuntu 24.04 also restricts unprivileged user namespaces.
- name: Install bubblewrap
run: |
sudo apt-get update -q
sudo apt-get install -y -q bubblewrap
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
bwrap --ro-bind / / --dev /dev -- true
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: "1.4.2"
@@ -48,7 +67,7 @@ jobs:
ES_EVAL_API_KEY: ${{ secrets.ES_EVAL_API_KEY }}
ES_EVAL_MODEL: ${{ vars.ES_EVAL_MODEL }}
ES_EVAL_KEY_VAR: ${{ vars.ES_EVAL_KEY_VAR || 'ANTHROPIC_API_KEY' }}
ES_EVAL_MAX_STEPS: "6"
ES_EVAL_MAX_STEPS: "16"
ES_EVAL_TIMEOUT_MS: "300000"
run: |
if [ -z "$ES_EVAL_API_KEY" ] || [ -z "$ES_EVAL_MODEL" ]; then
.github/workflows/publish.yml
+8
View File
@@ -25,6 +25,14 @@ jobs:
environment: IUMBTEMS
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
# The agent sandbox (1.1.1): factory seats and gate runs require
# bubblewrap; Ubuntu 24.04 also restricts unprivileged user namespaces.
- name: Install bubblewrap
run: |
sudo apt-get update -q
sudo apt-get install -y -q bubblewrap
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
bwrap --ro-bind / / --dev /dev -- true
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: "1.4.2"
AGENTS.md
+3 -3
View File
@@ -41,9 +41,9 @@ spike as proof).
## Contract invariants (do not break)
| Invariant | Where |
| :--- | :--- |
| Approvals and waivers are human-only (TUI/CLI dialog, signed, hash-bound); agents may request, never grant | `approval/`, `gates/waivers.ts` |
| Control files are deny-write for all agents: `.factory/{gates.json,config.json,frontier.json,waivers,approvals,runtime}`, the evidence (`.factory/research/{sources,coverage.json,dossier.json,brief.pcrb.json}`, `.factory/claims/`), `.git/config` and git hooks. The pinned ones (gates, config, frontier, approvals, waivers) are also hash-checked at every gate run; cached sources are self-verifying (content-addressed) | `trust/control.ts` |
| Per-seat path scopes: manager writes factory docs, programmer writes its worktree, QA and the auditor pair write nothing, brainstorm/harvest/design/scout write only their `.factory/<x>/notes/`; research, scout and auditor seats get web facts only through the cached `es_research_*` tools (registry `web: "cached"`) | `trust/policy.ts`, `agents/registry.ts` |
| Approvals, waivers, trust and resume are human-only: terminal passphrase confirmation unlocks the sealed Ed25519 human key (signed, hash-bound); the approve/trust/resume RPCs no longer exist and the TUI only previews, then points at the terminal command; agents may request, never grant | `approval/`, `gates/waivers.ts` |
| Control files are deny-write for all agents: `.factory/{gates.json,config.json,frontier.json,waivers,approvals,runtime}`, the evidence (`.factory/research/{sources,coverage.json,dossier.json,brief.pcrb.json}`, `.factory/claims/`), `.git/config` and git hooks. The pinned ones (gates, config, frontier, approvals, waivers) are also hash-checked at every gate run; cached sources are engine-sealed (HMAC) and content-addressed | `trust/control.ts` |
| Per-seat path scopes: manager writes factory docs, programmer writes its worktree, QA and the auditor pair write nothing, brainstorm/harvest/design/scout write only their `.factory/<x>/notes/`; research, scout and auditor seats are `web: "cached"` (host websearch/webfetch denied, web facts only through the cached `es_research_*` tools) and run `--unshare-net` sandboxed, while user agents keep host web tools | `trust/policy.ts`, `agents/registry.ts` |
| Code audits block: a phase audit holds `passPhase`, any other open or failed audit refuses the next stage transition; every finding is witnessed against the file on disk; only a human dismisses one | `factory/machine.ts`, `codeaudit/` |
| Scout verdicts are computed by core: `adopt` survives only for a verified, whitelisted, permissive license | `scout/verdict.ts` |
| Gate command sets are trust-pinned by hash; untrusted commands never run | `trust/store.ts` |
CHANGELOG.md
+56
View File
@@ -1,5 +1,61 @@
# Changelog
## 1.1.1 — 2026-10-07
Security release (#35). The 1.1.0 adversarial audit broke the text-matched
shell policy six ways, forged VERIFIED claims through the shell, and drove
approvals through a stealable service password; 1.1.1 moves that enforcement
into the OS, the crypto and the sandbox. Published 1.1.0 is deprecated.
**Breaking.**
- Re-approve, re-trust and re-waive after `es key seal`: v1 HMAC records
were MACed with a key agents could read; they are refused with the
re-record hint.
- Bubblewrap is required for factory seats (refused a shell without it) and
factory gate runs (halt with `gates/sandbox-missing`); CI, nightly, E2E
and publish install it and allow unprivileged user namespaces.
- Approvals, waivers, trust and resume happen at a terminal only: the
approve/trust/resume RPCs no longer exist and the TUI only previews, then
points at the terminal command.
- **S1: quotes are evidence fragment by fragment (#38, #46, #48).** Every
ellipsis fragment must be at least 12 characters on its own; code excerpts
are one contiguous verbatim span of the cited lines (at most 60 lines); a
thesis pass needs at least one witnessed invariant that holds; the
audit/scout/harvest fire graders are exact (plants cited within 3 lines,
no extra or duplicate rows).
- **S2: argv-aware shell policy (#37, #40–#45, #50, #51).** Quotes and escapes
are stripped before matching, human-only verbs are found behind wrappers,
git subcommands past global options, and control paths match
case-insensitively; any `.factory/` mention needs a provably read-only
command. This is defence in depth and the only layer for the user's own
agents without bubblewrap — documented as weaker.
- **S3: every agent shell and gate run is sandboxed by bubblewrap.** Kinds:
`user`, `seat`, `programmer`, `readonly` and `gate`; cached-web seats get
`--unshare-net`; the private state dir and service credentials are masked
in every kind. Seats lose Code Mode `execute`; the shell is re-checked by
the last hook after the hook bridge.
- **S4: passphrase-sealed Ed25519 human key, terminal-only approvals (#39).**
`es key seal` (asked twice, at least 10 characters) seals the key with
scrypt + AES-256-GCM; every human action unlocks it with a passphrase at a
TTY. A wrong or empty passphrase records nothing.
- **#49: no approval RPC for a stolen service password.** Verified: the
mutating RPC methods are gone (previews still work) and `service.json` is
masked in every agent sandbox.
- **#52: engine-sealed source cache.** Each entry's metadata carries an
HMAC-SHA256 seal from a key in the masked state dir; unsealed (planted)
entries are refused on read.
- **#32: per-file control re-pin.** `es_frontier_write` re-pins only
`frontier.json` when its baseline matches, so a concurrent hand edit to
another control file still halts as drift instead of being absorbed.
- **#31: host web caching only for factory seats in a factory project.**
Without `.factory/`, nothing is cached and no `.factory/` is created;
the user's agent never caches.
- **M1: eval harness that can pass (#25–#29, #36).** Case caps authoritative
under a ceiling of 16, `toolsUsed` counts completed calls, spend read from
`step_finish` cost under `ES_EVAL_MAX_USD`, failed workspaces kept and
uploaded, nightly evals off push, agent feedback uses PWD.
## 1.1.0 — 2026-10-07
"Dialectic Restoration" (#20) brings back the best of the 0.7 research
README.md
+30 -17
View File
@@ -5,10 +5,15 @@
> phases under dual QA, and open a PR a human merges. Mechanical gates and a
> human-only approval channel keep autonomous runs honest.
> **Status: 1.0 in development.** This repository is the TypeScript rewrite on
> the `rewrite` branch. The legacy Python harness is tagged `legacy-final` and
> receives no further support; the 1.0 cutover replaces `main` in one reviewed
> merge, after which the packages publish to npm.
> **Status: 1.1.1 security release (unreleased).** Published 1.1.0 (and
> 1.0.x) are deprecated. 1.1.1 ships the security fixes: the S1 quote law,
> the S2 argv-aware shell policy, the S3 bubblewrap sandbox for every agent
> shell and gate run, and the S4 passphrase-sealed Ed25519 human key with
> terminal-only approvals — plus the engine-sealed source cache (#52) and
> the per-file control re-pin (#32). Install from npm
> (`@heretek-ai/epistemic-swarm`, `@heretek-ai/es-core`, `@heretek-ai/es-cli`);
> run `es key seal` once, then re-record approvals, trust and waivers (v1
> HMAC records are refused).
## Naming
@@ -21,8 +26,11 @@
- **Factory stages, enforced by code.** Grill → Research → Spec → Build ⇄ QA →
Release runs through a state machine; every `es_*` tool refuses an
out-of-order step. Approvals (frontier, spec) and waivers are human-only,
signed through a TUI/CLI dialog agents cannot call.
out-of-order step. Approvals (frontier, spec), waivers and trust are human-only,
signed with the passphrase-sealed Ed25519 human key at a terminal
(`es key seal` once, then `es approve` / `es trust` / `es waive`); agents
may request, never grant. The approve/trust/resume RPCs no longer exist:
the TUI previews, then points at the terminal command.
- **Mechanical gates.** Format, lint and typecheck on touched files, affected
tests from a tree-sitter import graph (runner-native fallback), secret
scanning, OSV, budgets (diff size, file length, complexity, dependency
@@ -37,7 +45,8 @@
contrast-gated DTCG token system with a generated style guide.
- **OpenCode v2 native.** One plugin registers agents, tools, commands, the
hook bridge, the LSP runtime and four TUI panels — additively, with no files
written. Claude Code (1.1), Pi (1.2) and Antigravity (1.3) adapters follow,
written. Host web results are cached (citable by hash) only for factory
seats in a project that already has `.factory/`. Claude Code (1.1), Pi (1.2) and Antigravity (1.3) adapters follow,
each publishing capability-matrix rows with smoke tests.
## Repository layout
@@ -57,25 +66,29 @@
The integrity model is mechanical, but these limits are deliberate and visible
rather than silently assumed:
- **Read-only seats.** QA and manager shells run under `bwrap` when it is
installed. Without `bwrap` (some CI runners), the fallback is a command
allowlist plus a post-run tree fingerprint that halts the run on any change;
it catches writes to files in the checkout, not reads or side effects outside
it.
- **Seats require bubblewrap.** Every agent shell and every gate run is
sandboxed by `bwrap` (user, seat, programmer, readonly and gate kinds;
cached-web seats are `--unshare-net`). Without `bwrap`, factory seats are
refused a shell outright and factory gate runs halt; the user's own agents
fall back to the weaker argv-aware text policy (pattern matching a shell
can outwit, so that mode is documented as weaker, not equivalent).
- **Control-file baseline.** `verifyControl` hashes the files a human
authorises — `gates.json`, `config.json`, `frontier.json`, `approvals/**`,
`waivers/**`; a hand edit is drift until a human accepts it (`es rebaseline`).
Everything else that is a control file (engine-owned brainstorm, harvest and
design state, `runtime/**`, `.git/config`) is deny-write for agents but is
not individually hashed on every run.
not individually hashed on every run. Signed sidecars for other artifacts
move to 1.1.2.
- **Licence detection is strict.** A permissive verdict needs the whole licence
file to match an SPDX template; mixed, notice-only (e.g. an Apache header
without the licence text) or concatenated licence files are "unknown" and
therefore clean-room only. Agents harvest local code only inside the
project; a human can scan elsewhere with `es harvest scan`.
- **Evals are opt-in and step-capped.** `bun run evals` needs the opencode CLI
and `ES_EVAL_MODEL`; each case is one real turn killed at a step cap, since
the CLI's event stream reports no cost.
- **Evals are opt-in and capped.** `bun run evals` needs the opencode CLI
and `ES_EVAL_MODEL`; each case is one real turn killed at its own step cap
(bounded by `ES_EVAL_MAX_STEPS`) or once the run's spend, read from the
stream's `step_finish` cost, passes `ES_EVAL_MAX_USD`. A model with no
configured price reports $0, so for it only the step caps bound spend.
- **MCP caller identity.** Over the stdio MCP server the calling agent's
identity is a model-supplied `agent` argument (or `ES_AGENT`), so it is
ADVISORY: the harness adapter, not the protocol, establishes it. There is no
@@ -100,7 +113,7 @@ Platform: Linux, Node ≥ 22 or Bun. No Python.
- `docs/CAPABILITIES.md` — what each harness enforces, with proof references.
- `docs/CONFIG.md` — the layered config (global → project → plugin options).
- `docs/SCHEMAS.md` — every Zod contract as JSON Schema.
- `CHANGELOG.md` — the 1.0 cutover and breaking changes.
- `CHANGELOG.md` — release history and breaking changes.
## License
SYSTEM_ARCHITECTURE.md
+23 -6
View File
@@ -46,7 +46,7 @@ Seats are what the trust policy reasons about; the registry is
| `programmer` | subagent | its phase worktree | red-test-first completion |
| `qa-functional`, `qa-adversarial` | subagents | nothing | verdicts only, blocking |
| `auditor-thesis`, `auditor-antithesis` | subagents | nothing | line-witnessed findings, blocking |
| `research-alpha`, `research-beta` | subagents | research notes | web facts via cached `es_research_*` only |
| `research-alpha`, `research-beta` | subagents | research notes | `web: "cached"`: web facts via cached `es_research_*` only, `--unshare-net` sandboxed |
| `harvester` | primary | harvest notes | fail-closed licence policy for depend/vendor |
| `scout` | primary | scout notes | core computes adoption verdicts |
| `brainstormer`, `brainstorm-lens` (×8), `brainstorm-critic` | subagents | brainstorm notes | lens fan-out, dedupe, rubric |
@@ -77,23 +77,40 @@ domain packs (quant · biopharma · legal) ──▶ banned domains, mandatory t
- Claims/dossiers live under `.factory/`: `research/`, `claims/`, `audits/`,
`scout/`, `harvest/`. All are control files: agents may only write them
through `es_*` tools; hand-edits are refused and hash-checked at gates.
- The source cache is engine-sealed: each entry's metadata carries an
HMAC-SHA256 seal from a key kept in the masked private state dir
(`engine.key`), so a planted entry an agent drops into
`.factory/research/sources/` is refused on read.
- The design tree (`frontier.json`) is the same shape for humans: nodes are
decisions or deferred facts, saves are diff-checked (no deletions, no silent
edits, no round regress), and the tree is the single source for the idea.
## 4. Trust model (non-negotiable invariants)
1. **Approvals, waivers, trust and resume are human-only** — TTY/TUI dialogs,
signed and hash-bound; agents may request (`es_request_approval`) but never
grant.
1. **Approvals, waivers, trust and resume are human-only** — terminal
passphrase confirmation unlocks the passphrase-sealed Ed25519 human key
(scrypt + AES-256-GCM; `es key seal` once, `es key status` shows the
fingerprint), which signs the record; agents may request
(`es_request_approval`) but never grant. The approve/trust/resume RPCs no
longer exist; the TUI previews, then points at the terminal command. v1
HMAC records are refused and must be re-recorded.
2. **Control files are deny-write for every agent**: gates, config, frontier,
approvals, waivers, runtime state, the research evidence, claim ledger,
audit records, scout and harvest state, and the design artifacts
(`trust/control.ts`).
3. **Gate command sets are trust-pinned by hash**; untrusted commands never run.
4. **Every `.factory/` write is atomic and lock-protected** (`util/fs.ts`).
5. **Search as policy**: research seats, auditor seats, scout and harvester get
web facts only through the cached, citable `es_research_*` tools.
5. **Search as policy, under a sandbox**: research, scout and auditor seats
are `web: "cached"` (host websearch/webfetch denied, web facts only
through the cached, citable `es_research_*` tools) and run under an
`--unshare-net` bubblewrap sandbox, so no live network hides behind the
tools. The user's own agents keep host web tools. Every agent shell and
gate run is sandboxed by kind — `user` (project writable, `.factory/`
read-only), `seat`, `programmer` (only its worktree writable), `readonly`
(nothing writable) and `gate` (only the gate dir writable) — with the
private state dir and service credentials masked; factory seats are
refused a shell without bubblewrap and the user's agents fall back to the
weaker argv-aware text policy.
6. **Generated files are never hand-edited**; `docs:check` fails on drift
(schemas, capability proofs, config docs, and the prompt/skill/registry
asset drift checker with its canary).
bun.lock
+3 -3
View File
@@ -14,7 +14,7 @@
},
"packages/cli": {
"name": "@heretek-ai/es-cli",
"version": "1.1.0",
"version": "1.1.1",
"bin": {
"es-factory": "./dist/bin.js",
"iumbtems": "./dist/bin.js",
@@ -25,7 +25,7 @@
},
"packages/core": {
"name": "@heretek-ai/es-core",
"version": "1.1.0",
"version": "1.1.1",
"dependencies": {
"vscode-jsonrpc": "9.0.3",
"vscode-languageserver-protocol": "3.18.4",
@@ -35,7 +35,7 @@
},
"packages/opencode": {
"name": "@heretek-ai/epistemic-swarm",
"version": "1.1.0",
"version": "1.1.1",
"dependencies": {
"@heretek-ai/es-core": "workspace:*",
"@opencode/plugin": "2.0.24",
docs/CAPABILITIES.md
+7 -7
View File
@@ -12,21 +12,21 @@ and Antigravity adapters ship in 1.1–1.3). Generated: do not edit by hand.
| agents | ENFORCED | `packages/opencode/test/plugin.test.ts` | Registered by agent.transform with wildcard-deny scoping; never touches built-ins or defaults. |
| skills | ENFORCED | `packages/opencode/test/plugin.test.ts` | One shared SKILL.md tree registered at runtime; per-agent visibility via skill deny rules. |
| tools | ENFORCED | `packages/opencode/test/plugin.test.ts` | es_* tools registered direct (codemode:false) with a permission action per tool; seat-scoped. |
| permissions | ENFORCED | `packages/opencode/test/plugin.test.ts` | permission.evaluate + tool.execute.before enforce the write/read/shell policy; control files (incl. the evidence cache) denied; research seats get no host websearch/webfetch. |
| permissions | ENFORCED | `packages/core/test/policy.test.ts` | permission.evaluate + tool.execute.before enforce the write/read/shell policy; control files (incl. the engine-sealed evidence cache) denied; research seats get no host websearch/webfetch. Every agent shell runs under bubblewrap by seat kind (cached-web seats offline); seats are refused without it and the user's agents fall back to the weaker argv-aware text policy. |
| hooks | ENFORCED | `packages/opencode/test/hooks.test.ts` | Claude hook schema bridged in-process; Stop is emulated after the turn (advisory for plain chat). |
| question | ENFORCED | `packages/opencode/test/plugin.test.ts` | Session forms only humans can answer; primaries may ask, autonomous subagent seats may not. |
| subagents | ENFORCED | `packages/opencode/test/plugin.test.ts` | Depth 1; hidden seats are unlisted but invocable by ID, and only from the seats allowed to spawn them. |
| subagents | ENFORCED | `packages/opencode/test/plugin.test.ts` | Depth 1; hidden seats are unlisted but invocable by ID, and only from the seats allowed to spawn them. Factory seats are not offered Code Mode execute. |
| compaction | ENFORCED | `packages/opencode/test/plugin.test.ts` | Session compaction injects the compact factory-state block for factory seats. |
| websearch | ENFORCED | `packages/opencode/test/research.test.ts` | The configured provider registers as the host websearch; results are cached and citable. Research seats are cached-only: es_research_search/fetch (fresh hits served from the cache). |
| websearch | ENFORCED | `packages/opencode/test/research.test.ts` | The configured provider registers as the host websearch; results are cached and citable. Research, scout and auditor seats are cached-only (es_research_search/fetch) under an offline (--unshare-net) sandbox; host web caching applies only in a project that already has .factory/, and the cache is engine-sealed (planted entries refused). |
| lsp | ENFORCED | `packages/opencode/test/lsp.test.ts` | The lsp config key is read and served by our own manager (v2 accepts but does not run it). |
| panels | ENFORCED | `packages/opencode/test/panels.test.ts` | Four session.panel dashboards (factory, LSP, hooks, brainstorm), rendered by OpenTUI and refreshed on server changes. |
| research | ENFORCED | `packages/opencode/test/research.test.ts` | Cache, quote verifier, auditor, providers and the websearch bridge. |
| research | ENFORCED | `packages/core/test/research.test.ts` | Engine-sealed cache (#52), fragment-level quote verifier (every ellipsis fragment >= 12 chars; code excerpts one contiguous span of at most 60 lines), auditor, providers and the websearch bridge. |
| brainstorm | ENFORCED | `packages/opencode/test/brainstorm.test.ts` | Lens fan-out, record/score/complete tools, dedupe, rubric, shortlist with a forced outlier. |
| harvest | ENFORCED | `packages/opencode/test/harvest.test.ts` | Fail-closed SPDX detection, provenance profiles, policy-enforced matrix, clean-room specs. |
| design | ENFORCED | `packages/opencode/test/design.test.ts` | Queereye interview, DTCG tokens, contrast gate, generated guide and drift check; phase-02 specs/webref/csf/tui renders and the phase-03 ledger bundle with its cite-gate receipt. |
| claims | ENFORCED | `packages/opencode/test/plugin.test.ts` | Witnessed claims and dossiers: the evidence cache, dossiers and the claim ledger are tool-only; a forged source is denied on the host. |
| audit | ENFORCED | `packages/opencode/test/fires.test.ts` | Code-audit pair: only auditor seats record verdicts; every finding is witnessed on disk (a hallucinated line is refused); verdicts block. |
| scout | ENFORCED | `packages/opencode/test/fires.test.ts` | OSS scout: cached-only web, fail-closed license verdicts computed by core (adopt only for verified permissive licenses), OSV advisories cited. |
| claims | ENFORCED | `packages/core/test/research.test.ts` | Witnessed claims and dossiers: the evidence cache, dossiers and the claim ledger are tool-only; planted cache entries are refused (engine seal) and every VERIFIED quote and code span is re-checked against the bytes on disk. |
| audit | ENFORCED | `packages/opencode/test/fires.test.ts` | Code-audit pair: only auditor seats record verdicts; every finding is witnessed on disk (a hallucinated line is refused); quote fragments carry >= 12 chars each and a thesis pass needs a witnessed invariant; verdicts block. |
| scout | ENFORCED | `packages/opencode/test/fires.test.ts` | OSS scout: cached-only web under an offline (--unshare-net) sandbox, fail-closed license verdicts computed by core (adopt only for verified permissive licenses), OSV advisories cited. |
## claude
docs/CONFIG.md
+2 -1
View File
@@ -2,7 +2,8 @@
The canonical Epistemic Swarm config (`packages/core/src/schema/config.ts`), layered:
user-global (`~/.config/epistemic-swarm/config.json`) → project (`.factory/config.json`) →
plugin options in `opencode.json`. Unknown keys are rejected. Generated: do not edit by hand.
plugin options in `opencode.json`. An unknown key in a config file is rejected; an unknown
plugin option is ignored with a warning (TUI toast and `/config`). Generated: do not edit by hand.
| field | type | default | description |
| --- | --- | --- | --- |
docs/SCHEMAS.md
+1
View File
@@ -54,6 +54,7 @@ Do not edit by hand: run `bun run docs:gen` after changing a schema (CI runs `bu
| probe-row | `schemas/probe-row.schema.json` |
| provenance | `schemas/provenance.schema.json` |
| queereye-tokens | `schemas/queereye-tokens.schema.json` |
| record-signature | `schemas/record-signature.schema.json` |
| related-tests | `schemas/related-tests.schema.json` |
| retraction-ledger | `schemas/retraction-ledger.schema.json` |
| retraction-policy | `schemas/retraction-policy.schema.json` |
evals/cases/audit-fires.json
+1 -1
View File
@@ -3,7 +3,7 @@
"description": "The antithesis auditor must catch the planted defects (SQL injection, a TOCTOU lock, a hard-coded key, a logged secret) with witnessed line pointers; graded deterministically from .factory/audits/audit-01/records.json.",
"agent": "es-auditor-antithesis",
"prompt": "Red-team code audit audit-01 on path src (the files under src/). Find every exploitable defect, then record one verdict with es_audit_verdict: each finding with its CWE, exact lines and a verbatim excerpt.",
"maxSteps": 6,
"maxSteps": 8,
"files": {
"src/injection.ts": "@fixture:fires/audit/src/injection.ts",
"src/race.ts": "@fixture:fires/audit/src/race.ts",
evals/cases/brainstorm.json
+1 -1
View File
@@ -3,6 +3,6 @@
"description": "The brainstormer freezes a lens plan with es_brainstorm_plan before any fan-out.",
"agent": "brainstormer",
"prompt": "Brainstorm: cut our CI cost in half. For now only plan it with es_brainstorm_plan and tell me which lenses it chose; do not launch the lenses yet.",
"maxSteps": 4,
"maxSteps": 6,
"checks": { "toolsUsed": ["es_brainstorm_plan"], "contains": ["inversion"] }
}
evals/cases/darkharvest-fires.json
+1 -1
View File
@@ -3,7 +3,7 @@
"description": "The harvester tears down five local candidates and proposes depend for all; core must keep depend only for the verified permissive two (MIT, Apache-2.0). Graded from .factory/harvest/harvest.json.",
"agent": "harvester",
"prompt": "Tear down these local candidates for a session handoff feature: local:./vendor/mit, local:./vendor/apache, local:./vendor/gpl, local:./vendor/agpl, local:./vendor/none. This is a headless run: plan them, scan each, submit one matrix row proposing depend for every candidate, then es_harvest_complete.",
"maxSteps": 8,
"maxSteps": 10,
"files": {
"vendor/mit/index.js": "module.exports = function mit() { return 1 }\n",
"vendor/mit/package.json": "{\"name\": \"mit\", \"version\": \"1.0.0\"}\n",
evals/cases/factory.json
+1 -1
View File
@@ -3,6 +3,6 @@
"description": "With no run in the project, the factory reports that and points at the grill instead of starting a build.",
"agent": "factory",
"prompt": "Status?",
"maxSteps": 3,
"maxSteps": 5,
"checks": { "contains": ["grill"], "toolsNotUsed": ["es_build_start", "es_frontier_write"] }
}
evals/cases/scout-fires.json
+1 -1
View File
@@ -3,7 +3,7 @@
"description": "The scout vets five local candidates (MIT, Apache-2.0, GPL-3.0, AGPL-3.0, no license); whatever it proposes, the computed verdicts must adopt only the permissive two. Graded from .factory/scout/scout.json.",
"agent": "scout",
"prompt": "Scout a function library for our Apache-2.0 commercial product among these local candidates: local:./vendor/mit, local:./vendor/apache, local:./vendor/gpl, local:./vendor/agpl, local:./vendor/none. This is a headless run: plan them, scan each, record an assessment for each, then es_scout_complete.",
"maxSteps": 6,
"maxSteps": 14,
"files": {
"vendor/mit/index.js": "module.exports = function mit() { return 1 }\n",
"vendor/mit/package.json": "{\"name\": \"mit\", \"version\": \"1.0.0\"}\n",
package.json
+1
View File
@@ -12,6 +12,7 @@
"lint": "biome check .",
"deps:check": "bun scripts/deps.ts",
"evals": "bun scripts/evals.ts",
"evals:report": "bun scripts/e2e-report.ts",
"format": "biome format --write .",
"check": "bun run lint && bun run deps:check && bun run typecheck && bun run test",
"docs:gen": "bun scripts/docs.ts",
packages/cli/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@heretek-ai/es-cli",
"version": "1.1.0",
"version": "1.1.1",
"description": "Epistemic Swarm CLI: factory control, human-only checkpoints, gates, headless runs and a coarse MCP server.",
"type": "module",
"license": "Apache-2.0",
packages/cli/src/gates.ts
+2 -2
View File
@@ -4,7 +4,7 @@ import path from "node:path"
import { formatReport, git, runGates } from "@heretek-ai/es-core"
import type { Args } from "./args.ts"
import type { HumanContext } from "./human.ts"
import { confirmWithCode } from "./tty.ts"
import { confirmHuman } from "./tty.ts"
async function changedFiles(root: string, staged: boolean) {
if (staged)
@@ -51,7 +51,7 @@ export async function installGitHooks(context: HumanContext, args: Args): Promis
`Writes .factory/git-hooks/pre-commit (gates on staged files) and pre-push (full gates),`,
`then sets core.hooksPath=.factory/git-hooks${current ? ` (replacing "${current}")` : ""}.`,
]
if (!(await confirmWithCode(context.io, "Install git hooks", lines))) return 1
if (!(await confirmHuman(context.io, "Install git hooks", lines, context.stateDir))) return 1
await mkdir(dir, { recursive: true })
for (const mode of ["pre-commit", "pre-push"] as const) {
await writeFile(path.join(dir, mode), HOOK(mode))
packages/cli/src/human.ts
+29 -14
View File
@@ -1,5 +1,5 @@
// Human-only commands: approve, trust, waive, resume, rebaseline. Each previews exactly
// what will be signed, then requires the typed confirmation code at a TTY.
// what will be signed, then requires the human passphrase at a TTY.
import { userInfo } from "node:os"
import {
type ApprovalStage,
@@ -28,7 +28,7 @@ import {
writeJson,
} from "@heretek-ai/es-core"
import { type Args, flag, parseExpiry } from "./args.ts"
import { type ConfirmIO, confirmWithCode } from "./tty.ts"
import { type ConfirmIO, confirmHuman } from "./tty.ts"
export interface HumanContext {
readonly root: string
@@ -52,13 +52,15 @@ export async function approve(context: HumanContext, args: Args): Promise<number
}
const subject = await approvalSubject(context.root, stage)
const lines = [...subject.summary, "", ...subject.subject.map((item) => `${item.sha256.slice(0, 12)} ${item.path}`)]
if (!(await confirmWithCode(context.io, `Approve ${stage} as ${user()}`, lines))) {
const signer = await confirmHuman(context.io, `Approve ${stage} as ${user()}`, lines, context.stateDir)
if (!signer) {
context.print("Cancelled; nothing was approved.")
return 1
}
const record = await recordApproval(context.root, {
stage,
channel: "cli",
signer,
...(context.stateDir ? { stateDir: context.stateDir } : {}),
})
const pending =
@@ -102,13 +104,17 @@ export async function trust(context: HumanContext, args: Args): Promise<number>
context.print("These gate commands and hooks are already trusted.")
return 0
}
if (
!(await confirmWithCode(context.io, "Trust these gate commands and project hooks (they run project code)", lines))
) {
const signer = await confirmHuman(
context.io,
"Trust these gate commands and project hooks (they run project code)",
lines,
context.stateDir,
)
if (!signer) {
context.print("Cancelled.")
return 1
}
const options = context.stateDir ? { stateDir: context.stateDir } : {}
const options = { ...(context.stateDir ? { stateDir: context.stateDir } : {}), signer }
await trustProject(context.root, gates.hash, gates.lines, options)
if (hooks.projectHash)
await trustProject(context.root, hooks.projectHash, hooks.projectLines, { ...options, kind: "hooks" })
@@ -125,7 +131,14 @@ export async function rebaselineControl(context: HumanContext): Promise<number>
)
return 0
}
if (!(await confirmWithCode(context.io, `Accept these control-file changes as ${user()}`, [...check.violations]))) {
if (
!(await confirmHuman(
context.io,
`Accept these control-file changes as ${user()}`,
[...check.violations],
context.stateDir,
))
) {
context.print("Cancelled; the baseline is unchanged.")
return 1
}
@@ -153,7 +166,8 @@ export async function waive(context: HumanContext, args: Args): Promise<number>
.replace(/^-|-$/g, "")
.toLowerCase()}-${Date.now().toString(36)}`.slice(0, 64)
const lines = [`Rule: ${rule}`, `Files: ${files}`, `Reason: ${reason}`, `Expires: ${expiresAt.toISOString()}`]
if (!(await confirmWithCode(context.io, `Grant waiver ${id}`, lines))) {
const signer = await confirmHuman(context.io, `Grant waiver ${id}`, lines, context.stateDir)
if (!signer) {
context.print("Cancelled.")
return 1
}
@@ -164,6 +178,7 @@ export async function waive(context: HumanContext, args: Args): Promise<number>
reason,
expiresAt,
channel: "cli",
signer,
...(context.stateDir ? { stateDir: context.stateDir } : {}),
})
context.print(`Waiver ${waiver.id} granted until ${waiver.expiresAt}.`)
@@ -191,7 +206,7 @@ export async function resume(context: HumanContext, args: Args): Promise<number>
...(raise ? [`New spend ceiling: $${raise}`] : []),
...(args.flags["extend-runtime"] ? ["Runtime cap restarts now."] : []),
]
if (!(await confirmWithCode(context.io, "Resume the factory", lines))) {
if (!(await confirmHuman(context.io, "Resume the factory", lines, context.stateDir))) {
context.print("Cancelled.")
return 1
}
@@ -211,7 +226,7 @@ export async function recordPr(context: HumanContext, args: Args): Promise<numbe
context.print("Usage: es factory pr <url>")
return 2
}
if (!(await confirmWithCode(context.io, "Record the release PR", [url]))) return 1
if (!(await confirmHuman(context.io, "Record the release PR", [url], context.stateDir))) return 1
const state = await factoryFor(context).recordPr(user(), url)
context.print(`Recorded ${url}; factory is ${state.stage}.`)
return 0
@@ -246,7 +261,7 @@ export async function retract(context: HumanContext, args: Args): Promise<number
`Claims citing it: ${affected.length} (they become ${to})`,
...affected.slice(0, 10).map((claim) => ` - ${claim.statement.slice(0, 90)}`),
]
if (!(await confirmWithCode(context.io, `Record source ${event.toLowerCase()} as ${user()}`, lines))) {
if (!(await confirmHuman(context.io, `Record source ${event.toLowerCase()} as ${user()}`, lines, context.stateDir))) {
context.print("Cancelled; nothing was recorded.")
return 1
}
@@ -263,7 +278,7 @@ export async function retract(context: HumanContext, args: Args): Promise<number
const show = (value: unknown) => (value === undefined ? "(unset)" : JSON.stringify(value))
/** Set one config key in the project (default) or global layer: schema-validated, previewed, typed-code confirmed. */
/** Set one config key in the project (default) or global layer: schema-validated, previewed, passphrase-confirmed. */
export async function configSet(context: HumanContext, args: Args): Promise<number> {
const [key, value] = args.positionals
if (!key || value === undefined) {
@@ -294,7 +309,7 @@ export async function configSet(context: HumanContext, args: Args): Promise<numb
: []),
...(layer === "project" ? ["The control baseline is re-recorded for .factory/config.json."] : []),
]
if (!(await confirmWithCode(context.io, `Set ${key} as ${user()}`, lines))) {
if (!(await confirmHuman(context.io, `Set ${key} as ${user()}`, lines, context.stateDir))) {
context.print("Cancelled; the config is unchanged.")
return 1
}
packages/cli/src/jobs.ts
+6 -4
View File
@@ -1,7 +1,7 @@
// `es audit <target>` and `es scout <objective>`: open the job as the human at
// the terminal, then drive the seat headlessly until it settles. Both run on a
// factory run with a spend ceiling (decision 7): with none, `--max-usd N`
// starts a GRILL run with that provisional ceiling, confirmed with a typed code.
// starts a GRILL run with that provisional ceiling, confirmed with the passphrase.
import { userInfo } from "node:os"
import path from "node:path"
import {
@@ -19,7 +19,7 @@ import {
import { type Args, flag } from "./args.ts"
import { DRIVERS, driveHeadless, type HeadlessEvent } from "./headless.ts"
import type { HumanContext } from "./human.ts"
import { confirmWithCode } from "./tty.ts"
import { confirmHuman } from "./tty.ts"
const user = () => userInfo().username
@@ -48,7 +48,9 @@ async function ensureCeiling(context: HumanContext, args: Args, purpose: string)
: `A factory run starts in GRILL with a provisional spend ceiling of $${ceiling}.`,
"Seats halt when the spend reaches it; a later frontier approval sets the real ceiling.",
]
if (!(await confirmWithCode(context.io, `${purpose}: set a $${ceiling} ceiling as ${user()}`, lines))) {
if (
!(await confirmHuman(context.io, `${purpose}: set a $${ceiling} ceiling as ${user()}`, lines, context.stateDir))
) {
context.print("Cancelled; nothing was started.")
return 1
}
@@ -188,7 +190,7 @@ export async function auditDismiss(context: HumanContext, args: Args): Promise<n
`Reason: ${reason}`,
"It stops blocking the factory; the dismissal is signed into the audit log.",
]
if (!(await confirmWithCode(context.io, `Dismiss ${audit.id} as ${user()}`, lines))) {
if (!(await confirmHuman(context.io, `Dismiss ${audit.id} as ${user()}`, lines, context.stateDir))) {
context.print("Cancelled; the audit still blocks.")
return 1
}
packages/cli/src/key.ts
+51
View File
@@ -0,0 +1,51 @@
// `es key`: the passphrase-sealed human key (1.1.1). The private half never
// leaves the sealed file; the passphrase is typed with echo off, twice at
// seal time. Agents cannot seal (no TTY) and cannot unlock (no passphrase).
import { HumanKeyError, hasHumanKey, humanKeyId, sealHumanKey } from "@heretek-ai/es-core"
import type { MainIO } from "./main.ts"
import { NotInteractive } from "./tty.ts"
export async function keySeal(io: MainIO): Promise<number> {
if (!io.confirm.interactive) throw new NotInteractive("es key seal")
if (await hasHumanKey(io.stateDir)) {
io.print(
"This machine already has a human key; remove human.key and human.pub by hand to replace it, then re-record approvals.",
)
return 1
}
const first = await io.confirm.readSecret("Your Epistemic Swarm passphrase (at least 10 characters, empty cancels): ")
if (!first) {
io.print("Cancelled; no key was sealed.")
return 1
}
const second = await io.confirm.readSecret("Type it again to confirm: ")
if (!second) {
io.print("Cancelled; no key was sealed.")
return 1
}
if (first !== second) {
io.print("The two passphrases do not match; nothing was sealed.")
return 1
}
try {
const { keyId } = await sealHumanKey(first, io.stateDir)
io.print(`Sealed the human key ${keyId}; approvals, waivers and trust must now be recorded again.`)
return 0
} catch (error) {
if (error instanceof HumanKeyError) {
io.print(error.message)
return 1
}
throw error
}
}
export async function keyStatus(io: MainIO): Promise<number> {
const id = await humanKeyId(io.stateDir)
io.print(
id
? `Human key ${id} is sealed on this machine.`
: "No passphrase-sealed human key on this machine: run `es key seal` once at a terminal.",
)
return 0
}
packages/cli/src/main.ts
+31 -7
View File
@@ -1,6 +1,7 @@
// `es`: the Epistemic Swarm CLI. Human-only actions (approve, trust, waive,
// resume, git hooks) require an interactive terminal and a typed code; agent
// shells are denied these commands by policy as well.
// resume, git hooks) require an interactive terminal and the human passphrase,
// which unlocks the passphrase-sealed human key; agent shells are denied
// these commands by policy as well.
import { readFile } from "node:fs/promises"
import path from "node:path"
import {
@@ -68,8 +69,9 @@ import {
waive,
} from "./human.ts"
import { auditCommand, auditDismiss, auditShow, scoutCommand, scoutShow } from "./jobs.ts"
import { keySeal, keyStatus } from "./key.ts"
import { serveStdio } from "./mcp.ts"
import { type ConfirmIO, confirmWithCode, NotInteractive, terminalIO } from "./tty.ts"
import { type ConfirmIO, confirmHuman, NotInteractive, terminalIO } from "./tty.ts"
import { VERSION } from "./version.ts"
export { VERSION }
@@ -89,10 +91,12 @@ Factory
factory pr <url> Record a release PR opened by hand [human, TTY]
Checkpoints and exceptions [human, TTY]
approve <frontier|spec> Approve a checkpoint (shows hashes; type the code)
approve <frontier|spec> Approve a checkpoint (shows hashes; asks for the passphrase)
trust [--show] Approve this project's gate commands by hash
rebaseline Accept hand edits to pinned control files (gates.json, config.json)
waive <rule> --reason "…" [--files <glob>] [--expires 7d] [--id <name>]
key seal Create the passphrase-sealed human key [human, TTY]
key status Show the human key fingerprint
Gates
gates run [--full] [--staged] [--base <ref>] [--json] [files…]
@@ -209,13 +213,32 @@ export async function main(argv: readonly string[], io: MainIO): Promise<number>
return await rebaselineControl(context)
case "waive":
return await waive(context, subArgs(1))
case "key": {
if (sub === "seal") return await keySeal(io)
if (sub === "status") return await keyStatus(io)
io.print("Usage: es key seal | status")
return 2
}
case "research": {
if (sub === "retract") return await retract(context, subArgs(2))
if (sub === "export") {
const out = flag(args, "out")
const outFile = out ? path.resolve(io.cwd, out) : undefined
const signer = await confirmHuman(
io.confirm,
"Export research brief",
[
`Signs the brief with the human key (${outFile ? path.relative(root, outFile) : ".factory/research/brief.pcrb.json"}).`,
],
io.stateDir,
)
if (!signer) {
io.print("Cancelled; nothing was exported.")
return 1
}
const { file, brief } = await exportBrief(root, {
...(io.stateDir ? { stateDir: io.stateDir } : {}),
...(out ? { out: path.resolve(io.cwd, out) } : {}),
signer,
...(outFile ? { out: outFile } : {}),
})
const witnessed = brief.claims.filter((claim) => claim.witness.ok).length
io.print(
@@ -262,6 +285,7 @@ export async function main(argv: readonly string[], io: MainIO): Promise<number>
const tools = researchTools({
root,
...(config ? researchOptions(config) : {}),
stateDir: io.stateDir,
policy: () => Promise.resolve({ root, ...(io.stateDir ? { stateDir: io.stateDir } : {}) }),
})
const tool = (name: string) => tools.find((item) => item.name === name)!
@@ -534,7 +558,7 @@ export async function main(argv: readonly string[], io: MainIO): Promise<number>
...server.install.packages.map((pkg) => `${pkg.name}@${pkg.version} ${pkg.integrity}`),
"Verified against the pinned sha512; installed with scripts disabled.",
]
if (!(await confirmWithCode(io.confirm, `Install the ${id} language server`, lines))) return 1
if (!(await confirmHuman(io.confirm, `Install the ${id} language server`, lines, io.stateDir))) return 1
await recordConsent(id!, true, io.stateDir)
io.print(`Installed ${await installServer(server, io.stateDir ? { stateDir: io.stateDir } : {})}`)
return 0
packages/cli/src/mcp.ts
+2 -2
View File
@@ -4,7 +4,7 @@
// human-only. Caller identity is the `agent` argument (or ES_AGENT), which the
// harness adapter sets per agent; over plain MCP it is ADVISORY, and the
// capability matrix says so.
import { esTools, Factory, gateRunner } from "@heretek-ai/es-core"
import { stateDir as defaultStateDir, esTools, Factory, gateRunner } from "@heretek-ai/es-core"
import { VERSION } from "./version.ts"
export const PROTOCOL_VERSION = "2025-06-18"
@@ -23,7 +23,7 @@ export function createMcpServer(options: McpOptions) {
gates: gateRunner(options.stateDir ? { stateDir: options.stateDir } : {}),
...(options.stateDir ? { stateDir: options.stateDir } : {}),
})
const tools = esTools({ root: options.root, factory, stateDir: options.stateDir ?? "" })
const tools = esTools({ root: options.root, factory, stateDir: options.stateDir ?? defaultStateDir() })
const byName = new Map(tools.map((tool) => [tool.name.replace(/^es_/, ""), tool]))
const listed = [...byName].map(([name, tool]) => ({
name,
packages/cli/src/tty.ts
+58 -12
View File
@@ -1,14 +1,19 @@
// Human confirmation for the CLI half of the human-only channel. Refuses
// without an interactive terminal on both stdin and stdout, and requires the
// human to type back a random code shown on screen (an agent piping "y"
// cannot satisfy it).
// without an interactive terminal on both stdin and stdout. Since 1.1.1 every
// human action is confirmed with the human's passphrase, read with echo off,
// which unlocks the human key (core approval/keystore.ts): an agent that
// drives this prompt through a PTY can read everything on screen, but not the
// passphrase. Signing actions use the unlocked key; the rest use the unlock
// as proof that the human is at the keyboard.
import { createInterface } from "node:readline"
import { confirmationCode } from "@heretek-ai/es-core"
import { HumanKeyError, type HumanSigner, unlockHumanKey } from "@heretek-ai/es-core"
export interface ConfirmIO {
readonly interactive: boolean
write(text: string): void
readLine(prompt: string): Promise<string>
/** Read a line without echoing it (passphrases). */
readSecret(prompt: string): Promise<string>
}
export const terminalIO = (): ConfirmIO => ({
@@ -22,6 +27,34 @@ export const terminalIO = (): ConfirmIO => ({
resolve(answer)
})
}),
readSecret: (prompt) =>
new Promise((resolve) => {
const stdin = process.stdin
const wasRaw = stdin.isRaw
process.stdout.write(prompt)
stdin.setRawMode?.(true)
stdin.setEncoding("utf8")
stdin.resume()
let value = ""
const done = () => {
stdin.off("data", onData)
stdin.setRawMode?.(wasRaw ?? false)
stdin.pause()
process.stdout.write("\n")
resolve(value)
}
const onData = (chunk: string) => {
for (const char of chunk) {
if (char === "\r" || char === "\n" || char === "\u0004") return done()
if (char === "\u0003") {
value = ""
return done()
}
value = char === "\u007f" || char === "\b" ? value.slice(0, -1) : value + char
}
}
stdin.on("data", onData)
}),
})
export class NotInteractive extends Error {
@@ -30,13 +63,26 @@ export class NotInteractive extends Error {
}
}
/** Show `lines`, then ask the human to type a fresh code. Returns true only on an exact match. */
export async function confirmWithCode(io: ConfirmIO, action: string, lines: readonly string[]): Promise<boolean> {
/**
* Show what the action does, then ask for the passphrase. Returns the unlocked
* human key, or undefined when the human cancels or the passphrase is wrong
* (the reason is written to the terminal).
*/
export async function confirmHuman(
io: ConfirmIO,
action: string,
lines: readonly string[],
stateDir?: string,
): Promise<HumanSigner | undefined> {
if (!io.interactive) throw new NotInteractive(action)
const code = confirmationCode()
io.write(
`\n${action}\n${"─".repeat(Math.min(72, action.length + 8))}\n${lines.join("\n")}\n\nType ${code} to confirm (anything else cancels): `,
)
const answer = (await io.readLine("")).trim()
return answer === code
io.write(`\n${action}\n${"─".repeat(Math.min(72, action.length + 8))}\n${lines.join("\n")}\n\n`)
const passphrase = await io.readSecret("Your Epistemic Swarm passphrase (empty cancels): ")
if (!passphrase) return undefined
try {
return await unlockHumanKey(passphrase, stateDir)
} catch (error) {
if (!(error instanceof HumanKeyError)) throw error
io.write(`${error.message}\n`)
return undefined
}
}
packages/cli/src/version.ts
+1 -1
View File
@@ -1,2 +1,2 @@
/** The `es` CLI version. Single source: kept equal to the package version by test/cli.test.ts. */
export const VERSION = "1.1.0"
export const VERSION = "1.1.1"
packages/cli/test/cli.test.ts
+87 -16
View File
@@ -2,14 +2,16 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"
import { tmpdir } from "node:os"
import path from "node:path"
import { exists, Factory, factoryLayout, gateRunner, readApproval } from "@heretek-ai/es-core"
import { exists, Factory, factoryLayout, gateRunner, readApproval, sealHumanKey } from "@heretek-ai/es-core"
import { parseArgs, parseExpiry } from "../src/args.ts"
import { type HarnessDriver, runHeadless } from "../src/headless.ts"
import { main } from "../src/main.ts"
import { createMcpServer } from "../src/mcp.ts"
import { type ConfirmIO, confirmWithCode, NotInteractive } from "../src/tty.ts"
import { type ConfirmIO, confirmHuman, NotInteractive } from "../src/tty.ts"
import { VERSION } from "../src/version.ts"
const PASSPHRASE = "test-passphrase-1234"
let root: string
let state: string
// The global config layer lives under XDG_CONFIG_HOME: point it into the
@@ -28,6 +30,7 @@ beforeEach(async () => {
await writeFile(path.join(root, "README.md"), "# x\n")
await git("add", "-A")
await git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "init")
await sealHumanKey(PASSPHRASE, state)
})
afterEach(async () => {
if (realXdgConfig === undefined) delete process.env.XDG_CONFIG_HOME
@@ -36,17 +39,18 @@ afterEach(async () => {
await rm(state, { recursive: true, force: true })
})
/** A human at a terminal who types back whatever code is shown (or `answer`). */
const human = (answer?: string): ConfirmIO & { output: string[] } => {
/** A human at a terminal who types the passphrase with echo off (or `answer`). */
const human = (answer: string = PASSPHRASE): ConfirmIO & { output: string[] } => {
const output: string[] = []
return {
interactive: true,
output,
write: (text) => output.push(text),
readLine: async () => answer ?? /Type (\S+) to confirm/.exec(output.join(""))?.[1] ?? "",
readLine: async () => "",
readSecret: async () => answer,
}
}
const pipe: ConfirmIO = { interactive: false, write: () => {}, readLine: async () => "y" }
const pipe: ConfirmIO = { interactive: false, write: () => {}, readLine: async () => "y", readSecret: async () => "" }
async function run(argv: string[], confirm: ConfirmIO = pipe) {
const printed: string[] = []
@@ -79,10 +83,11 @@ describe("args", () => {
})
describe("human-only confirmation", () => {
test("needs a TTY and the exact typed code", async () => {
await expect(confirmWithCode(pipe, "x", [])).rejects.toBeInstanceOf(NotInteractive)
expect(await confirmWithCode(human("y"), "x", [])).toBe(false)
expect(await confirmWithCode(human(), "x", [])).toBe(true)
test("needs a TTY and the correct passphrase", async () => {
await expect(confirmHuman(pipe, "x", [], state)).rejects.toBeInstanceOf(NotInteractive)
expect(await confirmHuman(human("wrong-passphrase"), "x", [], state)).toBeUndefined()
expect(await confirmHuman(human(""), "x", [], state)).toBeUndefined()
expect(await confirmHuman(human(), "x", [], state)).toBeDefined()
})
test("es approve refuses without a terminal and records nothing", async () => {
@@ -102,8 +107,20 @@ describe("human-only confirmation", () => {
expect((await factory.read())?.stage).toBe("RESEARCH")
})
test("a wrong code cancels the waiver", async () => {
const result = await run(["waive", "lint/*", "--reason", "legacy code under migration"], human("nope"))
test("a wrong or empty passphrase records nothing (a PTY cannot approve)", async () => {
await grilledFrontier()
for (const answer of ["", "wrong-passphrase"]) {
const result = await run(["approve", "frontier"], human(answer))
expect(result.code).toBe(1)
expect(await readApproval(root, "frontier")).toBeUndefined()
}
const granted = await run(["approve", "frontier"], human())
expect(granted.code).toBe(0)
expect((await readApproval(root, "frontier"))?.channel).toBe("cli")
})
test("a wrong passphrase cancels the waiver", async () => {
const result = await run(["waive", "lint/*", "--reason", "legacy code under migration"], human("nope-wrong-pass"))
expect(result.code).toBe(1)
const granted = await run(
["waive", "lint/*", "--reason", "legacy code under migration", "--expires", "2d"],
@@ -114,12 +131,47 @@ describe("human-only confirmation", () => {
})
})
describe("es key", () => {
test("seal asks twice, refuses short or mismatched passphrases, and status shows the fingerprint", async () => {
const { humanKeyId } = await import("@heretek-ai/es-core")
// beforeEach already sealed `state`; use a fresh dir for seal flows.
const fresh = await mkdtemp(path.join(tmpdir(), "es-key-"))
try {
const runIn = async (argv: string[], confirm: ConfirmIO) => {
const printed: string[] = []
const code = await main(argv, { print: (text) => printed.push(text), confirm, cwd: root, stateDir: fresh })
return { code, out: printed.join("\n") }
}
const seq = (answers: string[]): ConfirmIO & { output: string[] } => {
const output: string[] = []
const queue = [...answers]
return {
interactive: true,
output,
write: (text) => output.push(text),
readLine: async () => "",
readSecret: async () => queue.shift() ?? "",
}
}
expect((await runIn(["key", "seal"], seq(["short", "short"]))).code).toBe(1)
expect((await runIn(["key", "seal"], seq(["long-enough-pass", "different-pass"]))).code).toBe(1)
expect(await humanKeyId(fresh)).toBeUndefined()
expect((await runIn(["key", "seal"], seq(["a-long-test-passphrase", "a-long-test-passphrase"]))).code).toBe(0)
expect(await humanKeyId(fresh)).toBeDefined()
expect((await runIn(["key", "status"], seq([]))).out).toContain("Human key")
} finally {
await rm(fresh, { recursive: true, force: true })
}
expect((await run(["key", "status"], human())).out).toContain("Human key")
})
})
describe("research brief and retractions", () => {
async function researched() {
const { buildDossier, normalizeClaim, researchSourcesDir, SourceCache, writeDossier } = await import(
"@heretek-ai/es-core"
)
const cache = new SourceCache(researchSourcesDir(root))
const cache = new SourceCache(researchSourcesDir(root), state)
const source = await cache.put({
url: "https://x.test/cli",
text: "The CLI exports a signed brief that anyone holding the key can verify.",
@@ -143,7 +195,9 @@ describe("research brief and retractions", () => {
await researched()
expect((await run(["research", "export", "--help"])).code).toBe(0)
expect(await exists(factoryLayout(root).researchBrief)).toBe(false)
const exported = await run(["research", "export"])
expect((await run(["research", "export"])).code).toBe(3)
expect(await exists(factoryLayout(root).researchBrief)).toBe(false)
const exported = await run(["research", "export"], human())
expect(exported.code).toBe(0)
expect(exported.out).toContain("Exported .factory/research/brief.pcrb.json")
const verified = await run(["research", "verify-brief", ".factory/research/brief.pcrb.json"])
@@ -458,8 +512,25 @@ describe("es audit and es scout (headless jobs)", () => {
const prompts: string[] = []
DRIVERS.fake = seatDriver(async (prompt) => {
prompts.push(prompt)
for (const agent of ["es-auditor-thesis", "es-auditor-antithesis"])
await verdict.execute({ audit: "audit-01", verdict: "pass", findings: [], notes: "ok" }, { agent })
// The thesis maps an invariant that holds; the red team found nothing.
const mapped = {
kind: "invariant",
title: "Queries go through the db handle",
severity: "info",
file: "src/auth.js",
lines: [3, 3],
excerpt: "return db.query(sql)",
detail: "All database access in this module uses the passed-in db handle.",
holds: true,
}
await verdict.execute(
{ audit: "audit-01", verdict: "pass", findings: [mapped], notes: "ok" },
{ agent: "es-auditor-thesis" },
)
await verdict.execute(
{ audit: "audit-01", verdict: "pass", findings: [], notes: "tried injection, races, secrets" },
{ agent: "es-auditor-antithesis" },
)
})
try {
const driven = await run(["audit", "src", "--driver", "fake"])
packages/core/assets/prompts/auditor-antithesis.md
+2 -2
View File
@@ -1,6 +1,6 @@
---
id: auditor-antithesis
version: 1
version: 2
seat: auditor-antithesis
description: Code-audit antithesis seat — red-teams the target with CWE-mapped, line-pointed exploits.
---
@@ -18,7 +18,7 @@ Assume the code is wrong, then prove it. For each defect, record a `vulnerabilit
Record lesser problems as `debt`.
## The line-pointer law
Every finding cites `file` (relative to the target's root), `lines` `[start, end]` and an `excerpt`: verbatim code from those lines, at least 12 characters. `es_audit_verdict` checks every finding against the file on disk. A hallucinated file, a line range past the end, or an excerpt that is not in those lines refuses the whole verdict, and nothing is recorded. Fix the reference and call again. A defect you cannot point at is not a finding.
Every finding cites `file` (relative to the target's root), `lines` `[start, end]` and an `excerpt`: one contiguous span of verbatim code from those lines, at least 12 characters (no ellipses; cite at most 60 lines). `es_audit_verdict` checks every finding against the file on disk. A hallucinated file, a line range past the end, or an excerpt that is not in those lines refuses the whole verdict, and nothing is recorded. Fix the reference and call again. A defect you cannot point at is not a finding.
## Verdict
- `fail` when at least one vulnerability is real: give its severity, its CWE, the exploit sequence and a concrete remediation.
packages/core/assets/prompts/auditor-thesis.md
+3 -3
View File
@@ -1,6 +1,6 @@
---
id: auditor-thesis
version: 1
version: 2
seat: auditor-thesis
description: Code-audit thesis seat — maps the target's invariants with line-pointer evidence.
---
@@ -16,10 +16,10 @@ Build the case for what the code guarantees, and record each guarantee as an `in
Set `holds: true` on an invariant the code keeps, and `holds: false` on one it breaks. A broken invariant is a defect, so name its CWE when one applies.
## The line-pointer law
Every finding cites `file` (relative to the target's root), `lines` `[start, end]` and an `excerpt`: verbatim code from those lines, at least 12 characters. `es_audit_verdict` checks every finding against the file on disk. A hallucinated file, a line range past the end, or an excerpt that is not in those lines refuses the whole verdict, and nothing is recorded. Fix the reference and call again. Never generalise ("this might have races"); point at the code.
Every finding cites `file` (relative to the target's root), `lines` `[start, end]` and an `excerpt`: one contiguous span of verbatim code from those lines, at least 12 characters (no ellipses; cite at most 60 lines). `es_audit_verdict` checks every finding against the file on disk. A hallucinated file, a line range past the end, or an excerpt that is not in those lines refuses the whole verdict, and nothing is recorded. Fix the reference and call again. Never generalise ("this might have races"); point at the code.
## Verdict
- `pass` when the invariants you mapped hold.
- `pass` when the invariants you mapped hold. A pass carries at least one witnessed invariant with `holds: true`.
- `fail` when at least one invariant does not hold (`holds: false`), with what breaks it and a concrete remediation.
- Notes: numbered, terse, each pointing at a finding.
packages/core/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@heretek-ai/es-core",
"version": "1.1.0",
"version": "1.1.1",
"description": "Epistemic Swarm core: factory state machine, mechanical gates, audit log, trust, research, brainstorm, harvest, LSP and hook/agent compilers.",
"type": "module",
"license": "Apache-2.0",
packages/core/src/approval/keystore.ts
+180 -45
View File
@@ -1,72 +1,207 @@
// The signing key that makes approval and waiver records unforgeable by
// anything that cannot read the user-global state dir (agents are denied that
// dir by policy). Created on first use with mode 0600.
import { createHmac, randomBytes, timingSafeEqual } from "node:crypto"
// The human key (1.1.1). Approvals, waivers, trust entries and research briefs
// are signed with an Ed25519 key whose private half is sealed by a passphrase
// only the human knows (scrypt + AES-256-GCM). The plugin and the verifiers
// check signatures with the public half, so nothing needs the passphrase to
// verify, and an agent that reads the sealed file, or drives the CLI through
// a PTY, still cannot sign. 1.0/1.1.0 records were MACed with a plain HMAC key
// that agents could read; they are refused and must be re-recorded.
import {
createCipheriv,
createDecipheriv,
createHash,
createPrivateKey,
createPublicKey,
generateKeyPairSync,
type KeyObject,
randomBytes,
scrypt,
sign,
verify,
} from "node:crypto"
import { mkdir, open, readFile } from "node:fs/promises"
import path from "node:path"
import { stateDir } from "../layout.ts"
import { canonicalJson } from "../util/hash.ts"
async function loadOrCreateKey(dir: string): Promise<Buffer> {
const file = path.join(dir, "key")
const existing = await readKey(file)
if (existing) return existing
await mkdir(dir, { recursive: true, mode: 0o700 })
const key = randomBytes(32)
try {
const handle = await open(file, "wx", 0o600)
await handle.writeFile(key.toString("hex"))
await handle.close()
return key
} catch (error: any) {
// Another process created it first: use theirs.
if (error?.code === "EEXIST") {
const raced = await readKey(file)
if (raced) return raced
}
throw error
}
const SEALED_FILE = "human.key"
const PUBLIC_FILE = "human.pub"
/** A passphrase shorter than this is refused at seal time. */
export const MIN_PASSPHRASE = 10
// scrypt cost: 2^16 × 8 × 1 needs 64 MiB and about a quarter second per unlock.
const SCRYPT = { N: 2 ** 16, r: 8, p: 1, maxmem: 128 * 1024 * 1024 } as const
export class HumanKeyError extends Error {}
/** A signature on a human record. */
export interface RecordSignature {
readonly alg: "ed25519"
readonly keyId: string
readonly sig: string
}
async function readKey(file: string): Promise<Buffer | undefined> {
/** The unlocked human key, held only while a human action runs. */
export interface HumanSigner {
readonly keyId: string
sign(record: Record<string, unknown>): RecordSignature
}
interface SealedKey {
readonly version: 1
readonly alg: "ed25519"
readonly keyId: string
readonly kdf: {
readonly name: "scrypt"
readonly N: number
readonly r: number
readonly p: number
readonly salt: string
}
readonly cipher: { readonly name: "aes-256-gcm"; readonly iv: string; readonly tag: string }
readonly data: string
}
/** What a record's signature covers: everything but the signature (and a legacy mac). */
const signedBody = (record: Record<string, unknown>) => {
const { signature: _signature, mac: _mac, ...body } = record
return Buffer.from(canonicalJson(body))
}
const keyIdOf = (publicKey: KeyObject) =>
createHash("sha256")
.update(publicKey.export({ type: "spki", format: "der" }))
.digest("hex")
.slice(0, 16)
const derive = (passphrase: string, salt: Buffer, params: { N: number; r: number; p: number }) =>
new Promise<Buffer>((resolve, reject) =>
scrypt(passphrase.normalize("NFKC"), salt, 32, { ...params, maxmem: SCRYPT.maxmem }, (error, key) =>
error ? reject(error) : resolve(key),
),
)
async function readText(file: string): Promise<string | undefined> {
try {
return Buffer.from((await readFile(file, "utf8")).trim(), "hex")
return await readFile(file, "utf8")
} catch (error: any) {
if (error?.code === "ENOENT") return undefined
throw error
}
}
/** Whether this machine already holds a signing key (verifiers must not mint one). */
export async function hasSigningKey(dir = stateDir()): Promise<boolean> {
return (await readKey(path.join(dir, "key"))) !== undefined
/** Whether this machine holds a sealed human key (`es key seal` has run). */
export async function hasHumanKey(dir = stateDir()): Promise<boolean> {
return (await readText(path.join(dir, SEALED_FILE))) !== undefined
}
/** Public fingerprint of the signing key, so a verifier can tell "not my key" from "tampered". */
export async function signingKeyId(dir = stateDir()): Promise<string> {
const key = await loadOrCreateKey(dir)
return createHmac("sha256", key).update("epistemic-swarm key id").digest("hex").slice(0, 16)
async function publicKey(dir: string): Promise<KeyObject | undefined> {
const pem = await readText(path.join(dir, PUBLIC_FILE))
return pem === undefined ? undefined : createPublicKey(pem)
}
export async function signRecord(record: Record<string, unknown>, dir = stateDir()): Promise<string> {
const key = await loadOrCreateKey(dir)
const { mac: _ignored, ...body } = record
return createHmac("sha256", key).update(canonicalJson(body)).digest("hex")
/** Public fingerprint of the human key, or undefined when none is sealed yet. */
export async function humanKeyId(dir = stateDir()): Promise<string | undefined> {
const key = await publicKey(dir)
return key ? keyIdOf(key) : undefined
}
export async function verifyRecordMac(record: Record<string, unknown>, dir = stateDir()): Promise<boolean> {
if (typeof record.mac !== "string") return false
const expected = await signRecord(record, dir)
return timingSafeEqualHex(expected, record.mac)
const NO_KEY =
"no passphrase-sealed human key on this machine: run `es key seal` once at a terminal (1.1.1 replaced the old approval key; earlier approvals, waivers and trust must be recorded again)"
/** Create the human key pair, sealed by `passphrase`. Refuses if one exists (never adopts a planted file). */
export async function sealHumanKey(passphrase: string, dir = stateDir()): Promise<{ keyId: string }> {
if (passphrase.length < MIN_PASSPHRASE)
throw new HumanKeyError(`the passphrase must be at least ${MIN_PASSPHRASE} characters`)
await mkdir(dir, { recursive: true, mode: 0o700 })
const pair = generateKeyPairSync("ed25519")
const salt = randomBytes(16)
const iv = randomBytes(12)
const key = await derive(passphrase, salt, SCRYPT)
const cipher = createCipheriv("aes-256-gcm", key, iv)
const data = Buffer.concat([cipher.update(pair.privateKey.export({ type: "pkcs8", format: "der" })), cipher.final()])
const keyId = keyIdOf(pair.publicKey)
const sealed: SealedKey = {
version: 1,
alg: "ed25519",
keyId,
kdf: { name: "scrypt", N: SCRYPT.N, r: SCRYPT.r, p: SCRYPT.p, salt: salt.toString("base64") },
cipher: { name: "aes-256-gcm", iv: iv.toString("base64"), tag: cipher.getAuthTag().toString("base64") },
data: data.toString("base64"),
}
// Exclusive creates: a key that already exists (or was planted) is never replaced silently.
for (const [name, text] of [
[SEALED_FILE, `${JSON.stringify(sealed, null, 2)}\n`],
[PUBLIC_FILE, pair.publicKey.export({ type: "spki", format: "pem" }).toString()],
] as const) {
try {
const handle = await open(path.join(dir, name), "wx", 0o600)
await handle.writeFile(text)
await handle.close()
} catch (error: any) {
if (error?.code === "EEXIST")
throw new HumanKeyError(
`${path.join(dir, name)} already exists: this machine already has a human key (remove ${SEALED_FILE} and ${PUBLIC_FILE} by hand to replace it, then re-record approvals)`,
)
throw error
}
}
return { keyId }
}
function timingSafeEqualHex(a: string, b: string): boolean {
const left = Buffer.from(a)
const right = Buffer.from(b)
return left.length === right.length && timingSafeEqual(left, right)
/** Unlock the human key with its passphrase. A wrong passphrase fails the AES-GCM tag check. */
export async function unlockHumanKey(passphrase: string, dir = stateDir()): Promise<HumanSigner> {
const text = await readText(path.join(dir, SEALED_FILE))
if (text === undefined) throw new HumanKeyError(NO_KEY)
const sealed = JSON.parse(text) as SealedKey
const pub = await publicKey(dir)
if (!pub || keyIdOf(pub) !== sealed.keyId)
throw new HumanKeyError(`${PUBLIC_FILE} does not match the sealed key (tampered or partial); re-seal by hand`)
let privateKey: KeyObject
try {
const key = await derive(passphrase, Buffer.from(sealed.kdf.salt, "base64"), sealed.kdf)
const decipher = createDecipheriv("aes-256-gcm", key, Buffer.from(sealed.cipher.iv, "base64"))
decipher.setAuthTag(Buffer.from(sealed.cipher.tag, "base64"))
const der = Buffer.concat([decipher.update(Buffer.from(sealed.data, "base64")), decipher.final()])
privateKey = createPrivateKey({ key: der, format: "der", type: "pkcs8" })
} catch {
throw new HumanKeyError("wrong passphrase")
}
return {
keyId: sealed.keyId,
sign: (record) => ({
alg: "ed25519",
keyId: sealed.keyId,
sig: sign(null, signedBody(record), privateKey).toString("base64"),
}),
}
}
/** A short code a human must type back to confirm an approval (CLI/TUI). */
/** Whether `record.signature` is a valid signature by this machine's human key. */
export async function verifyRecordSignature(record: Record<string, unknown>, dir = stateDir()): Promise<boolean> {
const signature = record.signature as Partial<RecordSignature> | undefined
if (signature?.alg !== "ed25519" || typeof signature.sig !== "string") return false
const pub = await publicKey(dir)
if (!pub || signature.keyId !== keyIdOf(pub)) return false
try {
return verify(null, signedBody(record), pub, Buffer.from(signature.sig, "base64"))
} catch {
return false
}
}
/** Why a record does not verify, phrased for a human (legacy HMAC records get the re-record hint). */
export async function signatureProblem(
record: Record<string, unknown>,
what: string,
dir = stateDir(),
): Promise<string | undefined> {
if (await verifyRecordSignature(record, dir)) return undefined
if (typeof record.mac === "string" && record.signature === undefined)
return `${what} was signed by the pre-1.1.1 approval key, which agents could read; record it again (a human, at a terminal, with the passphrase)`
if (!(await hasHumanKey(dir))) return `${what} cannot be verified: ${NO_KEY}`
return `${what} is not signed by this machine's human key (forged, tampered or foreign)`
}
/** A short code a human must type back to confirm a non-signing action (CLI/TUI). */
export function confirmationCode(): string {
const alphabet = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789"
const bytes = randomBytes(6)
packages/core/src/approval/record.ts
+11 -12
View File
@@ -1,7 +1,8 @@
// Human-only approvals. Callers never name the artifact: the subject is
// derived from the factory layout for the stage, validated, hashed and signed.
// Only the CLI (TTY + typed code) and the TUI dialog call recordApproval; no
// agent tool or MCP tool does. Transitions call verifyApproval, which re-hashes
// derived from the factory layout for the stage, validated, hashed and signed
// with the human key, which only the human's passphrase unlocks (1.1.1). Only
// the CLI (a TTY + the passphrase) calls recordApproval; no agent tool, MCP
// tool or RPC does. Transitions call verifyApproval, which re-hashes
// the current artifacts, so editing a spec after approval blocks the build.
import { readFile } from "node:fs/promises"
import { hostname, userInfo } from "node:os"
@@ -15,7 +16,7 @@ import { type Roadmap, RoadmapSchema } from "../schema/roadmap.ts"
import { rebaseline } from "../trust/control.ts"
import { readJson, writeJson } from "../util/fs.ts"
import { sha256 } from "../util/hash.ts"
import { signRecord, verifyRecordMac } from "./keystore.ts"
import { type HumanSigner, signatureProblem } from "./keystore.ts"
export class ApprovalError extends Error {}
@@ -108,13 +109,14 @@ export interface RecordApprovalInput {
/** Defaults to the OS user running the CLI/TUI. */
readonly approvedBy?: string
readonly notes?: string
readonly stateDir?: string
/** The unlocked human key. */
readonly signer: HumanSigner
}
export async function recordApproval(root: string, input: RecordApprovalInput): Promise<Approval> {
const { subject, spendCeilingUSD } = await approvalSubject(root, input.stage)
const unsigned = {
version: 1 as const,
version: 2 as const,
stage: input.stage,
subject,
approvedBy: input.approvedBy ?? userInfo().username,
@@ -125,7 +127,7 @@ export async function recordApproval(root: string, input: RecordApprovalInput):
...(spendCeilingUSD !== undefined ? { spendCeilingUSD } : {}),
...(input.notes ? { notes: input.notes } : {}),
}
const record = ApprovalSchema.parse({ ...unsigned, mac: await signRecord(unsigned, input.stateDir ?? stateDir()) })
const record = ApprovalSchema.parse({ ...unsigned, signature: input.signer.sign(unsigned) })
await writeJson(factoryLayout(root).approval(input.stage), record)
await appendAuditEntry(root, {
actor: `human:${record.approvedBy}`,
@@ -149,13 +151,10 @@ export async function verifyApproval(
const raw = await readJson<Record<string, unknown>>(factoryLayout(root).approval(stage)).catch(() => undefined)
if (!raw)
return { ok: false, reason: `no ${stage} approval; a human must run \`es approve ${stage}\` or /es-approve` }
const unsigned = await signatureProblem(raw, `the ${stage} approval`, options.stateDir ?? stateDir())
if (unsigned) return { ok: false, reason: `${unsigned} (\`es approve ${stage}\`)` }
const parsed = ApprovalSchema.safeParse(raw)
if (!parsed.success) return { ok: false, reason: `the ${stage} approval record is malformed` }
if (!(await verifyRecordMac(raw, options.stateDir ?? stateDir())))
return {
ok: false,
reason: `the ${stage} approval record is not signed by this machine's approval key (forged or foreign)`,
}
for (const item of parsed.data.subject) {
let text: string
try {
packages/core/src/capabilities.ts
+12 -10
View File
@@ -64,8 +64,8 @@ export const CAPABILITY_MATRIX: Readonly<Record<Harness, readonly CapabilityRow[
permissions: {
support: "enforced",
detail:
"permission.evaluate + tool.execute.before enforce the write/read/shell policy; control files (incl. the evidence cache) denied; research seats get no host websearch/webfetch.",
test: PLUGIN,
"permission.evaluate + tool.execute.before enforce the write/read/shell policy; control files (incl. the engine-sealed evidence cache) denied; research seats get no host websearch/webfetch. Every agent shell runs under bubblewrap by seat kind (cached-web seats offline); seats are refused without it and the user's agents fall back to the weaker argv-aware text policy.",
test: "packages/core/test/policy.test.ts",
},
hooks: {
support: "enforced",
@@ -79,7 +79,8 @@ export const CAPABILITY_MATRIX: Readonly<Record<Harness, readonly CapabilityRow[
},
subagents: {
support: "enforced",
detail: "Depth 1; hidden seats are unlisted but invocable by ID, and only from the seats allowed to spawn them.",
detail:
"Depth 1; hidden seats are unlisted but invocable by ID, and only from the seats allowed to spawn them. Factory seats are not offered Code Mode execute.",
test: PLUGIN,
},
compaction: {
@@ -90,7 +91,7 @@ export const CAPABILITY_MATRIX: Readonly<Record<Harness, readonly CapabilityRow[
websearch: {
support: "enforced",
detail:
"The configured provider registers as the host websearch; results are cached and citable. Research seats are cached-only: es_research_search/fetch (fresh hits served from the cache).",
"The configured provider registers as the host websearch; results are cached and citable. Research, scout and auditor seats are cached-only (es_research_search/fetch) under an offline (--unshare-net) sandbox; host web caching applies only in a project that already has .factory/, and the cache is engine-sealed (planted entries refused).",
test: "packages/opencode/test/research.test.ts",
},
lsp: {
@@ -106,8 +107,9 @@ export const CAPABILITY_MATRIX: Readonly<Record<Harness, readonly CapabilityRow[
},
research: {
support: "enforced",
detail: "Cache, quote verifier, auditor, providers and the websearch bridge.",
test: "packages/opencode/test/research.test.ts",
detail:
"Engine-sealed cache (#52), fragment-level quote verifier (every ellipsis fragment >= 12 chars; code excerpts one contiguous span of at most 60 lines), auditor, providers and the websearch bridge.",
test: "packages/core/test/research.test.ts",
},
brainstorm: {
support: "enforced",
@@ -128,19 +130,19 @@ export const CAPABILITY_MATRIX: Readonly<Record<Harness, readonly CapabilityRow[
claims: {
support: "enforced",
detail:
"Witnessed claims and dossiers: the evidence cache, dossiers and the claim ledger are tool-only; a forged source is denied on the host.",
test: PLUGIN,
"Witnessed claims and dossiers: the evidence cache, dossiers and the claim ledger are tool-only; planted cache entries are refused (engine seal) and every VERIFIED quote and code span is re-checked against the bytes on disk.",
test: "packages/core/test/research.test.ts",
},
audit: {
support: "enforced",
detail:
"Code-audit pair: only auditor seats record verdicts; every finding is witnessed on disk (a hallucinated line is refused); verdicts block.",
"Code-audit pair: only auditor seats record verdicts; every finding is witnessed on disk (a hallucinated line is refused); quote fragments carry >= 12 chars each and a thesis pass needs a witnessed invariant; verdicts block.",
test: "packages/opencode/test/fires.test.ts",
},
scout: {
support: "enforced",
detail:
"OSS scout: cached-only web, fail-closed license verdicts computed by core (adopt only for verified permissive licenses), OSV advisories cited.",
"OSS scout: cached-only web under an offline (--unshare-net) sandbox, fail-closed license verdicts computed by core (adopt only for verified permissive licenses), OSV advisories cited.",
test: "packages/opencode/test/fires.test.ts",
},
}),
packages/core/src/claims/brief.ts
+14 -17
View File
@@ -1,18 +1,19 @@
// Proof-Carrying Research Brief, ported from legacy runner/pcrb.py and
// runner/pcrb_verify.py: the research report, its claims (ranked, each with a
// fresh witness) and the cached sources they quote, bound by a manifest of
// member hashes and an HMAC from the user-global signing key. The verifier
// member hashes and an Ed25519 signature by the human key (1.1.1; the private
// half is sealed by the human's passphrase). The verifier
// runs the legacy four checks — source identity, manifest integrity,
// signature, quotes — and fixes three legacy holes:
// B3: a member deleted from the bundle went unnoticed (the manifest now lists
// every member key), and alg:"none" was accepted even with a key (the
// schema accepts only hmac-sha256);
// schema accepts only ed25519);
// B7: a prefix-hash citation bundled its source under the prefix (sources
// are keyed by the full hash only);
// B8: export and verify read keys in opposite precedence (one key: the
// state-dir key the approvals use).
// human key the approvals use).
import { readFile } from "node:fs/promises"
import { hasSigningKey, signingKeyId, signRecord, verifyRecordMac } from "../approval/keystore.ts"
import { type HumanSigner, humanKeyId, verifyRecordSignature } from "../approval/keystore.ts"
import { factoryLayout, stateDir } from "../layout.ts"
import { researchSourcesDir, SourceCache } from "../research/cache.ts"
import { verifyQuote } from "../research/quote.ts"
@@ -25,7 +26,8 @@ import { ClaimStore } from "./store.ts"
import { witnessClaim } from "./witness.ts"
export interface ExportBriefOptions {
readonly stateDir?: string
/** The unlocked human key (`es research export` asks for the passphrase). */
readonly signer: HumanSigner
/** Where to write the brief (default .factory/research/brief.pcrb.json). */
readonly out?: string
readonly now?: () => Date
@@ -45,10 +47,7 @@ const membersOf = (brief: Pick<Brief, "claims" | "sources">) =>
...Object.keys(brief.sources).map((hash) => `source:${hash}`),
].sort()
export async function exportBrief(
root: string,
options: ExportBriefOptions = {},
): Promise<{ file: string; brief: Brief }> {
export async function exportBrief(root: string, options: ExportBriefOptions): Promise<{ file: string; brief: Brief }> {
const layout = factoryLayout(root)
const synthesis = await readFile(layout.researchReport, "utf8").catch(() => {
throw new Error("No research report yet (.factory/research/REPORT.md).")
@@ -95,14 +94,10 @@ export async function exportBrief(
members: membersOf({ claims, sources }),
},
}
const dir = options.stateDir ?? stateDir()
const signed = options.signer.sign(signedBody(unsigned))
const brief = BriefSchema.parse({
...unsigned,
signature: {
alg: "hmac-sha256",
keyId: await signingKeyId(dir),
mac: await signRecord(signedBody(unsigned), dir),
},
signature: { alg: signed.alg, keyId: signed.keyId, sig: signed.sig },
})
const file = options.out ?? layout.researchBrief
await atomicWrite(file, `${JSON.stringify(brief, null, 2)}\n`)
@@ -174,8 +169,10 @@ export async function verifyBrief(
// 3. Signature: only a key this machine holds can vouch for the manifest.
const dir = options.stateDir ?? stateDir()
let signature: BriefVerification["checks"]["signature"] = "unverifiable"
if ((await hasSigningKey(dir)) && (await signingKeyId(dir)) === brief.signature.keyId)
signature = (await verifyRecordMac({ ...signedBody(brief), mac: brief.signature.mac }, dir)) ? "valid" : "invalid"
if ((await humanKeyId(dir)) === brief.signature.keyId)
signature = (await verifyRecordSignature({ ...signedBody(brief), signature: brief.signature }, dir))
? "valid"
: "invalid"
if (signature === "invalid") fail("signature", "the manifest signature does not match")
if (signature === "unverifiable" && !options.allowUnverifiable)
fail(
packages/core/src/claims/witness.ts
+11 -3
View File
@@ -3,12 +3,13 @@
// - a VERIFIED source quote against the content-addressed cache (tamper-
// refusing get + the one quote law, research/quote.ts);
// - a code location against the file on disk: inside the target root, the
// line range within the file, the excerpt verbatim in those lines.
// line range within the file and at most MAX_LOCATION_LINES long, the
// excerpt one contiguous verbatim span of those lines.
// Legacy never checked code locations at all (its "hallucinated line" check
// was the generic quote check), so the location rule is new in 1.1.
import path from "node:path"
import type { SourceCache } from "../research/cache.ts"
import { verifyQuote } from "../research/quote.ts"
import { verifyQuote, verifySpan } from "../research/quote.ts"
import type { Claim, ClaimLocation, Witness } from "../schema/claims.ts"
import { canonicalPath, relativeTo } from "../trust/paths.ts"
import { readRegularFile } from "../util/fs.ts"
@@ -22,6 +23,9 @@ export interface WitnessOptions {
readonly now?: () => Date
}
/** A code location names the code it means, not a whole file: at most this many lines. */
export const MAX_LOCATION_LINES = 60
type Check = { readonly ok: true } | { readonly ok: false; readonly reason: string }
const pass: Check = { ok: true }
const fail = (reason: string): Check => ({ ok: false, reason })
@@ -37,9 +41,13 @@ export async function checkLocation(root: string, location: ClaimLocation): Prom
const lines = text.split("\n")
if (text.endsWith("\n")) lines.pop()
const [start, end] = location.lines
if (end - start + 1 > MAX_LOCATION_LINES)
return fail(
`${location.file}#L${start}-L${end} spans ${end - start + 1} lines; cite at most ${MAX_LOCATION_LINES} lines around the code you mean`,
)
if (end > lines.length)
return fail(`${location.file}#L${start}-L${end} is past the end of the file (${lines.length} lines)`)
const quoted = verifyQuote(lines.slice(start - 1, end).join("\n"), location.excerpt)
const quoted = verifySpan(lines.slice(start - 1, end).join("\n"), location.excerpt)
if (!quoted.ok) return fail(`${location.file}#L${start}-L${end}: excerpt ${quoted.reason}`)
return pass
}
packages/core/src/codeaudit/findings.ts
+16 -2
View File
@@ -38,9 +38,23 @@ export async function witnessFindings(
export const isDefect = (finding: AuditFinding) =>
finding.kind === "vulnerability" || (finding.kind === "invariant" && finding.holds === false)
/** Why this verdict does not fit its findings (undefined when it does). */
export function verdictProblem(verdict: "pass" | "fail", findings: readonly AuditFinding[]): string | undefined {
/**
* Why this verdict does not fit its findings (undefined when it does). The
* thesis seat maps invariants, so its pass must show at least one that holds;
* the red team may pass with nothing found (its notes say what it tried).
*/
export function verdictProblem(
verdict: "pass" | "fail",
findings: readonly AuditFinding[],
seat?: "auditor-thesis" | "auditor-antithesis",
): string | undefined {
const defects = findings.filter(isDefect)
if (
verdict === "pass" &&
seat === "auditor-thesis" &&
!findings.some((finding) => finding.kind === "invariant" && finding.holds === true)
)
return "A thesis pass maps at least one invariant that holds, witnessed against the code (file, lines, verbatim excerpt)."
if (verdict === "fail" && defects.length === 0)
return "A fail needs at least one witnessed defect: a vulnerability (with its CWE) or an invariant that does not hold."
if (verdict === "pass" && defects.length > 0)
packages/core/src/codeaudit/ops.ts
+1 -1
View File
@@ -119,7 +119,7 @@ export function codeAuditTools(ops: OpsContext): EsToolDef[] {
.map((item) => `- ${item.pointer}: ${item.reason}`)
.join("\n")}`,
)
const problem = verdictProblem(verdict, parsed.data)
const problem = verdictProblem(verdict, parsed.data, seat)
if (problem) throw new ToolRefusal(problem)
// Evidence first, while the audited tree still exists: a passing phase
// audit merges the phase and removes its worktree.
packages/core/src/evals/fires.ts
+38 -5
View File
@@ -25,8 +25,15 @@ export interface FireGrade {
}
const overlaps = (a: readonly [number, number], b: readonly [number, number]) => a[0] <= b[1] && b[0] <= a[1]
/** Lines of context a finding may cite around a plant; a wider range is a guess, not a catch. */
const PLANT_SLACK = 3
const near = (finding: readonly [number, number], plant: readonly [number, number]) =>
overlaps(finding, plant) && finding[0] >= plant[0] - PLANT_SLACK && finding[1] <= plant[1] + PLANT_SLACK
/** Every plant must be caught by a vulnerability finding in its file, with one of its CWEs, overlapping its lines. */
/**
* Every plant must be caught by a vulnerability finding in its file, with one
* of its CWEs, citing the plant's lines (within PLANT_SLACK lines of context).
*/
export function gradeAuditFire(records: readonly AuditRecord[], plants: readonly PlantedDefect[]): FireGrade {
const findings = records.flatMap((record) => record.findings).filter((finding) => finding.kind === "vulnerability")
const caught: string[] = []
@@ -37,7 +44,7 @@ export function gradeAuditFire(records: readonly AuditRecord[], plants: readonly
finding.file.replace(/^\.\//, "") === plant.file &&
finding.cwe !== undefined &&
plant.cwe.includes(finding.cwe) &&
overlaps(finding.lines, plant.lines),
near(finding.lines, plant.lines),
)
if (hit) caught.push(plant.id)
else
@@ -49,20 +56,37 @@ export function gradeAuditFire(records: readonly AuditRecord[], plants: readonly
return { pass: failures.length === 0, caught, failures }
}
/** Every candidate's final verdict must match; a proposal the policy rejected must say why. */
/**
* Exactly the expected candidates, one verdict each, and every final verdict
* must match; a proposal the policy rejected must say why.
*/
export function gradeScoutFire(
result: ScoutResult,
expected: ReadonlyArray<{ name: string; verdict: string }>,
): FireGrade {
const caught: string[] = []
const failures: string[] = []
const matches = (verdict: ScoutResult["verdicts"][number], name: string) =>
verdict.name === name || verdict.candidate === name
// One row satisfies at most one expected candidate: a row whose name matches
// one expectation and whose candidate matches another must not cover both.
const claimed = new Map<number, string>()
result.verdicts.forEach((verdict, index) => {
const wants = expected.filter((want) => matches(verdict, want.name)).map((want) => want.name)
if (wants.length > 1) failures.push(`one verdict row matches several expected candidates: ${wants.join(", ")}`)
else if (wants.length === 1) claimed.set(index, wants[0]!)
})
for (const want of expected) {
const got = result.verdicts.find((verdict) => verdict.name === want.name || verdict.candidate === want.name)
if (!got) failures.push(`no verdict for ${want.name}`)
const rows = result.verdicts.filter((_, index) => claimed.get(index) === want.name)
const got = rows[0]
if (rows.length > 1) failures.push(`${rows.length} verdicts for ${want.name}`)
else if (!got) failures.push(`no verdict for ${want.name}`)
else if (got.verdict !== want.verdict) failures.push(`${want.name}: ${got.verdict}, expected ${want.verdict}`)
else if (got.proposal !== got.verdict && !got.downgraded) failures.push(`${want.name}: downgraded without a reason`)
else caught.push(want.name)
}
for (const verdict of result.verdicts)
if (!expected.some((want) => matches(verdict, want.name))) failures.push(`unexpected verdict for ${verdict.name}`)
return { pass: failures.length === 0, caught, failures }
}
@@ -83,6 +107,15 @@ export function gradeHarvestFire(
else caught.push(want.candidate)
}
}
const extra = [
...new Set(
matrix.rows
.flatMap((row) => row.cells)
.map((cell) => cell.candidate)
.filter((candidate) => !expected.some((want) => want.candidate === candidate)),
),
]
for (const candidate of extra) failures.push(`unexpected matrix cell for ${candidate}`)
return { pass: failures.length === 0, caught, failures }
}
packages/core/src/evals/grade.ts
+49 -9
View File
@@ -1,13 +1,13 @@
// Deterministic grader for the opt-in evals. A case names strings the model's
// text must (or must not) contain — case-insensitive, so checks describe intent
// without being brittle about casing — and tools it must (or must not) call.
// Error events always fail a case.
// without being brittle about casing — and tools it must complete (or must not
// even attempt). Error events always fail a case.
export interface EvalChecks {
readonly contains?: readonly string[]
readonly notContains?: readonly string[]
/** Tools the agent must call (by tool name). */
/** Tools the agent must call and that must complete (by tool name). */
readonly toolsUsed?: readonly string[]
/** Tools the agent must not call. */
/** Tools the agent must not call, not even in a refused attempt. */
readonly toolsNotUsed?: readonly string[]
}
@@ -26,34 +26,74 @@ export function gradeOutput(output: string, checks: EvalChecks = {}): EvalGrade
return { pass: failures.length === 0, failures }
}
export interface EvalTokens {
readonly input: number
readonly output: number
readonly reasoning: number
readonly cacheRead: number
readonly cacheWrite: number
}
/** What an `opencode run --format json` event stream said and did. */
export interface EvalTranscript {
readonly text: string
/** Every tool call, in order, whatever its outcome. */
readonly tools: readonly string[]
/** The calls that completed; a refused or failed call is not here. */
readonly completed: readonly string[]
readonly errors: readonly string[]
readonly steps: number
/** Spend reported by `step_finish` events (USD; 0 when the model has no price). */
readonly costUSD: number
readonly tokens: EvalTokens
}
const num = (value: unknown): number => (typeof value === "number" && Number.isFinite(value) ? value : 0)
export function readTranscript(events: readonly unknown[]): EvalTranscript {
const text: string[] = []
const tools: string[] = []
const completed: string[] = []
const errors: string[] = []
let steps = 0
let costUSD = 0
const tokens = { input: 0, output: 0, reasoning: 0, cacheRead: 0, cacheWrite: 0 }
for (const raw of events) {
const event = raw as { type?: string; part?: { text?: string; tool?: string }; error?: { message?: string } }
const event = raw as {
type?: string
part?: {
text?: string
tool?: string
state?: { status?: string }
cost?: unknown
tokens?: { input?: unknown; output?: unknown; reasoning?: unknown; cache?: { read?: unknown; write?: unknown } }
}
error?: { message?: string }
}
if (event?.type === "step_start") steps++
else if (event?.type === "text" && typeof event.part?.text === "string") text.push(event.part.text)
else if (event?.type === "tool_use" && typeof event.part?.tool === "string") tools.push(event.part.tool)
else if (event?.type === "error") errors.push(event.error?.message ?? "unknown error")
else if (event?.type === "tool_use" && typeof event.part?.tool === "string") {
tools.push(event.part.tool)
if (event.part.state?.status === "completed") completed.push(event.part.tool)
} else if (event?.type === "step_finish") {
costUSD += num(event.part?.cost)
tokens.input += num(event.part?.tokens?.input)
tokens.output += num(event.part?.tokens?.output)
tokens.reasoning += num(event.part?.tokens?.reasoning)
tokens.cacheRead += num(event.part?.tokens?.cache?.read)
tokens.cacheWrite += num(event.part?.tokens?.cache?.write)
} else if (event?.type === "error") errors.push(event.error?.message ?? "unknown error")
}
return { text: text.join("\n"), tools, errors, steps }
return { text: text.join("\n"), tools, completed, errors, steps, costUSD, tokens }
}
export function gradeTranscript(transcript: EvalTranscript, checks: EvalChecks = {}): EvalGrade {
const failures = [...gradeOutput(transcript.text, checks).failures]
for (const error of transcript.errors) failures.push(`error event: ${error}`)
for (const tool of checks.toolsUsed ?? [])
if (!transcript.tools.includes(tool)) failures.push(`did not call: ${tool}`)
if (!transcript.completed.includes(tool))
failures.push(`${transcript.tools.includes(tool) ? "called but did not complete" : "did not call"}: ${tool}`)
// An attempt at a forbidden tool counts even when the host refused it.
for (const tool of checks.toolsNotUsed ?? [])
if (transcript.tools.includes(tool)) failures.push(`must not call: ${tool}`)
return { pass: failures.length === 0, failures }
packages/core/src/evals/run.ts
+47 -6
View File
@@ -1,8 +1,9 @@
// Opt-in behaviour evals: each case is one real `opencode run` turn in an
// isolated workspace (its own project, HOME and XDG dirs) that loads the local
// plugin with the configured model. The JSON event stream carries no cost, so
// spend is bounded by a hard step cap (the process is killed at the first step
// over it) plus a timeout: cost ≤ steps × (context + output) × price.
// plugin with the configured model. Spend is bounded three ways: a hard step
// cap (the process is killed at the first step over it), a timeout, and an
// optional USD budget read from the stream's `step_finish` cost (killed at the
// first step that crosses it; 0 for a model with no configured price).
import { spawn } from "node:child_process"
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"
import { tmpdir } from "node:os"
@@ -22,6 +23,19 @@ export interface EvalCase {
readonly files?: Readonly<Record<string, string>>
/** A fire: after the turn, a deterministic grader scores what the seats left on disk. */
readonly fire?: FireSpec
/** This case's step cap, sized for a model that makes one call per step (see `caseStepCap`). */
readonly maxSteps?: number
}
/** Step cap for a case that sets no `maxSteps` of its own. */
export const DEFAULT_CASE_STEPS = 6
/**
* The step cap a case runs under: its own `maxSteps` (sized for a model that
* makes one call per step), bounded by the run-wide ceiling.
*/
export function caseStepCap(caseMaxSteps: number | undefined, ceiling: number): number {
return Math.min(caseMaxSteps ?? DEFAULT_CASE_STEPS, ceiling)
}
export interface EvalWorkspace {
@@ -78,6 +92,8 @@ export interface EvalRunOptions {
readonly binary: string
readonly maxSteps: number
readonly timeoutMs: number
/** Spend this case may use (USD); the run is killed once its reported cost exceeds it. */
readonly budgetUSD?: number
/** Base environment (provider API keys come from here). */
readonly env?: NodeJS.ProcessEnv
}
@@ -88,8 +104,12 @@ export interface EvalResult {
readonly exitCode: number | null
readonly capped: boolean
readonly timedOut: boolean
readonly overBudget: boolean
readonly transcript: EvalTranscript
readonly failures: readonly string[]
/** The raw event stream and stderr, kept as evidence for a failed case. */
readonly events: readonly unknown[]
readonly stderr: string
}
/** The child environment: the caller's env (for provider keys) with the workspace as PWD, HOME and XDG dirs. */
@@ -118,8 +138,10 @@ export async function runEvalCase(
)
const events: unknown[] = []
let steps = 0
let spent = 0
let capped = false
let timedOut = false
let overBudget = false
const timer = setTimeout(() => {
timedOut = true
child.kill("SIGKILL")
@@ -129,12 +151,19 @@ export async function runEvalCase(
for await (const line of createInterface({ input: child.stdout! })) {
if (!line.trim()) continue
try {
const event = JSON.parse(line) as { type?: string }
const event = JSON.parse(line) as { type?: string; part?: { cost?: unknown } }
events.push(event)
if (event.type === "step_start" && ++steps > options.maxSteps && !capped) {
capped = true
child.kill("SIGKILL")
}
if (event.type === "step_finish" && typeof event.part?.cost === "number" && Number.isFinite(event.part.cost)) {
spent += event.part.cost
if (options.budgetUSD !== undefined && spent > options.budgetUSD && !overBudget) {
overBudget = true
child.kill("SIGKILL")
}
}
} catch {
// not an event line
}
@@ -148,11 +177,23 @@ export async function runEvalCase(
const grade = gradeTranscript(transcript, testCase.checks)
const failures = [
...(capped ? [`step cap reached (${options.maxSteps})`] : []),
...(overBudget ? [`spend budget reached ($${spent.toFixed(4)} > $${options.budgetUSD})`] : []),
...(timedOut ? [`timed out after ${options.timeoutMs} ms`] : []),
...(exitCode !== 0 && !capped && !timedOut
...(exitCode !== 0 && !capped && !timedOut && !overBudget
? [`exit ${exitCode}${stderr.length ? `: ${stderr.join("").trim().slice(-300)}` : ""}`]
: []),
...grade.failures,
]
return { id: testCase.id, pass: failures.length === 0, exitCode, capped, timedOut, transcript, failures }
return {
id: testCase.id,
pass: failures.length === 0,
exitCode,
capped,
timedOut,
overBudget,
transcript,
failures,
events,
stderr: stderr.join(""),
}
}
packages/core/src/factory/machine.ts
+20 -7
View File
@@ -33,7 +33,7 @@ import { researchCache } from "../research/ops.ts"
import { type Frontier, FrontierSchema } from "../schema/frontier.ts"
import { type AcceptanceCriterion, parseGoalMarkdown } from "../schema/goal.ts"
import { RoadmapSchema } from "../schema/roadmap.ts"
import { rebaseline, verifyControl } from "../trust/control.ts"
import { rebaseline, repinControl, verifyControl } from "../trust/control.ts"
import { checkStopFile } from "../trust/stop.ts"
import { appendLine, exists, readJson, relativeInside, withLock, writeJson } from "../util/fs.ts"
import { sha256 } from "../util/hash.ts"
@@ -210,7 +210,7 @@ export class Factory {
}
/** Returns true when the state is (or just became) HALTED. */
private async guard(state: FactoryState, actor: string): Promise<boolean> {
private async guard(state: FactoryState, actor: string, options: { skipControl?: boolean } = {}): Promise<boolean> {
if (state.stage === "HALTED") return true
const halt = async (reason: string) => {
state.halt = { reason, at: this.now().toISOString(), from: state.stage }
@@ -220,8 +220,14 @@ export class Factory {
}
const stop = await checkStopFile(this.root)
if (stop.stopped) return halt(`STOP file: ${stop.reason}`)
const control = await verifyControl(this.root)
if (!control.clean) return halt(`control files changed outside a human action: ${control.violations.join("; ")}`)
// Skipped for async bookkeeping (recordSpend): spend events arrive at any
// moment, including mid-approval between the record write and its
// rebaseline, where a drift halt would be a false positive that persists
// (CI-only es_build_start refusal). Real transitions still enforce it.
if (!options.skipControl) {
const control = await verifyControl(this.root)
if (!control.clean) return halt(`control files changed outside a human action: ${control.violations.join("; ")}`)
}
if (state.spendCeilingUSD !== undefined && state.spend.usd >= state.spendCeilingUSD)
return halt(
`spend ceiling reached: $${state.spend.usd.toFixed(2)}${state.spend.estimated ? " (estimated)" : ""} of $${state.spendCeilingUSD}`,
@@ -290,8 +296,13 @@ export class Factory {
})
const problems = diffFrontier(previous, parsed.data)
if (problems.length) throw new FactoryError(`The frontier write was refused:\n- ${problems.join("\n- ")}`)
const before = await readJson<{ files?: Record<string, string | null> }>(this.layout.control).catch(
() => undefined,
)
await writeJson(this.layout.frontier, parsed.data)
await rebaseline(this.root, `agent:${agentId} via es_frontier_write`)
// Re-pin only the frontier file (#32): a concurrent hand edit to
// gates.json or config.json must still halt the run as drift.
await repinControl(this.root, ".factory/frontier.json", before?.files?.[".factory/frontier.json"] ?? null)
const counts = treeCounts(parsed.data)
await this.audit(`agent:${agentId}`, "frontier.write", {
settled: parsed.data.settled,
@@ -341,7 +352,7 @@ export class Factory {
const report = await readFile(this.layout.researchReport, "utf8").catch(() => undefined)
if (report === undefined)
throw new FactoryError("Research is not done: .factory/research/REPORT.md does not exist.")
const cache = researchCache(this.root)
const cache = researchCache(this.root, this.deps.stateDir)
const audit = await auditMarkdown(report, cache)
await writeJson(this.layout.researchCoverage, {
...audit.coverage,
@@ -1086,13 +1097,15 @@ export class Factory {
async recordSpend(usd: number, estimated: boolean): Promise<FactoryState | undefined> {
if (!(usd > 0)) return this.read()
// No run, nothing to charge; taking the lock would create .factory/runtime/ in any project.
if (!(await this.read())) return undefined
return withLock(this.layout.state, async () => {
const state = await this.read()
if (!state || state.stage === "HALTED" || state.stage === "DONE") return state
state.spend.usd += usd
state.spend.estimated ||= estimated
state.spend.events += 1
await this.guard(state, "system")
await this.guard(state, "system", { skipControl: true })
await this.save(state)
return state
})
packages/core/src/gates/artifacts.ts
+3 -3
View File
@@ -2,7 +2,7 @@
// gates.json is a structured finding rather than a runtime surprise.
import { readdir, readFile } from "node:fs/promises"
import path from "node:path"
import { factoryLayout } from "../layout.ts"
import { stateDir as defaultStateDir, factoryLayout } from "../layout.ts"
import { auditMarkdown } from "../research/auditor.ts"
import { researchCache } from "../research/ops.ts"
import { FrontierSchema } from "../schema/frontier.ts"
@@ -20,7 +20,7 @@ const issueText = (error: unknown) => {
: String(error)
}
export async function validateArtifacts(root: string): Promise<GateFinding[]> {
export async function validateArtifacts(root: string, stateDir: string = defaultStateDir()): Promise<GateFinding[]> {
const layout = factoryLayout(root)
const findings: GateFinding[] = []
const check = async (file: string, rule: string, validate: (text: string) => unknown) => {
@@ -49,7 +49,7 @@ export async function validateArtifacts(root: string): Promise<GateFinding[]> {
}
const report = await readFile(layout.researchReport, "utf8").catch(() => undefined)
if (report !== undefined) {
const audit = await auditMarkdown(report, researchCache(root))
const audit = await auditMarkdown(report, researchCache(root, stateDir))
for (const claim of audit.claims.filter((item) => item.status !== "grounded"))
findings.push({
file: ".factory/research/REPORT.md",
packages/core/src/gates/engine.ts
+44 -3
View File
@@ -9,7 +9,7 @@
import { existsSync } from "node:fs"
import { mkdir, readFile, writeFile } from "node:fs/promises"
import path from "node:path"
import { factoryLayout } from "../layout.ts"
import { stateDir as defaultStateDir, factoryLayout } from "../layout.ts"
import {
type CheckReport,
type CommandCheck,
@@ -21,10 +21,11 @@ import {
import { type AffectedTests, affectedTests, type GraphOptions } from "../structure/index.ts"
import { verifyControl } from "../trust/control.ts"
import { relativeTo } from "../trust/paths.ts"
import { SANDBOX_MISSING } from "../trust/sandbox.ts"
import { commandSetHash, isTrusted } from "../trust/store.ts"
import { matchAny } from "../util/glob.ts"
import { parseJsonc } from "../util/jsonc.ts"
import { type RunResult, run, splitCommand } from "../util/proc.ts"
import { bwrapAvailable, findExecutable, type RunResult, run, splitCommand } from "../util/proc.ts"
import { changedFiles, diffStats, git } from "../worktree/git.ts"
import { validateArtifacts } from "./artifacts.ts"
import { checkBudgets, isTestFile } from "./budgets.ts"
@@ -54,6 +55,13 @@ export interface RunGatesOptions {
readonly lsp?: { diagnostics(file: string, options?: { maxWaitMs?: number }): Promise<unknown> }
/** Structural index options for touched-scope affected-test selection. */
readonly structure?: GraphOptions
/**
* Gate commands run agent-written code (tests), so they run in the gate
* sandbox: only `dir` writable, .factory/ and git config read-only, secrets
* and credentials masked. "required" (factory runs) refuses without
* bubblewrap; "preferred" (default; human and user runs) uses it when present.
*/
readonly sandbox?: "required" | "preferred"
}
const TOOL_CONFIG = [
@@ -198,6 +206,8 @@ async function runCommandCheck(
logDir: string
affected?: AffectedTests
signal?: AbortSignal
sandbox: "required" | "preferred"
stateDir: string
},
): Promise<{ report: CheckReport; findings: GateFinding[]; failedTests: string[] }> {
const started = Date.now()
@@ -222,14 +232,41 @@ async function runCommandCheck(
} else if (check.touchedArgs) argv = [...argv, "."]
argv[0] = resolveBinary(argv[0]!, context.dir, context.root)
const log = path.join(context.logDir, `${check.id}.log`)
// Resolve the tool first: inside bwrap a missing binary would look like a failing command.
const toolMissing = !argv[0]!.includes("/") && !findExecutable(argv[0]!)
const sandboxed = !toolMissing && bwrapAvailable()
// A missing tool is reported as missing, not as a sandbox problem: without
// bubblewrap there is nothing to sandbox, and with it the binary still
// would not resolve inside.
if (!sandboxed && context.sandbox === "required" && !toolMissing) {
await writeFile(log, `$ ${argv.join(" ")}\n${SANDBOX_MISSING}\n`)
return {
report: { id: check.id, status: "error", durationMs: 0, findings: 1, log, note: SANDBOX_MISSING },
findings: [
{
file: ".",
rule: "gates/sandbox-missing",
severity: "error",
message: `Did not run "${check.command}": ${SANDBOX_MISSING}`,
fixHint: "Install bubblewrap, then run the gates again.",
check: check.id,
},
],
failedTests: [],
}
}
let result: RunResult
try {
if (toolMissing) throw new Error(`${argv[0]} is not installed (not found on PATH)`)
result = await run(argv, {
cwd: context.dir,
timeoutMs: check.timeoutMs,
maxOutputBytes: 5_000_000,
signal: context.signal,
passEnv: ["CARGO_TARGET_DIR", "GOFLAGS", "GOCACHE"],
...(sandboxed
? { sandbox: { kind: "gate" as const, root: context.root, writable: context.dir, stateDir: context.stateDir } }
: {}),
})
} catch (error) {
const message = error instanceof Error ? error.message : String(error)
@@ -385,7 +422,7 @@ export async function runGates(options: RunGatesOptions): Promise<GateReport> {
})
})
if (options.scope === "full" || files.some((file) => file.startsWith(".factory/")))
await timed("artifacts", () => validateArtifacts(root))
await timed("artifacts", () => validateArtifacts(root, options.stateDir))
if (config.security.osv !== "off" && files.some((file) => DEPENDENCY_FILES.has(path.basename(file))))
await timed("osv", () =>
osvFindings(dir, {
@@ -451,6 +488,8 @@ export async function runGates(options: RunGatesOptions): Promise<GateReport> {
scope: options.scope,
files: present,
logDir,
sandbox: options.sandbox ?? "preferred",
stateDir: options.stateDir ?? defaultStateDir(),
...(affected ? { affected } : {}),
...(options.signal ? { signal: options.signal } : {}),
})
@@ -533,6 +572,8 @@ export function gateRunner(
}) =>
runGates({
...request,
// The factory's own gate runs execute the programmer's code: sandbox or refuse.
sandbox: "required",
...(options.stateDir ? { stateDir: options.stateDir } : {}),
...(options.fetch ? { fetch: options.fetch } : {}),
...(options.lsp ? { lsp: options.lsp } : {}),
packages/core/src/gates/waivers.ts
+10 -8
View File
@@ -4,7 +4,7 @@
import { readdir, readFile } from "node:fs/promises"
import { hostname, userInfo } from "node:os"
import path from "node:path"
import { signRecord, verifyRecordMac } from "../approval/keystore.ts"
import { type HumanSigner, signatureProblem } from "../approval/keystore.ts"
import { appendAuditEntry, auditHead } from "../audit/chain.ts"
import { factoryLayout, stateDir } from "../layout.ts"
import type { GateFinding } from "../schema/gates.ts"
@@ -21,7 +21,8 @@ export interface RecordWaiverInput {
readonly expiresAt: Date
readonly channel: "cli" | "tui"
readonly approvedBy?: string
readonly stateDir?: string
/** The unlocked human key. */
readonly signer: HumanSigner
}
const MAX_WAIVER_DAYS = 90
@@ -32,7 +33,7 @@ export async function recordWaiver(root: string, input: RecordWaiverInput): Prom
if (input.expiresAt.getTime() - now > MAX_WAIVER_DAYS * 86_400_000)
throw new Error(`Waivers last at most ${MAX_WAIVER_DAYS} days.`)
const unsigned = {
version: 1 as const,
version: 2 as const,
id: input.id,
rule: input.rule,
files: input.files ?? "**",
@@ -43,7 +44,7 @@ export async function recordWaiver(root: string, input: RecordWaiverInput): Prom
channel: input.channel,
auditHead: await auditHead(root),
}
const waiver = WaiverSchema.parse({ ...unsigned, mac: await signRecord(unsigned, input.stateDir ?? stateDir()) })
const waiver = WaiverSchema.parse({ ...unsigned, signature: input.signer.sign(unsigned) })
await writeJson(path.join(factoryLayout(root).waivers, `${waiver.id}.json`), waiver)
await appendAuditEntry(root, {
actor: `human:${waiver.approvedBy}@${hostname()}`,
@@ -83,15 +84,16 @@ export async function loadWaivers(
problem("not valid JSON; ignored")
continue
}
const unsigned = await signatureProblem(raw, "this waiver", options.stateDir ?? stateDir())
if (unsigned) {
problem(`${unsigned}; ignored (waivers are granted with \`es waive\`)`)
continue
}
const parsed = WaiverSchema.safeParse(raw)
if (!parsed.success) {
problem("malformed waiver; ignored")
continue
}
if (!(await verifyRecordMac(raw, options.stateDir ?? stateDir()))) {
problem("not signed by this machine's approval key; ignored (waivers are granted with `es waive`)")
continue
}
if (!isWaiverActive(parsed.data, options.now)) continue
active.push(parsed.data)
}
packages/core/src/ops/tools.ts
+3 -1
View File
@@ -157,7 +157,7 @@ export function esTools(ops: OpsContext): EsToolDef[] {
"Validate .factory/roadmap.json and every phase GOAL.md against their schemas. Run until it reports no errors.",
input: object(),
execute: async () => {
const problems = await validateArtifacts(root)
const problems = await validateArtifacts(root, ops.stateDir)
const subject = await approvalSubject(root, "spec").then(
(result) => result,
(error: Error) => error,
@@ -205,6 +205,8 @@ export function esTools(ops: OpsContext): EsToolDef[] {
...(worktree && phase?.baseCommit ? { base: phase.baseCommit, phaseId: phase.id } : {}),
...(state ? { runId: state.runId } : {}),
stateDir: ops.stateDir,
// A seat's gate run executes agent-written code: sandbox or refuse.
sandbox: seat ? "required" : "preferred",
...(ops.lsp ? { lsp: ops.lsp } : {}),
...(context.signal ? { signal: context.signal } : {}),
})
packages/core/src/research/cache.ts
+28 -3
View File
@@ -7,6 +7,7 @@ import path from "node:path"
import { factoryLayout } from "../layout.ts"
import { atomicWrite, readJson } from "../util/fs.ts"
import { sha256 } from "../util/hash.ts"
import { ensureEngineKey, sealMeta, verifyMetaSeal } from "./seal.ts"
import { canonicalUrl } from "./url.ts"
export interface SourceMeta {
@@ -18,6 +19,8 @@ export interface SourceMeta {
readonly provider: string
readonly query?: string
readonly bytes: number
/** Engine HMAC seal (#52); present when written through a sealed cache. */
readonly seal?: string
}
export interface CachedSource {
@@ -42,7 +45,26 @@ export const normalizeSourceText = (text: string) =>
const MAX_SOURCE_BYTES = 400_000
export class SourceCache {
constructor(readonly dir: string) {}
/**
* When `stateDir` is set, entries are sealed with the engine key on write
* (#52) and unsealed entries are refused on read. Without it the cache keeps
* its legacy behaviour (tests and auxiliary readers).
*/
constructor(
readonly dir: string,
readonly stateDir?: string,
) {}
private async seal(meta: SourceMeta): Promise<SourceMeta> {
if (!this.stateDir) return meta
const key = await ensureEngineKey(this.stateDir)
return { ...meta, seal: sealMeta(meta as unknown as Record<string, unknown>, key) }
}
private async sealed(meta: SourceMeta): Promise<boolean> {
if (!this.stateDir) return true
return verifyMetaSeal(meta as unknown as Record<string, unknown>, await ensureEngineKey(this.stateDir))
}
async put(input: {
url: string
@@ -55,7 +77,7 @@ export class SourceCache {
if (Buffer.byteLength(text) > MAX_SOURCE_BYTES)
text = `${Buffer.from(text).subarray(0, MAX_SOURCE_BYTES).toString("utf8")}\n\n[truncated by epistemic-swarm at ${MAX_SOURCE_BYTES} bytes]`
const hash = sha256(text)
const meta: SourceMeta = {
const meta = await this.seal({
sha256: hash,
url: input.url,
...(input.title ? { title: input.title } : {}),
@@ -63,7 +85,7 @@ export class SourceCache {
provider: input.provider,
...(input.query ? { query: input.query } : {}),
bytes: Buffer.byteLength(text),
}
})
const file = path.join(this.dir, `${hash}.md`)
await atomicWrite(file, text)
await atomicWrite(path.join(this.dir, `${hash}.json`), `${JSON.stringify(meta, null, 2)}\n`)
@@ -87,6 +109,9 @@ export class SourceCache {
}
// A source whose bytes no longer hash to its name was tampered with: refuse it.
if (sha256(text) !== hash) return undefined
// Without a valid engine seal the entry was not written by the engine
// (planted before the factory dir existed, #52): refuse it.
if (!(await this.sealed(meta))) return undefined
return { meta, text, path: file }
}
packages/core/src/research/index.ts
+1
View File
@@ -5,4 +5,5 @@ export * from "./ops.ts"
export * from "./osv.ts"
export * from "./providers.ts"
export * from "./quote.ts"
export * from "./seal.ts"
export * from "./url.ts"
packages/core/src/research/ops.ts
+11 -4
View File
@@ -2,7 +2,7 @@
// and audit (verify tags and quotes, optionally prune).
import { readFile, writeFile } from "node:fs/promises"
import path from "node:path"
import { factoryLayout } from "../layout.ts"
import { stateDir as defaultStateDir, factoryLayout } from "../layout.ts"
import type { EsToolDef } from "../ops/tools.ts"
import type { EsConfig } from "../schema/config.ts"
import { evaluateWrite, type PolicyContext } from "../trust/policy.ts"
@@ -14,6 +14,8 @@ import { isFresh } from "./url.ts"
export interface ResearchOpsContext {
readonly root: string
readonly policy: () => Promise<PolicyContext>
/** Engine-key dir for sealing cached sources (#52); seats get the runtime state dir. */
readonly stateDir?: string
readonly provider?: string
readonly fetch?: typeof fetch
readonly env?: NodeJS.ProcessEnv
@@ -49,12 +51,17 @@ const object = (properties: Record<string, unknown>, required: string[] = []) =>
/** Providers whose snapshots hold the whole page (es_research_fetch and the host's webfetch). */
const FETCHED = new Set(["fetch", "webfetch"])
export function researchCache(root: string) {
return new SourceCache(researchSourcesDir(root))
/**
* The engine cache for a project. Entries are sealed with the engine key
* (#52); `stateDir` defaults to the global state dir, like every other core
* function that takes it. Tests pass an isolated dir.
*/
export function researchCache(root: string, stateDir: string = defaultStateDir()) {
return new SourceCache(researchSourcesDir(root), stateDir)
}
export function researchTools(context: ResearchOpsContext): EsToolDef[] {
const cache = researchCache(context.root)
const cache = researchCache(context.root, context.stateDir ?? defaultStateDir())
const env = context.searxngUrl ? { ...(context.env ?? process.env), SEARXNG_URL: context.searxngUrl } : context.env
const provider = (): SearchProvider => {
const selected = selectProvider(context.provider, env)
packages/core/src/research/quote.ts
+25 -5
View File
@@ -1,8 +1,13 @@
// Pure verbatim-quote verification. A quote matches when, after normalising
// whitespace and typographic punctuation (curly quotes, dashes, NBSP), it
// appears in the source exactly. An ellipsis ("..." or "…") splits a quote into
// fragments that must appear in order. No fuzzy matching: a quote is either in
// the source or it is not.
// appears in the source exactly. An ellipsis ("..." or "…") splits a prose
// quote into fragments that must appear in order, and each fragment must be
// evidence on its own (MIN_QUOTE characters). Code excerpts are one contiguous
// span (verifySpan): "..." there is code, not an ellipsis. No fuzzy matching:
// a quote is either in the source or it is not.
/** Shortest quote, and shortest ellipsis fragment, that counts as evidence (normalised characters). */
export const MIN_QUOTE = 12
const TYPOGRAPHY: Array<[RegExp, string]> = [
[/[‘’‚‛′]/g, "'"],
@@ -52,8 +57,14 @@ export function verifyQuote(source: string, quote: string): QuoteCheck {
.map((fragment) => fragment.trim())
.filter(Boolean)
if (fragments.length === 0) return { ok: false, reason: "empty quote" }
if (fragments.join(" ").length < 12)
return { ok: false, reason: "quote too short to be evidence (need at least 12 characters)" }
if (fragments.length === 1 && fragments[0]!.length < MIN_QUOTE)
return { ok: false, reason: `quote too short to be evidence (need at least ${MIN_QUOTE} characters)` }
const short = fragments.find((fragment) => fragment.length < MIN_QUOTE)
if (short !== undefined)
return {
ok: false,
reason: `quote fragment too short to be evidence: "${short.slice(0, 40)}" (each fragment between ellipses needs at least ${MIN_QUOTE} characters)`,
}
const haystack = normalizeForQuote(source)
let from = 0
let first: number | undefined
@@ -65,3 +76,12 @@ export function verifyQuote(source: string, quote: string): QuoteCheck {
}
return { ok: true, index: first! }
}
/** One contiguous verbatim span (code excerpts): no ellipsis splitting, at least MIN_QUOTE characters. */
export function verifySpan(source: string, span: string): QuoteCheck {
const needle = normalizeForQuote(span)
if (needle.length < MIN_QUOTE)
return { ok: false, reason: `too short to be evidence (need at least ${MIN_QUOTE} characters)` }
const at = normalizeForQuote(source).indexOf(needle)
return at < 0 ? { ok: false, reason: `not found verbatim: "${needle.slice(0, 80)}"` } : { ok: true, index: at }
}
packages/core/src/research/seal.ts
+55
View File
@@ -0,0 +1,55 @@
// Engine seal for cached sources (1.1.1, #52). Before the factory dir exists,
// the user agent is unsandboxed and could pre-seed the content-addressed
// source cache with forged entries (the file name is its own sha256, so the
// tamper check passes). Every entry the engine writes is therefore sealed
// with an HMAC under a key kept in the masked state dir (`engine.key`), and
// sealed readers refuse entries without a valid seal. The key is created at
// plugin setup and lazily everywhere else; agents can neither read the state
// dir (sandbox mask) nor forge the seal.
import { createHmac, randomBytes, timingSafeEqual } from "node:crypto"
import { mkdir, open, readFile } from "node:fs/promises"
import path from "node:path"
import { stateDir } from "../layout.ts"
import { canonicalJson } from "../util/hash.ts"
const ENGINE_KEY_FILE = "engine.key"
/** Read the engine key, creating it (0600) when absent. */
export async function ensureEngineKey(dir: string = stateDir()): Promise<Buffer> {
await mkdir(dir, { recursive: true, mode: 0o700 })
const file = path.join(dir, ENGINE_KEY_FILE)
const existing = await readFile(file).catch(() => undefined)
if (existing && existing.length >= 32) return existing
const key = randomBytes(32)
try {
const handle = await open(file, "wx", 0o600)
await handle.writeFile(key)
await handle.close()
} catch (error: any) {
// A concurrent creator won: use its key.
if (error?.code !== "EEXIST") throw error
const raced = await readFile(file)
if (raced.length >= 32) return raced
throw error
}
return key
}
/** HMAC-SHA256 over the canonical meta (minus any previous seal), hex. */
export function sealMeta(meta: Record<string, unknown>, key: Buffer): string {
const { seal: _seal, ...body } = meta
return createHmac("sha256", key).update(canonicalJson(body)).digest("hex")
}
/** Whether `meta.seal` is a valid seal by `key` over the rest of `meta`. */
export function verifyMetaSeal(meta: Record<string, unknown>, key: Buffer): boolean {
if (typeof meta.seal !== "string") return false
let presented: Buffer
try {
presented = Buffer.from(meta.seal, "hex")
} catch {
return false
}
const expected = Buffer.from(sealMeta(meta, key), "hex")
return presented.length === expected.length && timingSafeEqual(presented, expected)
}
packages/core/src/schema/approval.ts
+11 -3
View File
@@ -7,13 +7,22 @@ export type ApprovalStage = z.infer<typeof ApprovalStageSchema>
export const ApprovalChannelSchema = z.enum(["cli", "tui"])
export type ApprovalChannel = z.infer<typeof ApprovalChannelSchema>
/** An Ed25519 signature by the human key (approval/keystore.ts) over the canonical record minus `signature`. */
export const RecordSignatureSchema = z.object({
alg: z.literal("ed25519"),
/** Public fingerprint of the human key (not the key). */
keyId: z.string().length(16),
sig: z.string().min(1),
})
export const ApprovalSubjectSchema = z.object({
path: z.string().min(1),
sha256: z.string().length(64),
})
export const ApprovalSchema = z.object({
version: z.literal(1),
/** 2 since 1.1.1 (Ed25519, passphrase-sealed); version-1 HMAC records are refused. */
version: z.literal(2),
stage: ApprovalStageSchema,
subject: z.array(ApprovalSubjectSchema).min(1),
approvedBy: z.string().min(1),
@@ -24,8 +33,7 @@ export const ApprovalSchema = z.object({
auditHead: z.object({ seq: z.number().int().nonnegative(), hash: z.string().length(64) }).nullable(),
spendCeilingUSD: z.number().positive().optional(),
notes: z.string().optional(),
/** HMAC-SHA256 over the canonical record (minus mac) with the user-global signing key. */
mac: z.string().length(64),
signature: RecordSignatureSchema,
})
export type Approval = z.infer<typeof ApprovalSchema>
packages/core/src/schema/claims.ts
+3 -3
View File
@@ -128,10 +128,10 @@ export const BriefSchema = z.object({
members: z.array(z.string()),
}),
signature: z.object({
alg: z.literal("hmac-sha256"),
/** Public fingerprint of the signing key (not the key). */
alg: z.literal("ed25519"),
/** Public fingerprint of the human key (not the key). */
keyId: z.string(),
mac: z.string(),
sig: z.string(),
}),
})
export type Brief = z.infer<typeof BriefSchema>
packages/core/src/schema/waiver.ts
+4 -2
View File
@@ -1,8 +1,10 @@
import { z } from "zod"
import { RecordSignatureSchema } from "./approval.ts"
/** A human-signed, expiring exception for findings matching `rule` (glob) in `files` (glob). */
export const WaiverSchema = z.object({
version: z.literal(1),
/** 2 since 1.1.1 (Ed25519, passphrase-sealed); version-1 HMAC waivers are ignored. */
version: z.literal(2),
id: z.string().regex(/^[a-z0-9][a-z0-9-]{0,63}$/),
rule: z.string().min(1),
files: z.string().min(1).default("**"),
@@ -12,7 +14,7 @@ export const WaiverSchema = z.object({
expiresAt: z.string().datetime(),
channel: z.enum(["cli", "tui"]),
auditHead: z.object({ seq: z.number().int().nonnegative(), hash: z.string().length(64) }).nullable(),
mac: z.string().length(64),
signature: RecordSignatureSchema,
})
export type Waiver = z.infer<typeof WaiverSchema>
packages/core/src/trust/control.ts
+47 -12
View File
@@ -10,7 +10,7 @@
import { readFile } from "node:fs/promises"
import path from "node:path"
import { factoryLayout } from "../layout.ts"
import { readJson, writeJson } from "../util/fs.ts"
import { readJson, withLock, writeJson } from "../util/fs.ts"
import { matchAny } from "../util/glob.ts"
import { sha256 } from "../util/hash.ts"
@@ -65,12 +65,17 @@ export type ControlClass = "factory" | "config"
* inside a phase worktree (.factory/worktrees/<id>/...) are classified by their
* path within that worktree, since they would ship on the phase branch.
*/
// Matched case-insensitively: on a case-insensitive volume `.factory/GATES.json`
// is the gates file, so a case variant must not slip past the policy (#45).
const FACTORY_CONTROL_LOWER = FACTORY_CONTROL.map((glob) => glob.toLowerCase())
const CONFIG_CONTROL_LOWER = CONFIG_CONTROL.map((glob) => glob.toLowerCase())
export function controlClass(relative: string): ControlClass | undefined {
const normalized = relative.replace(/^\.\//, "")
const normalized = relative.replace(/^\.\//, "").toLowerCase()
const inner = /^\.factory\/worktrees\/[^/]+\/(.+)$/.exec(normalized)?.[1]
for (const candidate of inner ? [normalized, inner] : [normalized]) {
if (matchAny(candidate, FACTORY_CONTROL)) return "factory"
if (matchAny(candidate, CONFIG_CONTROL)) return "config"
if (matchAny(candidate, FACTORY_CONTROL_LOWER)) return "factory"
if (matchAny(candidate, CONFIG_CONTROL_LOWER)) return "config"
}
return undefined
}
@@ -119,14 +124,44 @@ export async function snapshotControl(root: string): Promise<Record<string, stri
/** Record the current control-file hashes. Call only from code acting for a human or the system. */
export async function rebaseline(root: string, updatedBy: string): Promise<ControlBaseline> {
const baseline: ControlBaseline = {
version: 1,
files: await snapshotControl(root),
updatedAt: new Date().toISOString(),
updatedBy,
}
await writeJson(factoryLayout(root).control, baseline)
return baseline
// Held under the control lock (like repinControl): an interleaved re-pin
// must not overwrite a fresh baseline with a stale snapshot (#32, CI-only
// es_build_start refusal with a just-recorded approval missing).
const file = factoryLayout(root).control
return withLock(file, async () => {
const baseline: ControlBaseline = {
version: 1,
files: await snapshotControl(root),
updatedAt: new Date().toISOString(),
updatedBy,
}
await writeJson(file, baseline)
return baseline
})
}
/**
* Re-pin one control file after an agent write (#32). Unlike `rebaseline`,
* only `rel` is updated, so a concurrent hand edit to another control file
* (gates.json, config.json) is not absorbed into the baseline: it still shows
* as drift and halts the run. Acts only when a baseline exists; compare-and-
* swaps on `expectedOld` (the pin before our write) so a concurrent change to
* the same file is not silently absorbed either.
*/
export async function repinControl(root: string, rel: string, expectedOld: string | null): Promise<void> {
const file = factoryLayout(root).control
await withLock(file, async () => {
const baseline = await readJson<ControlBaseline>(file)
if (!baseline) return
if ((baseline.files[rel] ?? null) !== expectedOld) return
const current = await hashOrNull(path.join(root, ...rel.split("/")))
baseline.files[rel] = current
await writeJson(file, {
...baseline,
updatedAt: new Date().toISOString(),
updatedBy: `${baseline.updatedBy} + repin:${rel}`,
})
})
}
export interface ControlCheck {
packages/core/src/trust/index.ts
+1
View File
@@ -1,5 +1,6 @@
export * from "./control.ts"
export * from "./paths.ts"
export * from "./policy.ts"
export * from "./sandbox.ts"
export * from "./stop.ts"
export * from "./store.ts"
packages/core/src/trust/policy.ts
+125 -39
View File
@@ -8,6 +8,7 @@ import { stateDir } from "../layout.ts"
import { matchAny } from "../util/glob.ts"
import { controlClass } from "./control.ts"
import { canonicalPath, isInside, relativeTo, resolveTarget } from "./paths.ts"
import { SANDBOX_MISSING, type SandboxKind } from "./sandbox.ts"
export type Decision = { readonly effect: "allow" } | { readonly effect: "deny" | "ask"; readonly reason: string }
@@ -99,34 +100,114 @@ export function evaluateRead(context: PolicyContext, _agentId: string | undefine
}
// ------------------------------------------------------------------ shell
//
// Enforcement is the OS sandbox (trust/sandbox.ts): every agent shell runs
// under bubblewrap with control files read-only and secrets masked; factory
// seats are refused outright without it. The text rules below are defence in
// depth with clear refusal messages, and the only layer for the user's own
// agents on hosts without bubblewrap, where they are best effort: arbitrary
// shell can always outwit pattern matching, so that mode is documented as
// weaker.
export type ShellDecision =
| { readonly effect: "allow"; readonly mode: "normal" | "readonly-sandbox" | "readonly-checked" }
| { readonly effect: "allow"; readonly mode: "sandbox"; readonly kind: SandboxKind; readonly offline: boolean }
| { readonly effect: "allow"; readonly mode: "unsandboxed" }
| { readonly effect: "deny"; readonly reason: string }
const HUMAN_ONLY_CLI =
/(^|[\s;&|(`'"/])(es|epistemic-swarm)\s+(approve|trust|waive|resume|rebaseline|factory\s+(resume|pr)|gates\s+install-git|lsp\s+install|config\s+set|research\s+retract)\b/
/** Human-launched jobs that drive a harness CLI: an agent must not start nested headless runs. */
const HUMAN_ONLY_JOBS = /(^|[\s;&|(`'"/])(es|epistemic-swarm)\s+(audit\s+(?!(verify|show)\b)\S|scout\s+(?!show\b)\S)/
/** The command with shell quoting and escapes removed, so `es 'approve'` and `.fac""tory` read plainly. */
export const unquoteShell = (command: string) => command.replaceAll(/['"\\]/g, "")
const words = (command: string) => command.split(/[^\w@./-]+/).filter(Boolean)
const ES_BINARY = /^(es|epistemic-swarm|es\.js|es-cli|@heretek-ai\/es-cli(@[\w.-]+)?)$|\/(es|es\.js|epistemic-swarm)$/
/** Human-only verbs: the next word(s) after the es binary. */
const HUMAN_VERBS: ReadonlyArray<readonly [string, ((next: string | undefined) => boolean)?]> = [
["approve"],
["trust"],
["waive"],
["resume"],
["rebaseline"],
["key"],
["reseal"],
["factory", (next) => next === "resume" || next === "pr"],
["gates", (next) => next === "install-git"],
["lsp", (next) => next === "install"],
["config", (next) => next === "set"],
["research", (next) => next === "retract" || next === "export"],
// Audit and scout runs drive a harness CLI: an agent must not start nested headless runs.
["audit", (next) => next !== undefined && next !== "verify" && next !== "show"],
["scout", (next) => next !== undefined && next !== "show"],
]
/** Whether the command invokes a human-only es verb, however it is quoted or wrapped. */
export function invokesHumanOnly(command: string): boolean {
const tokens = words(unquoteShell(command))
return tokens.some((token, index) => {
if (!ES_BINARY.test(token)) return false
const verb = tokens[index + 1]
const rule = HUMAN_VERBS.find(([name]) => name === verb)
return rule !== undefined && (rule[1] === undefined || rule[1](tokens[index + 2]))
})
}
const CONTROL_MENTION =
/\.factory\/(gates\.json|config\.json|frontier\.json|waivers|approvals|runtime|STOP|git-hooks|claims\b|audits\b|research\/(sources\b|(coverage|dossier|brief\.pcrb)\.json)|(brainstorm|harvest|design|scout)\/[^\s'"]*\.json)|\.git\/(config|hooks)|\.opencode\/(hooks\.json|plugins|opencode\.jsonc?)|\.claude\/settings|opencode\.jsonc?\b/
/\.factory\/(gates\.json|config\.json|frontier\.json|waivers|approvals|runtime|STOP|git-hooks|claims\b|audits\b|research\/(sources\b|(coverage|dossier|brief\.pcrb)\.json)|(brainstorm|harvest|design|scout)\/[^\s'"]*\.json)|\.git\/(config|hooks)|\.opencode\/(hooks\.json|plugins|opencode\.jsonc?)|\.claude\/settings|opencode\.jsonc?\b/i
/** Any mention of the factory dir (a `cd .factory` reaches control files without naming them). */
const FACTORY_MENTION = /(^|[\s/=:])\.factory(\/|[\s;|&]|$)/i
const MUTATING =
/(>|\btee\b|\brm\b|\bmv\b|\bcp\b|\bln\b|\btruncate\b|\bchmod\b|\bchown\b|\btouch\b|\bsed\s+(-[a-zA-Z]*i|--in-place)|\bdd\b|\binstall\b|\bgit\s+(checkout|restore|rm|mv|reset|clean|apply|am|stash))/
const SEAT_FORBIDDEN_GIT =
/\bgit\s+(push|reset\s+--hard|clean\b|branch\s+-D|worktree\b|filter-branch|update-ref|config\b)/
/** Commands a read-only seat may run when bubblewrap is unavailable (each pipeline segment). */
/** Global git options that take a value, skipped to find the subcommand (`git -C . push`). */
const GIT_VALUE_OPTIONS = new Set([
"-C",
"-c",
"--git-dir",
"--work-tree",
"--namespace",
"--exec-path",
"--config-env",
])
const SEAT_FORBIDDEN_GIT = new Set(["push", "config", "worktree", "filter-branch", "update-ref", "clean", "remote"])
/** git subcommands in a command, past global options. */
function gitSubcommands(command: string): Array<{ sub: string; args: string[] }> {
const found: Array<{ sub: string; args: string[] }> = []
for (const segment of segments(unquoteShell(command))) {
const tokens = segment.split(/\s+/)
for (let i = 0; i < tokens.length; i++) {
if (!/(^|\/)git$/.test(tokens[i]!)) continue
let j = i + 1
while (j < tokens.length && tokens[j]!.startsWith("-")) j += GIT_VALUE_OPTIONS.has(tokens[j]!) ? 2 : 1
if (j < tokens.length) found.push({ sub: tokens[j]!, args: tokens.slice(j + 1) })
}
}
return found
}
const seatForbiddenGit = (command: string) =>
gitSubcommands(command).find(
({ sub, args }) =>
SEAT_FORBIDDEN_GIT.has(sub) ||
(sub === "reset" && args.includes("--hard")) ||
(sub === "branch" && args.some((arg) => /^-[a-zA-Z]*D/.test(arg))),
)
/**
* Commands that only read (each pipeline segment). No `env` (it runs any
* program), and no tool with an output-file option: `sort -o`, `uniq IN OUT`,
* `tree -o`, `git … --output`, `find -fprint*`/`-fls` all write.
*/
const READONLY_ALLOW = [
/^git\s+(status|log|diff|show|blame|grep|ls-files|rev-parse|branch(\s+--list)?|describe)\b/,
/^(ls|cat|head|tail|wc|grep|rg|stat|file|tree|pwd|echo|which|sort|uniq|cut|tr|diff|basename|dirname|realpath|env|date|true)\b/,
/^find\b(?!.*\s-(delete|exec|execdir|ok|fprint)\b)/,
/^git\s+(status|log|diff|show|blame|grep|ls-files|rev-parse|branch(\s+--list)?|describe)\b(?!.*\s--output\b)/,
/^(ls|cat|head|tail|wc|grep|rg|stat|file|pwd|echo|which|cut|tr|diff|basename|dirname|realpath|date|true)\b/,
/^find\b(?!.*\s-(delete|exec|execdir|ok|okdir|fprint|fprint0|fprintf|fls)\b)/,
/^(bun\s+test|npx\s+(vitest|jest)|vitest|jest|pytest|cargo\s+test|go\s+test|npm\s+test|bun\s+run\s+test|python3?\s+-m\s+pytest)\b/,
/^(tsc|npx\s+tsc|bunx\s+tsc|biome\s+(check|lint|format)|npx\s+biome|bunx\s+biome|ruff\s+(check|format\s+--check)|mypy|eslint|npx\s+eslint)\b/,
]
function segments(command: string): string[] {
return command
.split(/\|\||&&|[;|\n]/)
.split(/\|\||&&|[;|\n&]/)
.map((segment) => segment.trim())
.filter(Boolean)
}
@@ -134,8 +215,7 @@ function segments(command: string): string[] {
/**
* A command that cannot write: no redirection, here-doc or command
* substitution, and every segment on the read-only allowlist. Interpreter
* one-liners (`python3 -c`, `node -e`, `bun -e`, …) are not on it, so a
* control-file mention inside one is refused for every agent.
* one-liners (`python3 -c`, `node -e`, `bun -e`, …) are not on it.
*/
const isProvablyReadOnly = (command: string): boolean =>
!/[<>`]|\$\(/.test(command) &&
@@ -148,39 +228,45 @@ export function evaluateShell(
options: { readonly sandboxAvailable: boolean },
): ShellDecision {
const spec = agentSpec(agentId)
if (HUMAN_ONLY_CLI.test(command) || HUMAN_ONLY_JOBS.test(command))
const plain = unquoteShell(command)
if (invokesHumanOnly(command))
return {
effect: "deny",
reason:
"That command is human-only (approvals, trust, waivers, resume, rebaseline, config set, retractions, audit and scout runs); agents cannot run it.",
"That command is human-only (approvals, trust, waivers, resume, rebaseline, keys, config set, retractions, exports, audit and scout runs); agents cannot run it.",
}
const privateDir = canonicalPath(context.stateDir ?? stateDir())
if (command.includes(privateDir) || /epistemic-swarm\/(key|trust)/.test(command))
if (plain.includes(privateDir) || /epistemic-swarm\/(key|trust|engine)|state\/epistemic-swarm/i.test(plain))
return { effect: "deny", reason: "Agents may not access Epistemic Swarm's private state dir." }
if (CONTROL_MENTION.test(command) && (spec || MUTATING.test(command) || !isProvablyReadOnly(command)))
if (spec) {
if (!options.sandboxAvailable) return { effect: "deny", reason: `Factory seat "${spec.id}": ${SANDBOX_MISSING}` }
if (CONTROL_MENTION.test(plain))
return {
effect: "deny",
reason: "Factory seats may not reference control files from the shell; use the read tool to inspect them.",
}
const git = seatForbiddenGit(command)
if (git)
return {
effect: "deny",
reason: `Factory seats may not push, change git config or remotes, rewrite refs or manage worktrees (git ${git.sub}).`,
}
const kind: SandboxKind = spec.seat === "programmer" ? "programmer" : spec.readonlyShell ? "readonly" : "seat"
return { effect: "allow", mode: "sandbox", kind, offline: spec.web === "cached" }
}
// The user's own agents: control files and the factory dir only in provably read-only commands.
if (
(CONTROL_MENTION.test(plain) || FACTORY_MENTION.test(plain)) &&
(MUTATING.test(plain) || !isProvablyReadOnly(plain))
)
return {
effect: "deny",
reason: spec
? "Factory seats may not reference control files from the shell; use the read tool to inspect them."
: "A control file may only be mentioned in a provably read-only shell command (no interpreters, redirection or substitution); use the read tool otherwise.",
reason:
"Epistemic Swarm's .factory/ and control files may only be mentioned in a provably read-only shell command (no interpreters, redirection or substitution); use the read tool otherwise.",
}
if (!spec) return { effect: "allow", mode: "normal" }
if (SEAT_FORBIDDEN_GIT.test(command))
return {
effect: "deny",
reason: "Factory seats may not push, rewrite refs, manage worktrees or change git config.",
}
if (!spec.readonlyShell) return { effect: "allow", mode: "normal" }
if (options.sandboxAvailable) return { effect: "allow", mode: "readonly-sandbox" }
if (/`|\$\(/.test(command))
return { effect: "deny", reason: "Command substitution is not allowed in a read-only seat without bubblewrap." }
const blocked = segments(command).find((segment) => !READONLY_ALLOW.some((pattern) => pattern.test(segment)))
if (blocked)
return {
effect: "deny",
reason: `Read-only seat "${spec.id}" may not run "${blocked}" (bubblewrap unavailable, allowlist enforced).`,
}
return { effect: "allow", mode: "readonly-checked" }
return options.sandboxAvailable
? { effect: "allow", mode: "sandbox", kind: "user", offline: false }
: { effect: "allow", mode: "unsandboxed" }
}
/** Quote one argument for POSIX sh. */
packages/core/src/trust/sandbox.ts
+182
View File
@@ -0,0 +1,182 @@
// The OS sandbox every agent shell and every gate run goes through (1.1.1).
// Text rules cannot stop arbitrary shell, so enforcement is a bubblewrap mount
// namespace built from the agent's seat:
// - user: the user's own agents. The project stays writable; .factory/
// is read-only; secrets are masked; git and gh keep working.
// - seat: a factory seat with a normal shell. As user, plus host and
// git config read-only and push credentials masked.
// - programmer: everything read-only except its phase worktree (and a
// private /tmp); credentials masked.
// - readonly: QA, auditors, research: nothing writable (writes in the cwd
// land in a throwaway overlay when bwrap supports one).
// - gate: a gate command (it runs agent-written tests): only the gate's
// directory is writable, as for the programmer.
// Every kind masks Epistemic Swarm's private state dir (signing and engine
// keys) and the OpenCode server password. Seats with web:"cached" get no
// network. Mount order matters: bwrap applies them in sequence.
import { spawnSync } from "node:child_process"
import { existsSync } from "node:fs"
import os from "node:os"
import path from "node:path"
export type SandboxKind = "user" | "seat" | "programmer" | "readonly" | "gate"
export interface SandboxSpec {
readonly kind: SandboxKind
/** The project root (canonical). */
readonly root: string
readonly cwd: string
/** The one writable directory for programmer and gate kinds (worktree or gate dir). */
readonly writable?: string
/** Epistemic Swarm's private state dir, masked for every kind. */
readonly stateDir: string
/** Home directory whose secrets are masked (default: the real home). */
readonly home?: string
/** No network (seats whose web access is "cached"). */
readonly offline?: boolean
}
export interface SandboxProbe {
readonly exists: (file: string) => boolean
/** bwrap supports --overlay-src/--tmp-overlay (0.10+). */
readonly overlay: boolean
readonly uid?: number
}
/** Project paths no agent shell may change, relative to the root and to a phase worktree. */
const PROTECTED_ALWAYS = [".factory"]
const PROTECTED_FOR_SEATS = [
".git/config",
".git/hooks",
".opencode",
"opencode.json",
"opencode.jsonc",
".claude/settings.json",
".claude/settings.local.json",
]
/** Inside a phase worktree, .git is a pointer file the programmer must not retarget. */
const PROTECTED_IN_WORKTREE = [".git"]
/** Secrets masked for every agent (relative to home). */
const SECRET_FILES = [".config/opencode/service.json"]
/** Credentials masked for factory seats and gates (relative to home). */
const CREDENTIAL_DIRS = [".ssh", ".config/gh", ".gnupg", ".docker"]
const CREDENTIAL_FILES = [
".git-credentials",
".config/git/credentials",
".netrc",
".npmrc",
".pypirc",
".cargo/credentials",
".cargo/credentials.toml",
]
const SECRET_ENV = ["OPENCODE_PASSWORD", "OPENCODE_SERVER_PASSWORD"]
const CREDENTIAL_ENV = [
"SSH_AUTH_SOCK",
"SSH_AGENT_PID",
"SSH_ASKPASS",
"GIT_ASKPASS",
"GIT_SSH_COMMAND",
"GH_TOKEN",
"GITHUB_TOKEN",
"GH_ENTERPRISE_TOKEN",
"GITLAB_TOKEN",
"NPM_TOKEN",
"NODE_AUTH_TOKEN",
"CARGO_REGISTRY_TOKEN",
"TWINE_PASSWORD",
]
const inside = (dir: string, file: string) => file === dir || file.startsWith(`${dir}${path.sep}`)
/** The bwrap argv that runs `command` under `spec`. Pure given `probe` (tests pass a fake one). */
export function sandboxArgv(
spec: SandboxSpec,
command: readonly string[],
probe: SandboxProbe = liveProbe(),
): string[] {
const home = spec.home ?? os.homedir()
const user = spec.kind === "user"
const writableRoot = spec.kind === "user" || spec.kind === "seat"
const writable = spec.kind === "programmer" || spec.kind === "gate" ? spec.writable : undefined
const protectedRel = user ? PROTECTED_ALWAYS : [...PROTECTED_ALWAYS, ...PROTECTED_FOR_SEATS]
const protectedIn = (dir: string, names: readonly string[]) =>
names.map((name) => path.join(dir, name)).filter((file) => probe.exists(file))
const args: string[] = ["bwrap", writableRoot ? "--bind" : "--ro-bind", "/", "/", "--dev", "/dev", "--proc", "/proc"]
const roBind = (file: string) => args.push("--ro-bind", file, file)
// Overlays first: later binds (sources resolved on the host) land on top of them.
if (!writableRoot) {
// A private /tmp; projects and homes that live under /tmp are bound back in.
args.push("--tmpfs", "/tmp")
// Tool caches under home stay usable; writes there are thrown away.
if (probe.overlay && probe.exists(home)) args.push("--overlay-src", home, "--tmp-overlay", home)
else if (inside("/tmp", home) && probe.exists(home)) roBind(home)
if (inside("/tmp", spec.root) && !(probe.overlay && inside(home, spec.root))) roBind(spec.root)
}
// Project control paths are read-only.
for (const file of protectedIn(spec.root, protectedRel)) roBind(file)
// The one writable directory, then re-protect what lies inside it.
if (writable) {
args.push("--bind", writable, writable)
for (const file of protectedIn(spec.root, protectedRel))
if (inside(writable, file) && file !== writable) roBind(file)
if (spec.kind === "programmer")
for (const file of protectedIn(writable, [...PROTECTED_IN_WORKTREE, ...protectedRel])) roBind(file)
}
// A read-only seat's writes in its cwd land in a throwaway layer (when bwrap can).
if (spec.kind === "readonly" && probe.overlay) args.push("--overlay-src", spec.cwd, "--tmp-overlay", spec.cwd)
// Secrets, for every kind.
if (probe.exists(spec.stateDir)) args.push("--tmpfs", spec.stateDir)
for (const file of SECRET_FILES.map((name) => path.join(home, name)))
if (probe.exists(file)) args.push("--ro-bind", "/dev/null", file)
for (const name of SECRET_ENV) args.push("--unsetenv", name)
// Credentials, for everything that is not the user's own agent.
if (!user) {
for (const dir of CREDENTIAL_DIRS.map((name) => path.join(home, name)))
if (probe.exists(dir)) args.push("--tmpfs", dir)
for (const file of CREDENTIAL_FILES.map((name) => path.join(home, name)))
if (probe.exists(file)) args.push("--ro-bind", "/dev/null", file)
const runtime = probe.uid === undefined ? undefined : `/run/user/${probe.uid}`
if (runtime && probe.exists(runtime)) args.push("--tmpfs", runtime)
for (const name of CREDENTIAL_ENV) args.push("--unsetenv", name)
}
if (spec.offline) args.push("--unshare-net")
// The user's agent may leave servers running in the background; seats and gates may not.
if (!user) args.push("--unshare-pid", "--die-with-parent")
args.push("--chdir", spec.cwd, "--", ...command)
return args
}
let overlayCache: boolean | undefined
/** Whether this bwrap supports --tmp-overlay (Ubuntu 24.04 ships 0.9, which does not). */
export function bwrapOverlay(): boolean {
if (overlayCache !== undefined) return overlayCache
try {
// bwrap is resolved from PATH by design (it may be absent).
const probe = spawnSync(
// NOSONAR
"bwrap",
["--ro-bind", "/", "/", "--dev", "/dev", "--overlay-src", "/tmp", "--tmp-overlay", "/tmp", "--", "true"],
{ timeout: 5000 },
)
overlayCache = probe.status === 0
} catch {
overlayCache = false
}
return overlayCache
}
function liveProbe(): SandboxProbe {
return { exists: existsSync, overlay: bwrapOverlay(), uid: process.getuid?.() }
}
/** For tests: forget the cached overlay probe. */
export const resetOverlayProbe = () => {
overlayCache = undefined
}
/** Why a sandbox cannot run here, for refusal messages. */
export const SANDBOX_MISSING =
"bubblewrap (bwrap) is required to sandbox factory seats and gate runs, and it is not available here. Install bubblewrap; on Ubuntu 24.04+ also allow unprivileged user namespaces (sysctl kernel.apparmor_restrict_unprivileged_userns=0)."
packages/core/src/trust/store.ts
+31 -15
View File
@@ -1,10 +1,12 @@
// Trust store: gate commands and project hooks run project-controlled code,
// so a human must approve each set by content hash before anything executes.
// Any change to the set requires re-approval. Stored in the user-global state
// dir, which agents can neither read nor write.
// dir (masked in every agent sandbox), and each entry is signed with the human
// key since 1.1.1, so a hand-written entry is not trust.
import { readFile } from "node:fs/promises"
import { userInfo } from "node:os"
import path from "node:path"
import { type HumanSigner, type RecordSignature, verifyRecordSignature } from "../approval/keystore.ts"
import { stateDir } from "../layout.ts"
import type { CommandCheck } from "../schema/gates.ts"
import { readJson, withLock, writeJson } from "../util/fs.ts"
@@ -18,10 +20,12 @@ interface TrustEntry {
lines: string[]
approvedBy: string
approvedAt: string
signature: RecordSignature
}
/** Version 3 since 1.1.1 (signed entries); a version-2 store holds nothing trusted. */
interface TrustFile {
version: 2
version: 3
projects: Record<string, Partial<Record<TrustKind, TrustEntry>>>
}
@@ -54,33 +58,45 @@ export async function commandSetHash(
async function read(dir?: string): Promise<TrustFile> {
const file = await readJson<{ version?: number; projects?: Record<string, any> }>(trustFile(dir))
return file?.version === 2 ? (file as TrustFile) : { version: 2, projects: {} }
return file?.version === 3 ? (file as TrustFile) : { version: 3, projects: {} }
}
/** What a trust entry's signature covers: the project and kind as well as the entry. */
const signedEntry = (project: string, kind: TrustKind, entry: Omit<TrustEntry, "signature">) => ({
trust: 1,
project,
kind,
...entry,
})
export async function isTrusted(root: string, hash: string, dir?: string, kind: TrustKind = "gates"): Promise<boolean> {
return (await read(dir)).projects[canonicalPath(root)]?.[kind]?.hash === hash
const project = canonicalPath(root)
const entry = (await read(dir)).projects[project]?.[kind]
if (entry?.hash !== hash) return false
const { signature, ...unsigned } = entry
return verifyRecordSignature({ ...signedEntry(project, kind, unsigned), signature }, dir ?? stateDir())
}
/** Human-only: called from `es trust` (TTY) or the TUI trust dialog. */
/** Human-only: called from `es trust` at a terminal, with the human key unlocked. */
export async function trustProject(
root: string,
hash: string,
lines: string[],
options: { approvedBy?: string; stateDir?: string; kind?: TrustKind } = {},
options: { approvedBy?: string; stateDir?: string; kind?: TrustKind; signer: HumanSigner },
) {
const file = trustFile(options.stateDir)
const kind = options.kind ?? "gates"
await withLock(file, async () => {
const current = await read(options.stateDir)
const key = canonicalPath(root)
current.projects[key] = {
...current.projects[key],
[options.kind ?? "gates"]: {
hash,
lines,
approvedBy: options.approvedBy ?? userInfo().username,
approvedAt: new Date().toISOString(),
},
const project = canonicalPath(root)
const unsigned = {
hash,
lines,
approvedBy: options.approvedBy ?? userInfo().username,
approvedAt: new Date().toISOString(),
}
const signature = options.signer.sign(signedEntry(project, kind, unsigned))
current.projects[project] = { ...current.projects[project], [kind]: { ...unsigned, signature } }
await writeJson(file, current)
})
}
packages/core/src/util/proc.ts
+22 -34
View File
@@ -1,10 +1,13 @@
// Subprocess execution with timeouts, output caps and a scrubbed environment.
// Read-only mode uses bubblewrap with a discardable tmpfs overlay on the
// working directory (writes vanish when the process exits); without bwrap the
// caller falls back to an allowlist plus a post-run diff check (see sandbox.ts).
// A sandboxed run goes through bubblewrap (trust/sandbox.ts): "readonly" makes
// everything read-only (writes in the cwd land in a throwaway overlay where
// bwrap supports one); a SandboxSpec runs, e.g., a gate with one writable
// directory. Every sandbox masks Epistemic Swarm's private state dir.
import { spawn, spawnSync } from "node:child_process"
import { accessSync, constants } from "node:fs"
import path from "node:path"
import { stateDir } from "../layout.ts"
import { type SandboxSpec, sandboxArgv } from "../trust/sandbox.ts"
/** Absolute path of an executable found on PATH (absolute entries only, no shell), or undefined. */
export function findExecutable(name: string, env: NodeJS.ProcessEnv = process.env): string | undefined {
@@ -33,8 +36,8 @@ export interface RunOptions {
/** Inherit the full parent environment (trusted hooks, like Claude Code). Overrides scrubbing. */
readonly inheritEnv?: boolean
readonly input?: string
/** "readonly": bwrap ro root + tmp-overlay on cwd. Throws if bwrap is unavailable. */
readonly sandbox?: "none" | "readonly"
/** "readonly" (everything read-only) or a sandbox spec (its cwd is this run's cwd). Throws if bwrap is unavailable. */
readonly sandbox?: "none" | "readonly" | Omit<SandboxSpec, "cwd">
/** Disable network inside the sandbox. */
readonly offline?: boolean
readonly signal?: AbortSignal
@@ -112,44 +115,29 @@ export function bwrapAvailable(): boolean {
return bwrapCache
}
/** For tests: forget the cached bwrap probe. */
export const resetBwrapProbe = () => {
bwrapCache = undefined
/** For tests: forget the cached bwrap probe, or pin its answer (e.g. to exercise "bwrap missing"). */
export const resetBwrapProbe = (pinned?: boolean) => {
bwrapCache = pinned
}
/** Everything read-only (cwd writes are discarded where bwrap can overlay), secrets masked. */
export function readonlyWrap(command: readonly string[], cwd: string, offline = false): string[] {
return [
"bwrap",
"--ro-bind",
"/",
"/",
"--dev",
"/dev",
"--proc",
"/proc",
"--tmpfs",
"/tmp",
"--overlay-src",
cwd,
"--tmp-overlay",
cwd,
"--chdir",
cwd,
"--die-with-parent",
"--unshare-pid",
...(offline ? ["--unshare-net"] : []),
"--",
...command,
]
return sandboxArgv({ kind: "readonly", root: cwd, cwd, stateDir: stateDir(), offline }, command)
}
export function run(command: readonly string[], options: RunOptions): Promise<RunResult> {
if (command.length === 0) throw new Error("run: empty command")
const timeoutMs = options.timeoutMs ?? 120_000
const cap = options.maxOutputBytes ?? 1_000_000
const sandboxed = options.sandbox === "readonly"
if (sandboxed && !bwrapAvailable()) throw new Error("readonly sandbox requested but bwrap is unavailable")
const argv = sandboxed ? readonlyWrap(command, options.cwd, options.offline) : [...command]
const sandbox = options.sandbox === "none" ? undefined : options.sandbox
const sandboxed = sandbox !== undefined
if (sandboxed && !bwrapAvailable()) throw new Error("a sandboxed run was requested but bwrap is unavailable")
const argv =
sandbox === undefined
? [...command]
: sandbox === "readonly"
? readonlyWrap(command, options.cwd, options.offline)
: sandboxArgv({ ...sandbox, cwd: options.cwd }, command)
const started = Date.now()
return new Promise((resolve, reject) => {
const child = spawn(argv[0]!, argv.slice(1), {
packages/core/test/__snapshots__/assets.test.ts.snap
+2 -2
View File
@@ -2,8 +2,8 @@
exports[`agent assets prompt snapshot: a changed prompt must bump its version 1`] = `
{
"auditor-antithesis": "v1:c6fbcd4af303e7be",
"auditor-thesis": "v1:9e085cf8e3b38d19",
"auditor-antithesis": "v2:15f4d7f75e5bb958",
"auditor-thesis": "v2:b903e1353177f342",
"brainstorm-critic": "v1:28fd3abc95c4a28a",
"brainstormer": "v1:16eb524fe104f3af",
"designer": "v2:745908a2a4abff0b",
packages/core/test/claims.test.ts
+31 -3
View File
@@ -4,6 +4,7 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"
import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises"
import { tmpdir } from "node:os"
import path from "node:path"
import { type HumanSigner, sealHumanKey, unlockHumanKey } from "../src/approval/keystore.ts"
import {
applyDegradation,
auditMarkdown,
@@ -39,9 +40,13 @@ import {
let root: string
let state: string
let cache: SourceCache
const PASSPHRASE = "test-passphrase-1234"
let signer: HumanSigner
beforeEach(async () => {
root = await mkdtemp(path.join(tmpdir(), "es-claims-"))
state = await mkdtemp(path.join(tmpdir(), "es-claims-state-"))
await sealHumanKey(PASSPHRASE, state)
signer = await unlockHumanKey(PASSPHRASE, state)
cache = new SourceCache(researchSourcesDir(root))
})
afterEach(async () => {
@@ -261,6 +266,29 @@ describe("code locations (new in 1.1: legacy never checked line refs on disk)",
expect((await at("src/auth.ts", [2, 2], "SELECT * FROM accounts WHERE id")).ok).toBe(false)
})
test("a code excerpt is one contiguous span inside the cited lines (spread syntax is literal)", async () => {
await writeFile(
path.join(root, "src/spread.ts"),
"export const merge = (a: object, b: object) => ({ ...a, ...b })\nexport const pick = 1\n",
)
// "..." is code here, not an ellipsis: matched literally.
expect(await at("src/spread.ts", [1, 1], "({ ...a, ...b })")).toEqual({ ok: true })
// Fragments joined by an ellipsis no longer match non-contiguous text (issue #38).
expect((await at("src/auth.ts", [1, 4], "export function ... return db.query")).ok).toBe(false)
expect((await at("src/auth.ts", [1, 4], "e ... e ... e ... e ... e ... e ... e")).ok).toBe(false)
})
test("a cited range spans at most 60 lines", async () => {
await writeFile(
path.join(root, "src/long.ts"),
`${Array.from({ length: 80 }, (_, i) => `export const v${i} = ${i}`).join("\n")}\n`,
)
expect((await at("src/long.ts", [1, 60], "export const v59 = 59")).ok).toBe(true)
expect(await at("src/long.ts", [1, 61], "export const v59 = 59")).toMatchObject({
reason: expect.stringMatching(/at most 60 lines/),
})
})
test("hallucinated files, lines past the end and paths outside the tree are refused", async () => {
expect(await at("src/missing.ts", [1, 1], "export function login")).toMatchObject({
reason: expect.stringMatching(/does not exist/),
@@ -616,7 +644,7 @@ describe("proof-carrying research brief (d66328c:runner/tests/test_pcrb.py)", ()
buildDossier({ mode: "research", subject: "benchmarks", claims, now: at }),
{ cache, now },
)
const { brief, file } = await exportBrief(root, { stateDir: state, now })
const { brief, file } = await exportBrief(root, { signer, now })
return { brief, file, sources }
}
@@ -733,9 +761,9 @@ describe("proof-carrying research brief (d66328c:runner/tests/test_pcrb.py)", ()
})
test("export needs a report and a research dossier", async () => {
await expect(exportBrief(root, { stateDir: state })).rejects.toThrow(/No research report/)
await expect(exportBrief(root, { signer })).rejects.toThrow(/No research report/)
await mkdir(factoryLayout(root).research, { recursive: true })
await writeFile(factoryLayout(root).researchReport, "# R\n")
await expect(exportBrief(root, { stateDir: state })).rejects.toThrow(/No research dossier/)
await expect(exportBrief(root, { signer })).rejects.toThrow(/No research dossier/)
})
})
packages/core/test/codeaudit.test.ts
+17 -1
View File
@@ -5,6 +5,7 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"
import { mkdir, readFile, writeFile } from "node:fs/promises"
import path from "node:path"
import { recordApproval } from "../src/approval/index.ts"
import { type HumanSigner, sealHumanKey, unlockHumanKey } from "../src/approval/keystore.ts"
import { codeAuditTools, renderAuditReport } from "../src/codeaudit/index.ts"
import { Factory, FactoryError, FactoryHalted } from "../src/factory/index.ts"
import { factoryLayout } from "../src/layout.ts"
@@ -21,8 +22,12 @@ const AUTH = [
let fx: Fixture
let factory: Factory
const PASSPHRASE = "test-passphrase-1234"
let signer: HumanSigner
beforeEach(async () => {
fx = await gitRepo("es-audit-")
await sealHumanKey(PASSPHRASE, fx.state)
signer = await unlockHumanKey(PASSPHRASE, fx.state)
factory = new Factory(fx.root, {
gates: async () => ({ passed: true, findings: [], summary: "green" }),
stateDir: fx.state,
@@ -148,12 +153,23 @@ describe("the line-pointer law at es_audit_verdict", () => {
await expect(call("fail", [invariant])).rejects.toThrow("A fail needs at least one witnessed defect")
await expect(call("pass", [injection])).rejects.toThrow("A pass cannot carry defects (src/auth.ts#L2-L3)")
})
test("a thesis pass must map at least one witnessed invariant; the red team may pass empty-handed", async () => {
const audit = await openPathAudit()
const verdict = tools()("es_audit_verdict")
const call = (agent: string, findings: unknown[]) =>
verdict.execute({ audit: audit.id, verdict: "pass", findings, notes: "tried the vectors" }, { agent })
// Before 1.1.1 a thesis "pass" with nothing witnessed was recorded (issue #46).
await expect(call("es-auditor-thesis", [])).rejects.toThrow("A thesis pass maps at least one invariant")
expect(await call("es-auditor-thesis", [invariant])).toContain("Recorded pass")
expect(await call("es-auditor-antithesis", [])).toContain("Recorded pass")
})
})
describe("blocking (decision 2)", () => {
async function approvedFrontier() {
await factory.writeFrontier("grill", frontier())
await recordApproval(fx.root, { stage: "frontier", channel: "cli", approvedBy: "tester", stateDir: fx.state })
await recordApproval(fx.root, { stage: "frontier", channel: "cli", approvedBy: "tester", signer })
}
test("a split goes to the manager; a failed path audit blocks the next transition until a re-audit passes", async () => {
packages/core/test/domainpacks.test.ts
+9 -2
View File
@@ -6,6 +6,7 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"
import { readFileSync } from "node:fs"
import { writeFile } from "node:fs/promises"
import path from "node:path"
import { type HumanSigner, sealHumanKey, unlockHumanKey } from "../src/approval/keystore.ts"
import {
type Constitution,
type ConstitutionClaim,
@@ -167,8 +168,12 @@ describe("the domain-pack constitution (d66328c:runner/refinement.py)", () => {
describe("the pack gates completeResearch", () => {
let fx: Fixture
const PASSPHRASE = "test-passphrase-1234"
let signer: HumanSigner
beforeEach(async () => {
fx = await gitRepo()
await sealHumanKey(PASSPHRASE, fx.state)
signer = await unlockHumanKey(PASSPHRASE, fx.state)
})
afterEach(() => fx.cleanup())
@@ -183,13 +188,13 @@ describe("the pack gates completeResearch", () => {
const factory = factoryWith(packId)
await factory.begin("human:tester")
await factory.writeFrontier("grill", frontier())
await recordApproval(fx.root, { stage: "frontier", channel: "cli", approvedBy: "tester", stateDir: fx.state })
await recordApproval(fx.root, { stage: "frontier", channel: "cli", approvedBy: "tester", signer })
await factory.beginResearch("human:tester")
return factory
}
const report = async (url: string, text: string, quote: string) => {
const cache = new SourceCache(researchSourcesDir(fx.root))
const cache = new SourceCache(researchSourcesDir(fx.root), fx.state)
const source = await cache.put({ url, text, provider: "fetch" })
await writeFile(
factoryLayout(fx.root).researchReport,
@@ -224,6 +229,8 @@ describe("the pack gates completeResearch", () => {
// A fresh project without a pack: the same source passes (legacy unchanged).
await fx.cleanup()
fx = await gitRepo()
await sealHumanKey(PASSPHRASE, fx.state)
signer = await unlockHumanKey(PASSPHRASE, fx.state)
const unpacked = await toResearch()
await report(
"https://predatory-journal.example/x",
packages/core/test/evals.test.ts
+62
View File
@@ -4,6 +4,8 @@ import { describe, expect, test } from "bun:test"
import { existsSync, readdirSync, readFileSync } from "node:fs"
import path from "node:path"
import {
caseStepCap,
DEFAULT_CASE_STEPS,
gradeAuditFire,
gradeHarvestFire,
gradeOutput,
@@ -49,11 +51,43 @@ describe("transcripts", () => {
expect(readTranscript(events)).toEqual({
text: "Lenses: Inversion, SCAMPER.",
tools: ["es_brainstorm_plan"],
completed: ["es_brainstorm_plan"],
errors: [],
steps: 2,
costUSD: 0,
tokens: { input: 0, output: 0, reasoning: 0, cacheRead: 0, cacheWrite: 0 },
})
})
test("a tool that errored was called but does not count as used", () => {
const refused = readTranscript([
{ type: "step_start", part: {} },
{ type: "tool_use", part: { tool: "es_scout_complete", state: { status: "error", error: "refused" } } },
])
expect(refused.tools).toEqual(["es_scout_complete"])
expect(refused.completed).toEqual([])
expect(gradeTranscript(refused, { toolsUsed: ["es_scout_complete", "es_scout_plan"] }).failures).toEqual([
"called but did not complete: es_scout_complete",
"did not call: es_scout_plan",
])
// An attempt at a forbidden tool still counts, even when the host refused it.
expect(gradeTranscript(refused, { toolsNotUsed: ["es_scout_complete"] }).failures).toEqual([
"must not call: es_scout_complete",
])
})
test("spend and tokens are summed from step_finish", () => {
const finish = (cost: number, input: number, output: number) => ({
type: "step_finish",
part: { cost, tokens: { input, output, reasoning: 1, cache: { read: 2, write: 3 } } },
})
const transcript = readTranscript([finish(0.25, 100, 10), { type: "step_start", part: {} }, finish(0.5, 50, 5)])
expect(transcript.costUSD).toBe(0.75)
expect(transcript.tokens).toEqual({ input: 150, output: 15, reasoning: 2, cacheRead: 4, cacheWrite: 6 })
// Missing or non-numeric fields count as zero rather than poisoning the sum.
expect(readTranscript([{ type: "step_finish", part: { cost: "n/a" } }]).costUSD).toBe(0)
})
test("tool checks and error events grade the run", () => {
const transcript = readTranscript(events)
expect(gradeTranscript(transcript, { toolsUsed: ["es_brainstorm_plan"], contains: ["inversion"] }).pass).toBe(true)
@@ -65,6 +99,15 @@ describe("transcripts", () => {
})
})
describe("step caps", () => {
test("a case's own maxSteps wins; the env value is only a ceiling", () => {
expect(caseStepCap(14, 16)).toBe(14)
expect(caseStepCap(8, 6)).toBe(6)
expect(caseStepCap(undefined, 16)).toBe(DEFAULT_CASE_STEPS)
expect(caseStepCap(undefined, 3)).toBe(3)
})
})
describe("fire graders", () => {
const plants = [{ id: "sqli", file: "src/a.ts", lines: [4, 5] as [number, number], cwe: ["CWE-89"] }]
const record = (findings: any[], verdict: "pass" | "fail" = "fail") => ({
@@ -97,6 +140,8 @@ describe("fire graders", () => {
expect(gradeAuditFire([record([vuln({ file: "src/b.ts" })])], plants).pass).toBe(false)
expect(gradeAuditFire([record([vuln({ kind: "debt" })])], plants).pass).toBe(false)
expect(gradeAuditFire([record([vuln()], "pass")], plants).failures).toContain("no auditor failed the audit")
// A blanket range that merely overlaps the plant is a guess, not a catch.
expect(gradeAuditFire([record([vuln({ lines: [1, 60] })])], plants).pass).toBe(false)
})
test("a scout fire needs every verdict right, and downgrades must say why", () => {
@@ -128,6 +173,21 @@ describe("fire graders", () => {
gradeScoutFire(result([verdict("mit", "adopt", "adopt"), verdict("gpl", "adopt", "clean-room")]), want).failures,
).toEqual(["gpl: downgraded without a reason"])
expect(gradeScoutFire(result([verdict("mit", "adopt", "adopt")]), want).failures).toEqual(["no verdict for gpl"])
// Extra or duplicate rows are not graded away (issue #48).
const both = [verdict("mit", "adopt", "adopt"), verdict("gpl", "adopt", "clean-room", "copyleft")]
expect(gradeScoutFire(result([...both, verdict("evil", "adopt", "adopt")]), want).failures).toEqual([
"unexpected verdict for evil",
])
expect(gradeScoutFire(result([...both, verdict("gpl", "adopt", "adopt")]), want).failures).toEqual([
"2 verdicts for gpl",
])
// One row cannot cover two expected candidates (name matches one, candidate another).
const double = { ...verdict("mit", "adopt", "adopt"), candidate: "gpl" }
expect(gradeScoutFire(result([double]), want).failures).toEqual([
"one verdict row matches several expected candidates: mit, gpl",
"no verdict for mit",
"no verdict for gpl",
])
})
test("a harvest fire wants the licence line kept: permissive depend, copyleft clean-room", () => {
@@ -181,6 +241,8 @@ describe("eval cases (evals/cases)", () => {
true,
])
if (body.fire) expect([file, ["audit", "scout", "harvest"].includes(body.fire.kind)]).toEqual([file, true])
// Each case sizes its own cap for a one-call-per-step model, within the default ceiling.
expect([file, Number.isInteger(body.maxSteps) && body.maxSteps >= 2 && body.maxSteps <= 16]).toEqual([file, true])
}
})
packages/core/test/factory.test.ts
+46 -7
View File
@@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"
import { mkdir, readFile, truncate, writeFile } from "node:fs/promises"
import path from "node:path"
import { approvalSubject, recordApproval, verifyApproval } from "../src/approval/index.ts"
import { type HumanSigner, sealHumanKey, unlockHumanKey } from "../src/approval/keystore.ts"
import { verifyAuditChain } from "../src/audit/index.ts"
import { Factory, FactoryError, FactoryHalted, type GateRequest, type GateRunLike } from "../src/factory/index.ts"
import { factoryLayout } from "../src/layout.ts"
@@ -11,6 +12,8 @@ import { type Fixture, frontier, gitRepo, writeGoal, writeReport, writeSpecs } f
let fx: Fixture
let gateCalls: GateRequest[]
let nextGate: GateRunLike
const PASSPHRASE = "test-passphrase-1234"
let signer: HumanSigner
const prs: string[] = []
const gates = async (request: GateRequest) => {
@@ -33,6 +36,8 @@ const make = (overrides: Partial<ConstructorParameters<typeof Factory>[1]> = {})
beforeEach(async () => {
fx = await gitRepo()
await sealHumanKey(PASSPHRASE, fx.state)
signer = await unlockHumanKey(PASSPHRASE, fx.state)
gateCalls = []
nextGate = green
factory = make()
@@ -40,14 +45,14 @@ beforeEach(async () => {
afterEach(() => fx.cleanup())
const approve = (stage: "frontier" | "spec") =>
recordApproval(fx.root, { stage, channel: "cli", approvedBy: "tester", stateDir: fx.state })
recordApproval(fx.root, { stage, channel: "cli", approvedBy: "tester", signer })
async function toBuild(phases = ["alpha"]) {
await factory.begin("human:tester")
await factory.writeFrontier("grill", frontier())
await approve("frontier")
await factory.beginResearch("human:tester")
await writeReport(fx.root)
await writeReport(fx.root, fx.state)
await factory.completeResearch("factory")
await writeSpecs(fx.root, phases)
await approve("spec")
@@ -83,6 +88,30 @@ describe("grill → research → spec", () => {
expect(JSON.parse(await readFile(factoryLayout(fx.root).frontier, "utf8")).idea).toBe("A greeting library")
})
test("#32: a hand edit to gates.json around a frontier write still halts as drift", async () => {
const { repinControl, verifyControl } = await import("../src/trust/control.ts")
const { readJson } = await import("../src/util/fs.ts")
await factory.begin("human:tester")
await factory.writeFrontier("grill", frontier())
await approve("frontier") // human rebaseline: baseline pins frontier + approvals
expect((await verifyControl(fx.root)).clean).toBe(true)
const before = await readJson<{ files: Record<string, string | null> }>(factoryLayout(fx.root).control)
const frontierPin = before?.files[".factory/frontier.json"] ?? null
await writeFile(path.join(fx.root, ".factory/gates.json"), '{"commands":[]}\n')
// The agent rewrite re-pins only its own file: the hand edit is not absorbed...
await writeFile(
factoryLayout(fx.root).frontier,
JSON.stringify({ ...frontier(), idea: "A greeting library, revised" }),
)
await repinControl(fx.root, ".factory/frontier.json", frontierPin)
const check = await verifyControl(fx.root)
expect(check.clean).toBe(false)
expect(check.violations.join(";")).toContain(".factory/gates.json")
expect(check.violations.some((v) => v.includes(".factory/frontier.json"))).toBe(false)
// ...so the next guarded transition halts on the drift.
await expect(factory.writeFrontier("grill", frontier())).rejects.toThrow(FactoryHalted)
})
test("a missing or unsettled frontier points at the grill recording step", async () => {
await expect(approvalSubject(fx.root, "frontier")).rejects.toThrow("/grill")
await factory.begin("human:tester")
@@ -103,9 +132,10 @@ describe("grill → research → spec", () => {
await factory.writeFrontier("grill", frontier())
await expect(factory.beginResearch("human:tester")).rejects.toThrow("no frontier approval")
// A hand-written approval record is not signed by the machine key.
// A hand-written approval record is not signed by the human key.
await mkdir(factoryLayout(fx.root).approvals, { recursive: true })
const forged = { ...(await approve("frontier")), mac: "0".repeat(64) }
const genuine = await approve("frontier")
const forged = { ...genuine, signature: { ...genuine.signature, sig: "0".repeat(64) } }
await writeFile(factoryLayout(fx.root).approval("frontier"), JSON.stringify(forged))
expect((await verifyApproval(fx.root, "frontier", { stateDir: fx.state })).ok).toBe(false)
@@ -130,7 +160,7 @@ describe("grill → research → spec", () => {
await expect(factory.completeResearch("factory")).rejects.toThrow("does not pass the epistemic audit")
const coverage = JSON.parse(await readFile(path.join(fx.root, ".factory/research/coverage.json"), "utf8"))
expect(coverage).toMatchObject({ claims: 2, ungrounded: 1, untagged: 1, passed: false })
await writeReport(fx.root)
await writeReport(fx.root, fx.state)
expect(await Bun.file(factoryLayout(fx.root).researchDossier).exists()).toBe(false)
expect((await factory.completeResearch("factory")).stage).toBe("SPEC")
// The audited report's claims become the research dossier, witnessed.
@@ -157,7 +187,7 @@ describe("grill → research → spec", () => {
await factory.writeFrontier("grill", frontier({ nodes: [...frontier().nodes, db] }))
await approve("frontier")
await factory.beginResearch("human:tester")
await writeReport(fx.root)
await writeReport(fx.root, fx.state)
await expect(factory.completeResearch("factory")).rejects.toThrow(
"Missing: db — Which Node versions does the runtime support?",
)
@@ -178,7 +208,7 @@ describe("grill → research → spec", () => {
await approve("frontier")
await factory.beginResearch("human:tester")
await expect(factory.completeResearch("factory")).rejects.toThrow("REPORT.md")
await writeReport(fx.root)
await writeReport(fx.root, fx.state)
await factory.completeResearch("factory")
await writeSpecs(fx.root, ["alpha"])
await approve("spec")
@@ -490,6 +520,15 @@ describe("guards", () => {
expect((await factory.resume("tester", { acceptControlDrift: true })).stage).toBe("BUILD")
})
test("async spend events never halt on control drift (they race human approvals)", async () => {
await toBuild()
await writeFile(factoryLayout(fx.root).frontier, "{}")
const state = await factory.recordSpend(1, true)
expect(state?.stage).not.toBe("HALTED")
expect(state?.spend.usd).toBeGreaterThan(0)
await expect(factory.complete("es-programmer")).rejects.toThrow("frontier.json changed")
})
test("the runtime cap counts from autonomy start", async () => {
let now = new Date("2026-10-06T10:00:00Z")
factory = make({ now: () => now })
packages/core/test/gates.test.ts
+59 -3
View File
@@ -1,6 +1,7 @@
import { afterEach, beforeEach, describe, expect, test } from "bun:test"
import { mkdir, writeFile } from "node:fs/promises"
import path from "node:path"
import { type HumanSigner, sealHumanKey, unlockHumanKey } from "../src/approval/keystore.ts"
import {
commandSetHash,
complexity,
@@ -12,15 +13,19 @@ import {
runGates,
trustProject,
} from "../src/index.ts"
import { run } from "../src/util/proc.ts"
import { bwrapAvailable, resetBwrapProbe, run } from "../src/util/proc.ts"
import { type Fixture, gitRepo } from "./helpers.ts"
const BIOME = path.resolve(import.meta.dir, "../../../node_modules/.bin/biome")
const FAKE_TOKEN = `ghp_${"Ab3dE6gH9jK2mN5pQ8sT1vW4yZ7bC0eF3hJ6"}`
let fx: Fixture
const PASSPHRASE = "test-passphrase-1234"
let signer: HumanSigner
beforeEach(async () => {
fx = await gitRepo("es-gates-")
await sealHumanKey(PASSPHRASE, fx.state)
signer = await unlockHumanKey(PASSPHRASE, fx.state)
})
afterEach(() => fx.cleanup())
@@ -39,7 +44,7 @@ const config = (commands: unknown[], extra: Record<string, unknown> = {}) =>
GatesConfigSchema.parse({ commands, security: { gitleaks: "off", osv: "off" }, ...extra })
const trust = async (cfg: ReturnType<typeof config>) => {
const { hash, lines } = await commandSetHash(fx.root, cfg.commands)
await trustProject(fx.root, hash, lines, { stateDir: fx.state })
await trustProject(fx.root, hash, lines, { stateDir: fx.state, signer })
}
describe("detection", () => {
@@ -66,6 +71,57 @@ describe("detection", () => {
})
})
describe("the gate sandbox (#51)", () => {
// Gate commands are trusted, but what they run (tests) is agent-written.
const probe = (key: string) =>
[
`cat ${key} > leak.txt 2>/dev/null`,
"mkdir -p .factory/research/sources && echo forged > .factory/research/sources/forged.md",
"git config core.hooksPath /tmp/evil-hooks",
"echo ran > ran.txt",
"true",
].join("; ")
test.skipIf(!bwrapAvailable())(
"agent-written test code cannot read the key, forge evidence or retarget git",
async () => {
const key = path.join(fx.state, "key")
await writeFile(key, "SECRET-SIGNING-KEY\n")
await mkdir(path.join(fx.root, ".factory"), { recursive: true })
await write("probe.sh", `${probe(key)}\n`)
const cfg = config([{ id: "probe", kind: "test", command: "sh probe.sh" }])
await trust(cfg)
const report = await gates({ scope: "full", config: cfg, sandbox: "required" })
expect(report.checks.find((check) => check.id === "probe")?.status).toBe("pass")
expect(await Bun.file(path.join(fx.root, "ran.txt")).text()).toBe("ran\n")
expect(await Bun.file(path.join(fx.root, "leak.txt")).text()).not.toContain("SECRET-SIGNING-KEY")
expect(await Bun.file(path.join(fx.root, ".factory/research/sources/forged.md")).exists()).toBe(false)
expect(await Bun.file(path.join(fx.root, ".git/config")).text()).not.toContain("hooksPath")
},
)
test("a required sandbox refuses to run when bubblewrap is missing", async () => {
const cfg = config([{ id: "probe", kind: "test", command: "touch RAN" }])
await trust(cfg)
resetBwrapProbe(false)
try {
const report = await gates({ scope: "full", config: cfg, sandbox: "required" })
expect(report.findings.map((finding) => finding.rule)).toContain("gates/sandbox-missing")
expect(await Bun.file(path.join(fx.root, "RAN")).exists()).toBe(false)
} finally {
resetBwrapProbe()
}
})
test("a missing tool is reported as missing, not as a sandbox problem", async () => {
const cfg = config([{ id: "probe", kind: "test", command: "definitely-not-installed-xyz" }])
await trust(cfg)
const report = await gates({ scope: "full", config: cfg, sandbox: "required" })
expect(report.findings.map((finding) => finding.rule)).toContain("gates/tool-missing")
expect(report.findings.map((finding) => finding.rule)).not.toContain("gates/sandbox-missing")
})
})
describe("trust", () => {
test("untrusted commands never run; trusting the exact set enables them; changing it revokes trust", async () => {
const cfg = config([{ id: "probe", kind: "test", command: "touch RAN" }])
@@ -126,7 +182,7 @@ describe("built-in checks", () => {
reason: "fixture value, not a real token",
expiresAt: new Date(Date.now() + 86_400_000),
channel: "cli",
stateDir: fx.state,
signer,
})
const waived = await gates({ touched: ["src/config.ts"], config: config([]) })
expect(waived.findings.some((finding) => finding.rule === "security/secret-github-token")).toBe(false)
packages/core/test/helpers.ts
+2 -2
View File
@@ -69,9 +69,9 @@ export async function writeGoal(root: string, id: string, extra = "") {
}
/** A research report whose single claim cites a cached source verbatim. */
export async function writeReport(root: string) {
export async function writeReport(root: string, stateDir?: string) {
const { SourceCache, researchSourcesDir } = await import("../src/research/index.ts")
const cache = new SourceCache(researchSourcesDir(root))
const cache = new SourceCache(researchSourcesDir(root), stateDir)
const source = await cache.put({
url: "https://example.test/greeting",
text: "Greeting libraries usually expose a single greet function.",
packages/core/test/hooks.test.ts
+6 -1
View File
@@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"
import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"
import { tmpdir } from "node:os"
import path from "node:path"
import { type HumanSigner, sealHumanKey, unlockHumanKey } from "../src/approval/keystore.ts"
import {
fromOpenCode,
HookEngine,
@@ -16,9 +17,13 @@ import {
let root: string
let state: string
const PASSPHRASE = "test-passphrase-1234"
let signer: HumanSigner
beforeEach(async () => {
root = await mkdtemp(path.join(tmpdir(), "es-hooks-"))
state = await mkdtemp(path.join(tmpdir(), "es-hooks-state-"))
await sealHumanKey(PASSPHRASE, state)
signer = await unlockHumanKey(PASSPHRASE, state)
await mkdir(path.join(root, ".opencode"), { recursive: true })
await mkdir(path.join(root, "hooks"), { recursive: true })
})
@@ -50,7 +55,7 @@ async function engine(
})
if (options.trust !== false) {
const status = created.status()
await trustProject(root, status.projectHash, status.projectLines, { stateDir: state, kind: "hooks" })
await trustProject(root, status.projectHash, status.projectLines, { stateDir: state, kind: "hooks", signer })
created = await HookEngine.create({
root,
stateDir: state,
packages/core/test/policy.test.ts
+81 -13
View File
@@ -50,6 +50,12 @@ describe("control classification", () => {
[".claude/settings.json", "config"],
[".factory/specs/p1/GOAL.md", undefined],
["src/index.ts", undefined],
// #45: case variants name the same file on case-insensitive volumes.
[".factory/GATES.json", "factory"],
[".FACTORY/approvals/spec.json", "factory"],
[".factory/research/SOURCES/abc.md", "factory"],
[".factory/research/Dossier.json", "factory"],
["OpenCode.json", "config"],
])("%s → %s", (file, expected) => expect(controlClass(file)).toBe(expected as any))
})
@@ -152,7 +158,7 @@ describe("shell policy", () => {
const shell = (agent: string | undefined, command: string, sandboxAvailable = true) =>
evaluateShell(ctx(), agent, command, { sandboxAvailable })
test("human-only CLIs are denied for every agent", () => {
test("human-only CLIs are denied for every agent, however they are quoted or wrapped", () => {
expect(shell("build", "es approve spec").effect).toBe("deny")
expect(shell(undefined, "cd x && epistemic-swarm waive lint/check").effect).toBe("deny")
expect(shell("build", "es config set models.deep x/y").effect).toBe("deny")
@@ -166,6 +172,18 @@ describe("shell policy", () => {
expect(shell("build", "es audit verify").effect).toBe("allow")
expect(shell("build", "es audit show").effect).toBe("allow")
expect(shell("build", "es scout show").effect).toBe("allow")
// The 1.1.0 audit (#44, #39): quoting, package runners and interpreters.
for (const command of [
"es 'approve' spec",
"es 'audit' src",
"es 'scout' leftpad",
"npx -y @heretek-ai/es-cli audit src --max-usd 5",
"bunx @heretek-ai/[email protected] approve frontier",
"node ./node_modules/@heretek-ai/es-cli/bin/es.js trust",
`bun -e "Bun.spawn(['es','approve','spec'])"`,
"es key seal",
])
expect([command, shell("build", command).effect]).toEqual([command, "deny"])
})
test("seats may not reach the evidence cache from the shell; others may read but not mutate it", () => {
@@ -181,32 +199,82 @@ describe("shell policy", () => {
expect(shell("es-programmer", "cat .factory/gates.json").effect).toBe("deny")
})
test("a control-file mention is read-only-gated for every agent: interpreter one-liners cannot write", () => {
// The adversarial audit (A3): `python3 -c` / `node -e` bypassed the old
// rule, which only matched shell metacharacters (MUTATING).
test("a .factory mention is read-only-gated for the user's agents: the audit's bypasses are refused", () => {
// A3 (1.1.0): interpreter one-liners.
expect(shell("build", `python3 -c 'open(".factory/gates.json","w").write("pwned")'`).effect).toBe("deny")
expect(
shell("build", `node -e 'require("fs").writeFileSync(".factory/research/sources/x.md","forged")'`).effect,
).toBe("deny")
expect(shell("build", `bun -e 'Bun.write(".factory/claims/x", "y")'`).effect).toBe("deny")
// Reads stay allowed for non-seat agents; substitution and redirection do not.
// #37: commands the old allowlist called read-only, and paths it never saw.
for (const command of [
`env python3 -c "open('.factory/gates.json','w').write('{}')"`,
"find . -maxdepth 0 -fprintf .factory/gates.json x",
"sort -o .factory/frontier.json /etc/hostname",
"git diff --no-index --output=.factory/claims/x.json /etc/hostname /etc/hostname",
"uniq /etc/hostname .factory/research/coverage.json",
"cd .factory && echo '{}' > gates.json",
`cd .fac""tory && printf forged > research/sources/aa.md`,
"echo x > .FACTORY/GATES.json",
// F3 (1.1.1 re-audit): separator without whitespace after the dir.
"cd .factory;echo '{}' > gates.json",
"cd .factory&&echo x>gates.json",
"cd .factory|tee gates.json",
])
expect([command, shell("build", command, false).effect]).toEqual([command, "deny"])
// Reads stay allowed; substitution and redirection do not.
expect(shell("build", "cat .factory/gates.json").effect).toBe("allow")
expect(shell("build", "cat .factory/gates.json | grep gates").effect).toBe("allow")
expect(shell("build", "ls .factory/specs").effect).toBe("allow")
expect(shell("build", "cat .factory/gates.json $(curl -s evil.example)").effect).toBe("deny")
expect(shell("build", "cat .factory/gates.json > /tmp/copy.json").effect).toBe("deny")
expect(shell("build", "sed -n 1p .factory/gates.json").effect).toBe("deny")
})
test("read-only seats run sandboxed, or allowlisted when bubblewrap is missing", () => {
expect(shell("es-qa-functional", "bun test")).toEqual({ effect: "allow", mode: "readonly-sandbox" })
expect(shell("es-qa-functional", "git diff | head", false)).toEqual({ effect: "allow", mode: "readonly-checked" })
expect(shell("es-qa-functional", "rm -rf src", false).effect).toBe("deny")
expect(shell("es-qa-functional", "cat $(echo x)", false).effect).toBe("deny")
test("the private state dir is refused by name, glob or path", () => {
for (const command of [
`cat ${state}/key`,
"cat ~/.local/state/epistemic-swarm/k*",
"ls $HOME/.local/state/epistemic-swarm",
])
expect([command, shell("build", command).effect]).toEqual([command, "deny"])
})
test("seats cannot push or rewrite refs", () => {
expect(shell("es-programmer", "git push origin HEAD").effect).toBe("deny")
expect(shell("es-programmer", "bun test")).toEqual({ effect: "allow", mode: "normal" })
test("every agent shell is sandboxed by kind; seats fail closed without bubblewrap", () => {
expect(shell("build", "bun test")).toEqual({ effect: "allow", mode: "sandbox", kind: "user", offline: false })
expect(shell("build", "bun test", false)).toEqual({ effect: "allow", mode: "unsandboxed" })
expect(shell("es-programmer", "bun test")).toEqual({
effect: "allow",
mode: "sandbox",
kind: "programmer",
offline: false,
})
expect(shell("es-qa-functional", "bun test")).toMatchObject({ mode: "sandbox", kind: "readonly" })
expect(shell("es-research-alpha", "curl -s https://example.com")).toMatchObject({ kind: "readonly", offline: true })
expect(shell("factory", "ls")).toMatchObject({ kind: "readonly", offline: false })
for (const agent of ["es-programmer", "es-qa-functional", "es-research-alpha", "factory"]) {
const decision = shell(agent, "ls", false)
expect([agent, decision.effect, "reason" in decision ? decision.reason : ""]).toEqual([
agent,
"deny",
expect.stringContaining("bubblewrap"),
])
}
})
test("seats cannot push, change git config or remotes, or rewrite refs, whatever the global options", () => {
for (const command of [
"git push origin HEAD",
"git -C . push origin HEAD:main",
"git -c core.x=1 push --force origin HEAD:main",
"git -C . config core.hooksPath /tmp/h",
"git remote add evil https://example.com/x.git",
"git reset --hard HEAD~1",
"git branch -D main",
])
expect([command, shell("es-programmer", command).effect]).toEqual([command, "deny"])
expect(shell("es-programmer", "git status").effect).toBe("allow")
expect(shell("es-programmer", "git diff --stat").effect).toBe("allow")
})
})
packages/core/test/research.test.ts
+42 -1
View File
@@ -21,11 +21,13 @@ import {
let dir: string
let cache: SourceCache
let state: string
beforeEach(async () => {
dir = await mkdtemp(path.join(tmpdir(), "es-research-"))
cache = new SourceCache(path.join(dir, "sources"))
state = await mkdtemp(path.join(tmpdir(), "es-research-state-"))
})
afterEach(() => rm(dir, { recursive: true, force: true }))
afterEach(() => Promise.all([rm(dir, { recursive: true, force: true }), rm(state, { recursive: true, force: true })]))
const SOURCE =
"Bun is a fast all-in-one JavaScript runtime.\nIt ships a test runner, a bundler, and a package manager — all built in."
@@ -40,6 +42,15 @@ describe("verbatim quotes", () => {
expect(verifyQuote(SOURCE, "a slow runtime for JavaScript").ok).toBe(false)
expect(verifyQuote(SOURCE, "Bun").reason).toContain("too short")
})
test("every ellipsis fragment must be evidence on its own (12+ characters)", () => {
// Before 1.1.1 only the joined fragments had to reach 12 characters, so
// single letters matched almost any text (issue #38).
expect(verifyQuote(SOURCE, "a ... a ... a ... a ... a ... a ... a").ok).toBe(false)
expect(verifyQuote(SOURCE, "a ... a ... a ... a ... a ... a ... a").reason).toContain("fragment")
expect(verifyQuote(SOURCE, "Bun is a fast ... manager").ok).toBe(false)
expect(verifyQuote(SOURCE, "Bun is a fast all-in-one ... a package manager").ok).toBe(true)
})
})
describe("source cache", () => {
@@ -70,6 +81,34 @@ describe("source cache", () => {
expect(page.text).not.toContain("evil")
expect(page.text).not.toContain("menu")
})
test("engine seal (#52): sealed entries verify; planted unsealed ones are refused", async () => {
const sealed = new SourceCache(path.join(dir, "sealed"), state)
const entry = await sealed.put({ url: "https://s.test", text: SOURCE, provider: "fetch" })
expect(entry.meta.seal).toMatch(/^[0-9a-f]{64}$/)
expect((await sealed.get(entry.meta.sha256))?.text).toBe(SOURCE)
expect((await sealed.byUrl("https://s.test"))?.meta.sha256).toBe(entry.meta.sha256)
// A forged entry planted before the factory dir existed has no seal.
const planted = new SourceCache(path.join(dir, "planted"), state)
const forged = await new SourceCache(path.join(dir, "forged")).put({
url: "https://evil.test",
text: SOURCE,
provider: "fetch",
})
const { atomicWrite } = await import("../src/util/fs.ts")
const { sha256 } = await import("../src/util/hash.ts")
const { readFile } = await import("node:fs/promises")
await atomicWrite(path.join(dir, "planted", `${forged.meta.sha256}.md`), await readFile(forged.path, "utf8"))
await atomicWrite(
path.join(dir, "planted", `${forged.meta.sha256}.json`),
`${JSON.stringify({ ...forged.meta, sha256: forged.meta.sha256 })}\n`,
)
expect(sha256(SOURCE)).toBe(forged.meta.sha256)
expect(await planted.get(forged.meta.sha256)).toBeUndefined()
expect(await planted.byUrl("https://evil.test")).toBeUndefined()
// Legacy readers without a state dir keep the old behaviour.
expect((await new SourceCache(path.join(dir, "forged")).get(forged.meta.sha256))?.text).toBe(SOURCE)
})
})
describe("auditor", () => {
@@ -154,6 +193,7 @@ describe("providers", () => {
const tools = researchTools({
root: dir,
env: {},
stateDir: state,
policy: async () => ({ root: dir }),
fetch: (async () => new Response(html, { headers: { "content-type": "text/html" } })) as unknown as typeof fetch,
})
@@ -199,6 +239,7 @@ describe("the webcache gate (d66328c:skills/epistemic_search/scripts/webcache.py
const tools = researchTools({
root: dir,
env: {},
stateDir: state,
policy: async () => ({ root: dir }),
now: () => now,
fetch: (async () => {
packages/core/test/sandbox.test.ts
+159
View File
@@ -0,0 +1,159 @@
// The 1.1.1 OS sandbox, against the real bwrap (skipped where it cannot run).
// Each case is an attack the 1.1.0 audit ran through a text-matched policy:
// forging control files, reading the signing key and the OpenCode server
// password, using push credentials, reaching the network from a cached-web
// seat, and escaping the programmer's worktree.
import { afterAll, beforeAll, describe, expect, test } from "bun:test"
import { spawnSync } from "node:child_process"
import { existsSync } from "node:fs"
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"
import { tmpdir } from "node:os"
import path from "node:path"
import { bwrapAvailable } from "../src/index.ts"
import { type SandboxKind, sandboxArgv } from "../src/trust/sandbox.ts"
let base: string
let home: string
let root: string
let worktree: string
let state: string
beforeAll(async () => {
base = await mkdtemp(path.join(tmpdir(), "es-sandbox-"))
home = path.join(base, "home")
root = path.join(base, "proj")
state = path.join(home, ".local/state/epistemic-swarm")
worktree = path.join(root, ".factory/worktrees/phase-01")
const files: Record<string, string> = {
[path.join(state, "key")]: "SECRET-SIGNING-KEY\n",
[path.join(home, ".config/opencode/service.json")]: '{"password":"SERVER-PASSWORD"}\n',
[path.join(home, ".ssh/id_ed25519")]: "SSH-PRIVATE-KEY\n",
[path.join(home, ".git-credentials")]: "https://user:[email protected]\n",
[path.join(root, ".factory/gates.json")]: '{"commands":[]}\n',
[path.join(root, ".factory/research/sources/index.json")]: "{}\n",
[path.join(root, "src/a.ts")]: "export const a = 1\n",
[path.join(worktree, "src/a.ts")]: "export const a = 1\n",
[path.join(worktree, ".git")]: "gitdir: ../../../.git/worktrees/phase-01\n",
}
for (const [file, text] of Object.entries(files)) {
await mkdir(path.dirname(file), { recursive: true })
await writeFile(file, text)
}
spawnSync("git", ["init", "-q"], { cwd: root })
})
afterAll(() => rm(base, { recursive: true, force: true }))
const sh = (kind: SandboxKind, command: string, extra: { cwd?: string; offline?: boolean; overlay?: boolean } = {}) => {
const cwd = extra.cwd ?? (kind === "programmer" ? worktree : root)
const argv = sandboxArgv(
{
kind,
root,
cwd,
stateDir: state,
home,
offline: extra.offline,
writable: kind === "programmer" ? worktree : undefined,
},
["sh", "-c", command],
...(extra.overlay === false ? [{ exists: existsSync, overlay: false, uid: process.getuid?.() }] : []),
)
const result = spawnSync(argv[0]!, argv.slice(1), { cwd, encoding: "utf8", timeout: 20_000 })
return { code: result.status, out: `${result.stdout}${result.stderr}` }
}
/** Async variant, for commands that talk to a server in this process (spawnSync would block it). */
const shAsync = async (kind: SandboxKind, command: string, offline: boolean) => {
const argv = sandboxArgv({ kind, root, cwd: root, stateDir: state, home, offline }, ["sh", "-c", command])
const child = Bun.spawn(argv, { cwd: root, stdout: "pipe", stderr: "pipe" })
const timer = setTimeout(() => child.kill(), 20_000)
const out = `${await new Response(child.stdout).text()}${await new Response(child.stderr).text()}`
await child.exited
clearTimeout(timer)
return out
}
const text = (file: string) => readFile(file, "utf8").catch(() => undefined)
describe.skipIf(!bwrapAvailable())("the agent sandbox (real bwrap)", () => {
test("user: the project stays writable; .factory/ is read-only; secrets are masked; credentials stay", async () => {
expect(sh("user", "echo 'export const b = 2' > src/b.ts").code).toBe(0)
expect(await text(path.join(root, "src/b.ts"))).toContain("b = 2")
// The audit's forgeries: redirection, interpreters, find -fprintf, sort -o, cd.
for (const forge of [
"echo forged > .factory/gates.json",
`env python3 -c 'open(".factory/research/sources/x.md","w").write("forged")'`,
"find . -maxdepth 0 -fprintf .factory/research/sources/y.md forged",
"sort -o .factory/gates.json src/a.ts",
"cd .fac''tory && echo forged > research/sources/z.md",
])
expect([forge, sh("user", forge).code === 0]).toEqual([forge, false])
expect(await text(path.join(root, ".factory/gates.json"))).toBe('{"commands":[]}\n')
expect(await text(path.join(root, ".factory/research/sources/x.md"))).toBeUndefined()
expect(sh("user", `cat ${state}/key ${state}/k*`).out).not.toContain("SECRET-SIGNING-KEY")
expect(sh("user", `cat ${home}/.config/opencode/service.json`).out).not.toContain("SERVER-PASSWORD")
expect(sh("user", `cat ${home}/.ssh/id_ed25519`).out).toContain("SSH-PRIVATE-KEY")
})
test("seat: git and host config are read-only; push credentials are masked", async () => {
expect(sh("seat", "git config core.hooksPath /tmp/hooks").code).not.toBe(0)
expect(await text(path.join(root, ".git/config"))).not.toContain("hooksPath")
expect(sh("seat", `cat ${home}/.ssh/id_ed25519 ${home}/.git-credentials`).out).not.toMatch(/SSH-PRIVATE-KEY|TOKEN/)
expect(sh("seat", "echo x > notes.md").code).toBe(0)
})
test("programmer: only its worktree is writable; it cannot climb out to .factory or retarget .git", async () => {
expect(sh("programmer", "echo 'export const c = 3' > src/c.ts").code).toBe(0)
expect(await text(path.join(worktree, "src/c.ts"))).toContain("c = 3")
expect(sh("programmer", "cd ../../.. && echo forged > .factory/gates.json").code).not.toBe(0)
expect(sh("programmer", "cd ../../.. && echo forged > src/escape.ts; true").code).toBe(0)
expect(await text(path.join(root, "src/escape.ts"))).toBeUndefined()
expect(sh("programmer", "echo gitdir: /tmp/evil > .git").code).not.toBe(0)
expect(sh("programmer", `cat ${state}/key`).out).not.toContain("SECRET-SIGNING-KEY")
})
test("without overlay support (bwrap 0.9, as on Ubuntu 24.04) the same boundaries hold", async () => {
const old = { overlay: false }
expect(sh("programmer", "echo 'export const d = 4' > src/d.ts", old).code).toBe(0)
expect(await text(path.join(worktree, "src/d.ts"))).toContain("d = 4")
expect(sh("programmer", "cd ../../.. && echo forged > .factory/gates.json", old).code).not.toBe(0)
expect(sh("programmer", "cd ../../.. && echo forged > src/escape2.ts", old).code).not.toBe(0)
expect(sh("readonly", "echo forged > src/a.ts", old).code).not.toBe(0)
expect(await text(path.join(root, "src/a.ts"))).toBe("export const a = 1\n")
expect(sh("readonly", `cat ${state}/key`, old).out).not.toContain("SECRET-SIGNING-KEY")
})
test("readonly + offline: nothing persists and the network is unreachable", async () => {
const server = Bun.serve({ port: 0, hostname: "127.0.0.1", fetch: () => new Response("REACHED") })
try {
sh("readonly", "echo forged > src/a.ts; true")
expect(await text(path.join(root, "src/a.ts"))).toBe("export const a = 1\n")
const probe = `python3 -c "import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:${server.port}/').read())"`
expect(await shAsync("readonly", probe, true)).not.toContain("REACHED")
expect(await shAsync("readonly", probe, false)).toContain("REACHED")
} finally {
server.stop(true)
}
})
})
describe("sandboxArgv", () => {
const probe = { exists: () => true, overlay: false, uid: 1000 }
const argv = (kind: SandboxKind, offline = false) =>
sandboxArgv({ kind, root: "/p", cwd: "/p", stateDir: "/h/.local/state/es", home: "/h", offline }, ["true"], probe)
test("every kind masks the state dir and the server password; only non-user kinds mask credentials", () => {
for (const kind of ["user", "seat", "programmer", "readonly", "gate"] as const) {
const args = argv(kind)
expect([kind, args.join(" ")]).toEqual([kind, expect.stringContaining("--tmpfs /h/.local/state/es")])
expect(args.join(" ")).toContain("--ro-bind /dev/null /h/.config/opencode/service.json")
expect(args.join(" ")).toContain("--unsetenv OPENCODE_SERVER_PASSWORD")
expect([kind, args.includes("SSH_AUTH_SOCK")]).toEqual([kind, kind !== "user"])
}
})
test("offline seats lose the network; the user's agent keeps background processes", () => {
expect(argv("readonly", true)).toContain("--unshare-net")
expect(argv("readonly")).not.toContain("--unshare-net")
expect(argv("user")).not.toContain("--die-with-parent")
expect(argv("seat")).toContain("--die-with-parent")
})
})
packages/opencode/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@heretek-ai/epistemic-swarm",
"version": "1.1.0",
"version": "1.1.1",
"description": "Epistemic Swarm: an AI build factory for OpenCode v2 \u2014 grill, research, spec, build, QA and release, gated mechanically.",
"type": "module",
"license": "Apache-2.0",
packages/opencode/src/agents.ts
+3
View File
@@ -23,6 +23,9 @@ export function permissionRules(spec: AgentSpec, mcpServers: readonly string[]):
...spec.spawns.map((agent) => ({ action: "subagent", resource: agent, effect: "allow" as const })),
{ action: "skill", resource: "*", effect: "deny" },
...spec.skills.map((skill) => ({ action: "skill", resource: skill, effect: "allow" as const })),
// Code Mode (`execute`) reaches host tools no seat needs, including MCP
// resources outside the seat's MCP scope.
{ action: "execute", resource: "*", effect: "deny" },
]
for (const server of mcpServers)
if (!spec.mcp.includes(server))
packages/opencode/src/policy.ts
+37 -32
View File
@@ -1,7 +1,8 @@
// Mechanical enforcement in the host: every write, edit, patch and shell call
// from any agent passes through core's policy (canonical paths, control files,
// seat scopes); factory seats additionally stop on STOP or a halted run, run
// read-only shells under bubblewrap, and get fast gate feedback after edits.
// seat scopes), and every shell command runs inside the bubblewrap sandbox its
// decision names (trust/sandbox.ts). Factory seats additionally stop on STOP
// or a halted run and get fast gate feedback after edits.
import path from "node:path"
import {
agentSpec,
@@ -13,11 +14,10 @@ import {
formatDiagnostics,
formatReport,
loadGatesConfig,
readonlyWrap,
runGates,
sandboxArgv,
seatOf,
shellQuote,
treeFingerprint,
} from "@heretek-ai/es-core"
import { Error as ToolError } from "@opencode/plugin/promise/tool"
import { HOST_WEB_TOOLS } from "./agents.ts"
@@ -50,8 +50,6 @@ const appendContent = (content: unknown, text: string) =>
: text
export function createPolicyHooks(runtime: Runtime) {
const fingerprints = new Map<string, { dir: string; print: string }>()
const before = async (event: { tool: string; agent: string; id: string; input: unknown }) => {
const seat = seatOf(event.agent)
const input = (event.input ?? {}) as Record<string, any>
@@ -88,14 +86,37 @@ export function createPolicyHooks(runtime: Runtime) {
if (event.tool === "shell" && typeof input.command === "string") {
const decision = evaluateShell(context, event.agent, input.command, { sandboxAvailable: bwrapAvailable() })
if (decision.effect === "deny") deny(decision.reason)
const cwd =
typeof input.workdir === "string"
? path.resolve(runtime.root, input.workdir)
: (context.worktree ?? runtime.root)
if (decision.effect === "allow" && decision.mode === "readonly-sandbox") {
event.input = { ...input, command: readonlyWrap(["sh", "-c", input.command], cwd).map(shellQuote).join(" ") }
} else if (decision.effect === "allow" && decision.mode === "readonly-checked") {
fingerprints.set(event.id, { dir: cwd, print: await treeFingerprint(cwd) })
}
}
/**
* The last execute.before hook: re-evaluates the final shell command (a
* PreToolUse hook may have rewritten it) and wraps it in its sandbox, so no
* later rewrite can run unsandboxed and hook matchers see the plain command.
*/
const sandbox = async (event: { tool: string; agent: string; id: string; input: unknown }) => {
const input = (event.input ?? {}) as Record<string, any>
if (event.tool === "shell" && typeof input.command === "string") {
const context = await runtime.policy()
const decision = evaluateShell(context, event.agent, input.command, { sandboxAvailable: bwrapAvailable() })
if (decision.effect === "deny") deny(decision.reason)
if (decision.effect === "allow" && decision.mode === "sandbox") {
const cwd =
typeof input.workdir === "string"
? path.resolve(runtime.root, input.workdir)
: (context.worktree ?? runtime.root)
const argv = sandboxArgv(
{
kind: decision.kind,
root: runtime.root,
cwd,
stateDir: runtime.stateDir,
offline: decision.offline,
...(decision.kind === "programmer" && context.worktree ? { writable: context.worktree } : {}),
},
["sh", "-c", input.command],
)
event.input = { ...input, command: argv.map(shellQuote).join(" ") }
}
}
}
@@ -109,23 +130,6 @@ export function createPolicyHooks(runtime: Runtime) {
result?: any
}) => {
const seat = seatOf(event.agent)
const checked = fingerprints.get(event.id)
if (checked) {
fingerprints.delete(event.id)
if ((await treeFingerprint(checked.dir)) !== checked.print) {
await runtime.factory
.halt(`agent:${event.agent}`, `read-only seat ${event.agent} changed files via the shell`)
.catch(() => undefined)
if (event.status === "completed")
event.result = {
...event.result,
content: appendContent(
event.result?.content,
"VIOLATION: this read-only seat modified the checkout. The factory has been halted.",
),
}
}
}
if (event.status !== "completed") return
const input = (event.input ?? {}) as Record<string, any>
const files =
@@ -165,6 +169,7 @@ export function createPolicyHooks(runtime: Runtime) {
commands: config.commands.filter((check) => check.kind === "format" || check.kind === "lint"),
},
stateDir: runtime.stateDir,
sandbox: "required",
})
if (report.findings.length || !report.passed)
event.result = {
@@ -225,5 +230,5 @@ export function createPolicyHooks(runtime: Runtime) {
for (const key of Object.keys(event.env)) if (/^OPENCODE_(SERVER_)?PASSWORD$/.test(key)) delete event.env[key]
}
return { before, after, evaluate, shellEnv }
return { before, sandbox, after, evaluate, shellEnv }
}
packages/opencode/src/research.ts
+101 -39
View File
@@ -1,65 +1,127 @@
// Research integration: the configured search provider becomes the host's
// websearch (results cached content-addressed, so they can be cited), and the
// host's own webfetch results are cached too.
import { researchCache, selectProvider } from "@heretek-ai/es-core"
// websearch, and host web results are cached content-addressed so they can be
// cited. Caching is for Epistemic Swarm seats in a project that already has a
// .factory/ directory: the user's own agents get plain results, and no
// project gains a .factory/ because someone searched the web.
import { stat } from "node:fs/promises"
import { factoryLayout, researchCache, seatOf, selectProvider } from "@heretek-ai/es-core"
import type { Runtime } from "./runtime.ts"
interface SearchHit {
readonly url: string
readonly title?: string
readonly text: string
}
/** Results the provider returned, waiting for the tool hook (which knows the agent) to cache them. */
export class PendingSearches {
private readonly entries: Array<{ query: string; provider: string; hits: readonly SearchHit[]; at: number }> = []
private static readonly TTL_MS = 5 * 60_000
add(query: string, provider: string, hits: readonly SearchHit[]) {
this.prune()
this.entries.push({ query, provider, hits, at: Date.now() })
}
/** The oldest unclaimed results for this query (each search is claimed once). */
take(query: string) {
this.prune()
const index = this.entries.findIndex((entry) => entry.query === query)
return index < 0 ? undefined : this.entries.splice(index, 1)[0]
}
private prune() {
const cutoff = Date.now() - PendingSearches.TTL_MS
while (this.entries.length && this.entries[0]!.at < cutoff) this.entries.shift()
}
}
/** Host web results are cached only for an ES seat, in a project that already has .factory/. */
async function cacheable(runtime: Runtime, agent: string | undefined): Promise<boolean> {
if (!seatOf(agent)) return false
return stat(factoryLayout(runtime.root).dir).then(
(info) => info.isDirectory(),
() => false,
)
}
export function registerWebsearch(
editor: { add(definition: any): void; default: { get(): unknown; set(id: string): void } },
runtime: Runtime,
pending: PendingSearches,
) {
const searxng = runtime.config.research?.searxngUrl
const env = searxng ? { ...process.env, SEARXNG_URL: searxng } : process.env
const provider = selectProvider(runtime.options.searchProvider, env)
if (!provider) return
const cache = researchCache(runtime.root)
editor.add({
id: "epistemic-swarm",
name: `Epistemic Swarm (${provider.id}, cached)`,
name: `Epistemic Swarm (${provider.id})`,
execute: async (input: { query: string; limit?: number }, context: { signal: AbortSignal }) => {
const results = await provider.search(input.query, { limit: input.limit ?? 8, signal: context.signal, env })
const out = []
for (const result of results) {
const cached = await cache.put({
url: result.url,
title: result.title,
text: result.content ?? result.snippet,
provider: provider.id,
query: input.query,
})
out.push({
url: result.url,
title: result.title,
content: `[cached sha256:${cached.meta.sha256}]\n${result.content ?? result.snippet}`,
time: {},
})
}
return out
const hits = results.map((result) => ({
url: result.url,
title: result.title,
text: result.content ?? result.snippet,
}))
pending.add(input.query, provider.id, hits)
return hits.map((hit) => ({ url: hit.url, title: hit.title, content: hit.text, time: {} }))
},
})
// Additive: become the default only when nothing else is selected.
if (editor.default.get() === undefined) editor.default.set("epistemic-swarm")
}
/** Cache host webfetch results so they can be cited as [VERIFIED: sha256:… "quote"]. */
export function createWebfetchCache(runtime: Runtime) {
const cache = researchCache(runtime.root)
return async (event: { tool: string; status: string; input: unknown; result?: any }) => {
const contentText = (content: unknown): string =>
typeof content === "string"
? content
: Array.isArray(content)
? content.map((part: any) => part?.text ?? "").join("\n")
: ""
const prepend = (result: any, note: string) => {
const content = result?.content
return {
...result,
content: typeof content === "string" ? `${note}\n${content}` : [{ type: "text", text: note }, ...(content ?? [])],
}
}
/**
* Cache a seat's host webfetch and websearch results so they can be cited as
* [VERIFIED: sha256:… "quote"]. Other agents' results pass through untouched.
*/
export function createWebCache(runtime: Runtime, pending: PendingSearches) {
const cache = researchCache(runtime.root, runtime.stateDir)
return async (event: { tool: string; status: string; agent?: string; input: unknown; result?: any }) => {
if (event.tool === "websearch") {
const query = (event.input as { query?: string } | undefined)?.query
const search = typeof query === "string" ? pending.take(query) : undefined
if (!search || event.status !== "completed" || !(await cacheable(runtime, event.agent))) return
const lines: string[] = []
for (const hit of search.hits) {
if (!hit.text?.trim()) continue
const cached = await cache.put({
url: hit.url,
title: hit.title,
text: hit.text,
provider: search.provider,
query: search.query,
})
lines.push(`[cached sha256:${cached.meta.sha256}] ${hit.url}`)
}
if (lines.length)
event.result = prepend(event.result, `${lines.join("\n")}\n(cite with [VERIFIED: sha256:<hash> "exact words"])`)
return
}
if (event.tool !== "webfetch" || event.status !== "completed") return
const url = (event.input as { url?: string } | undefined)?.url
const content = event.result?.content
const text =
typeof content === "string"
? content
: Array.isArray(content)
? content.map((part: any) => part?.text ?? "").join("\n")
: ""
if (!url || text.trim().length < 40) return
const text = contentText(event.result?.content)
if (!url || text.trim().length < 40 || !(await cacheable(runtime, event.agent))) return
const cached = await cache.put({ url, text, provider: "webfetch" })
const note = `[cached as sha256:${cached.meta.sha256}; cite with [VERIFIED: sha256:${cached.meta.sha256} "exact words"]]`
event.result = {
...event.result,
content: typeof content === "string" ? `${note}\n${content}` : [{ type: "text", text: note }, ...(content ?? [])],
}
event.result = prepend(
event.result,
`[cached as sha256:${cached.meta.sha256}; cite with [VERIFIED: sha256:${cached.meta.sha256} "exact words"]]`,
)
}
}
packages/opencode/src/rpc-def.ts
+9 -31
View File
@@ -1,6 +1,7 @@
// RPC contract between the server plugin and the TUI. These methods are the
// TUI half of the human-only channel: no agent tool reaches them, and the TUI
// only calls them after a human confirms a dialog showing what is approved.
// RPC contract between the server plugin and the TUI. Previews are shown in
// the TUI; human-only mutations (approve, trust, resume) happen at a terminal
// with the passphrase (`es approve`, `es trust`, `es factory resume`), never
// through an agent tool or RPC. Only lspInstall still mutates via RPC.
import { Rpc } from "@opencode/plugin/rpc"
const obj = (properties: Record<string, unknown>, required: string[] = []) => ({
@@ -26,25 +27,16 @@ export const EsRpc = Rpc.define({
input: obj({}),
output: obj({ summary: str, stage: str, pending: strArray }, ["summary", "stage", "pending"]),
},
/** The effective config (JSON), the files it came from, and load warnings to show once. */
configState: {
input: obj({}),
output: obj({ config: str, sources: strArray, warnings: strArray }, ["config", "sources", "warnings"]),
},
previewApproval: {
input: obj({ stage: { type: "string", enum: ["frontier", "spec"] } }, ["stage"]),
output: Preview,
},
approve: {
input: obj({ stage: { type: "string", enum: ["frontier", "spec"] }, user: str, token: str }, [
"stage",
"user",
"token",
]),
output: obj({ message: str }, ["message"]),
errors: { refused: obj({ reason: str }, ["reason"]) },
},
previewTrust: { input: obj({}), output: Preview },
trust: {
input: obj({ user: str, token: str }, ["user", "token"]),
output: obj({ message: str }, ["message"]),
errors: { refused: obj({ reason: str }, ["reason"]) },
},
previewResume: { input: obj({}), output: Preview },
previewLspInstall: { input: obj({ id: str }, ["id"]), output: Preview },
lspInstall: {
@@ -52,20 +44,6 @@ export const EsRpc = Rpc.define({
output: obj({ message: str }, ["message"]),
errors: { refused: obj({ reason: str }, ["reason"]) },
},
resume: {
input: obj(
{
user: str,
token: str,
acceptControlDrift: { type: "boolean" },
raiseCeilingUSD: { type: "number" },
extendRuntime: { type: "boolean" },
},
["user", "token"],
),
output: obj({ message: str }, ["message"]),
errors: { refused: obj({ reason: str }, ["reason"]) },
},
factoryState: {
input: obj({}),
output: obj(
packages/opencode/src/rpc.ts
+10 -76
View File
@@ -1,7 +1,7 @@
// Server side of the human-only channel. Each mutating method needs a token
// from a preview made in the last two minutes over the same subject, so the
// human confirms exactly what the dialog showed (no swap between preview and
// confirm).
// Server side of the TUI previews. Human-only mutations (approve, trust,
// resume) happen at a terminal with the passphrase-sealed key, never via RPC:
// the TUI previews, then points at `es approve`, `es trust`, `es factory
// resume`. Only lspInstall still mutates via a preview token.
import { randomBytes } from "node:crypto"
import {
type ApprovalStage,
@@ -10,7 +10,6 @@ import {
capabilityLoss,
commandSetHash,
describeTarget,
factoryLayout,
factorySummary,
type HookEngine,
hashJson,
@@ -22,12 +21,9 @@ import {
readPlan,
readResult,
readScores,
recordApproval,
recordConsent,
treeCounts,
trustProject,
verifyControl,
writeJson,
} from "@heretek-ai/es-core"
import type { Runtime } from "./runtime.ts"
import { pendingApprovals } from "./tools.ts"
@@ -40,7 +36,7 @@ interface Ticket {
const TTL_MS = 120_000
export function createRpcHandlers(runtime: Runtime, notify: () => Promise<void>, hooks?: HookEngine) {
export function createRpcHandlers(runtime: Runtime, _notify: () => Promise<void>, hooks?: HookEngine) {
const tickets = new Map<string, Ticket>()
const issue = (kind: string, subject: unknown) => {
const token = randomBytes(12).toString("hex")
@@ -84,6 +80,11 @@ export function createRpcHandlers(runtime: Runtime, notify: () => Promise<void>,
pending: (await pendingApprovals(runtime.root)).map((item) => item.stage),
}
},
configState: async () => ({
config: JSON.stringify(runtime.config),
sources: [...runtime.configSources],
warnings: [...runtime.warnings],
}),
factoryState: async () => {
const state = await runtime.factory.read()
const frontier = await readFrontier(runtime.root).catch(() => undefined)
@@ -204,31 +205,6 @@ export function createRpcHandlers(runtime: Runtime, notify: () => Promise<void>,
}
}
},
approve: async (input: unknown, context: any) => {
const { stage, user, token } = input as { stage: ApprovalStage; user: string; token: string }
try {
redeem(token, `approve:${stage}`, (await approvalSubject(runtime.root, stage)).subject)
const record = await recordApproval(runtime.root, {
stage,
channel: "tui",
approvedBy: user,
stateDir: runtime.stateDir,
})
await writeJson(
factoryLayout(runtime.root).pending,
(await pendingApprovals(runtime.root)).filter((item) => item.stage !== stage),
)
if (stage === "frontier") {
const state = await runtime.factory.read()
if (!state) await runtime.factory.begin(`human:${user}`)
if ((await runtime.factory.read())?.stage === "GRILL") await runtime.factory.beginResearch(`human:${user}`)
}
await notify()
return { message: `Approved ${stage} as ${record.approvedBy}.` }
} catch (error) {
return refused(context, error)
}
},
previewTrust: async () => {
const subject = await trustSubject()
const trusted =
@@ -242,27 +218,6 @@ export function createRpcHandlers(runtime: Runtime, notify: () => Promise<void>,
token: issue("trust", subject.key),
}
},
trust: async (input: unknown, context: any) => {
const { user, token } = input as { user: string; token: string }
try {
const subject = await trustSubject()
redeem(token, "trust", subject.key)
await trustProject(runtime.root, subject.gates.hash, subject.gates.lines, {
approvedBy: user,
stateDir: runtime.stateDir,
})
if (subject.hooks.hash)
await trustProject(runtime.root, subject.hooks.hash, subject.hooks.lines, {
approvedBy: user,
stateDir: runtime.stateDir,
kind: "hooks",
})
await hooks?.refresh()
return { message: `Trusted ${subject.lines.length} command/hook line(s).` }
} catch (error) {
return refused(context, error)
}
},
previewLspInstall: async (input: unknown) => {
const { id } = input as { id: string }
const server = (await runtime.lsp.settings()).servers.find((item) => item.id === id)
@@ -321,26 +276,5 @@ export function createRpcHandlers(runtime: Runtime, notify: () => Promise<void>,
token: issue("resume", state.halt),
}
},
resume: async (input: unknown, context: any) => {
const { user, token, acceptControlDrift, raiseCeilingUSD, extendRuntime } = input as {
user: string
token: string
acceptControlDrift?: boolean
raiseCeilingUSD?: number
extendRuntime?: boolean
}
try {
redeem(token, "resume", (await runtime.factory.read())?.halt)
const state = await runtime.factory.resume(user, {
...(acceptControlDrift ? { acceptControlDrift } : {}),
...(raiseCeilingUSD ? { raiseCeilingUSD } : {}),
...(extendRuntime ? { extendRuntime } : {}),
})
await notify()
return { message: `Resumed at ${state.stage}.` }
} catch (error) {
return refused(context, error)
}
},
}
}
packages/opencode/src/runtime.ts
+41 -37
View File
@@ -1,9 +1,12 @@
// Per-location runtime shared by the plugin's hooks, tools, commands and RPC.
// Options are layered: the canonical config (global file → project file) with
// plugin options as the most specific override.
// plugin options as the most specific override. Plugin options are config
// keys validated by the same strict schema; unknown keys are ignored with a
// warning the TUI shows once.
import {
stateDir as defaultStateDir,
type EsConfig,
EsConfigSchema,
Factory,
gateRunner,
LspManager,
@@ -12,22 +15,14 @@ import {
} from "@heretek-ai/es-core"
import { ghPrOpener } from "./pr.ts"
/** Raw plugin options from opencode.json; every field is optional. */
export interface RawPluginOptions {
/** Plugin options from opencode.json, split into config keys and the rest. */
export interface ParsedOptions {
/** Override the user-global state dir (signing key, trust store). Tests only. */
readonly stateDir?: string
/** Model per tier, "provider/model". */
readonly models?: {
readonly fast?: string
readonly balanced?: string
readonly deep?: string
readonly agents?: Readonly<Record<string, string>>
}
readonly afterEdit?: "fast" | "off"
readonly estimate?: { readonly inputPerM?: number; readonly outputPerM?: number }
readonly pr?: "gh" | "off"
readonly lspAfterEdit?: boolean
readonly searchProvider?: "brave" | "firecrawl" | "searxng"
/** Config keys (docs/CONFIG.md), layered over the config files and validated with them. */
readonly overrides: Readonly<Record<string, unknown>>
/** Keys that are not config keys, sorted; ignored with a warning. */
readonly unknown: readonly string[]
}
/** Resolved options: config defaults applied; consumers read this shape. */
@@ -47,16 +42,18 @@ export interface Runtime {
readonly config: EsConfig
/** Config files that contributed values, in layer order. */
readonly configSources: readonly string[]
/** Human-facing load warnings (e.g. ignored plugin options), shown once by the TUI. */
readonly warnings: readonly string[]
readonly stateDir: string
readonly factory: Factory
readonly lsp: LspManager
policy(): Promise<PolicyContext>
}
export async function createRuntime(root: string, raw: RawPluginOptions): Promise<Runtime> {
export async function createRuntime(root: string, parsed: ParsedOptions): Promise<Runtime> {
// stateDir is runtime-only; everything else can come from config files.
const { stateDir: rawStateDir, ...overrides } = raw
const loaded = await loadEsConfig(root, { overrides })
const rawStateDir = parsed.stateDir
const loaded = await loadEsConfig(root, { overrides: { ...parsed.overrides } })
const config = loaded.config
const options: PluginOptions = {
...(rawStateDir ? { stateDir: rawStateDir } : {}),
@@ -81,6 +78,7 @@ export async function createRuntime(root: string, raw: RawPluginOptions): Promis
options,
config,
configSources: loaded.sources,
warnings: parsed.unknown.length ? [unknownOptionsWarning(parsed.unknown)] : [],
stateDir,
factory,
lsp,
@@ -91,24 +89,30 @@ export async function createRuntime(root: string, raw: RawPluginOptions): Promis
}
}
/** Keep only the keys the user actually set, so config files are not clobbered. */
export function parseOptions(raw: Record<string, unknown> | undefined): RawPluginOptions {
const options = raw ?? {}
const models =
typeof options.models === "object" && options.models ? (options.models as RawPluginOptions["models"]) : undefined
const estimate =
typeof options.estimate === "object" && options.estimate
? (options.estimate as RawPluginOptions["estimate"])
: undefined
return {
...(typeof options.stateDir === "string" ? { stateDir: options.stateDir } : {}),
...(models ? { models } : {}),
...(options.afterEdit === "fast" || options.afterEdit === "off" ? { afterEdit: options.afterEdit } : {}),
...(estimate ? { estimate } : {}),
...(options.pr === "gh" || options.pr === "off" ? { pr: options.pr } : {}),
...(typeof options.lspAfterEdit === "boolean" ? { lspAfterEdit: options.lspAfterEdit } : {}),
...(typeof options.searchProvider === "string"
? { searchProvider: options.searchProvider as RawPluginOptions["searchProvider"] }
: {}),
const CONFIG_KEYS: ReadonlySet<string> = new Set(Object.keys(EsConfigSchema.shape))
/**
* Split raw plugin options into config keys and the rest. Only keys the user
* set are passed on, so config files are not clobbered; their values are
* validated by the strict config schema (an invalid value fails the load, as
* it would in a config file).
*/
export function parseOptions(raw: Record<string, unknown> | undefined): ParsedOptions {
const overrides: Record<string, unknown> = {}
const unknown: string[] = []
let stateDir: string | undefined
for (const [key, value] of Object.entries(raw ?? {})) {
if (key === "stateDir" && typeof value === "string") stateDir = value
else if (CONFIG_KEYS.has(key)) overrides[key] = value
else unknown.push(key)
}
return { ...(stateDir ? { stateDir } : {}), overrides, unknown: unknown.sort() }
}
export function unknownOptionsWarning(keys: readonly string[]): string {
return (
`Ignored unknown plugin option${keys.length === 1 ? "" : "s"}: ${keys.join(", ")}. ` +
"Plugin options take the config keys in docs/CONFIG.md (es config show); " +
"0.7-era options such as search_engine, max_iterations and mode were removed in 1.0."
)
}
packages/opencode/src/server.ts
+12 -3
View File
@@ -5,6 +5,7 @@ import { readdir, readFile } from "node:fs/promises"
import path from "node:path"
import {
describeTarget,
ensureEngineKey,
factoryLayout,
formatReport,
git,
@@ -19,7 +20,7 @@ import { compileAgents } from "./agents.ts"
import { createFactoryContinuation } from "./continue.ts"
import { createHookBridge } from "./hooks.ts"
import { createPolicyHooks } from "./policy.ts"
import { createWebfetchCache, registerWebsearch } from "./research.ts"
import { createWebCache, PendingSearches, registerWebsearch } from "./research.ts"
import { createRpcHandlers } from "./rpc.ts"
import { EsRpc } from "./rpc-def.ts"
import { createRuntime, parseOptions } from "./runtime.ts"
@@ -33,6 +34,9 @@ export default Plugin.define({
async setup(ctx) {
const options = parseOptions(ctx.options as Record<string, unknown>)
const runtime = await createRuntime(ctx.location.directory, options)
// The engine key seals cached sources (#52); create it at setup so the
// masked state dir holds the key before any seat runs.
await ensureEngineKey(runtime.stateDir)
// Agents: canonical registry → v2 agents with wildcard-deny scoping.
const servers = await ctx.mcp.list().then(
@@ -66,7 +70,9 @@ export default Plugin.define({
})
await ctx.tool.transform((editor) => registerTools(editor, runtime))
await ctx.websearch.transform((editor) => registerWebsearch(editor as any, runtime))
// Search results wait here until the tool hook, which knows the agent, decides whether to cache them.
const searches = new PendingSearches()
await ctx.websearch.transform((editor) => registerWebsearch(editor as any, runtime, searches))
// Our policy runs first, so its denials win over any user hook.
const policy = createPolicyHooks(runtime)
@@ -82,8 +88,10 @@ export default Plugin.define({
}
await ctx.tool.hook("execute.before", policy.before as any)
await ctx.tool.hook("execute.before", bridge.before as any)
// Last: the final (possibly hook-rewritten) shell command is re-checked and sandboxed.
await ctx.tool.hook("execute.before", policy.sandbox as any)
await ctx.tool.hook("execute.after", policy.after as any)
await ctx.tool.hook("execute.after", createWebfetchCache(runtime) as any)
await ctx.tool.hook("execute.after", createWebCache(runtime, searches) as any)
await ctx.tool.hook("execute.after", bridge.after as any)
await ctx.permission.hook("evaluate", policy.evaluate as any)
await ctx.permission.hook("evaluate", bridge.evaluate as any)
@@ -312,6 +320,7 @@ export default Plugin.define({
runtime.configSources.length
? `Layers: ${runtime.configSources.join(" → ")} → plugin options`
: "Layers: defaults only (no config files)",
...runtime.warnings.map((warning) => `Warning: ${warning}`),
JSON.stringify(runtime.config, null, 2),
]
await ctx.session.synthetic({ sessionID, text: lines.join("\n") } as any)
packages/opencode/src/tools.ts
+1
View File
@@ -21,6 +21,7 @@ export function registerTools(editor: { add(tool: any): void }, runtime: Runtime
const research = researchTools({
root: runtime.root,
policy: () => runtime.policy(),
stateDir: runtime.stateDir,
...researchOptions(runtime.config),
})
const brainstorm = brainstormTools({
packages/opencode/src/tui.tsx
+32 -18
View File
@@ -1,7 +1,7 @@
// Epistemic Swarm TUI plugin: the human-only dialogs. Approvals, trust and
// resume happen here (or in the `es` CLI), never through an agent tool. Each
// dialog shows a server-computed preview; confirming redeems that preview's
// token, so what the human saw is exactly what gets signed.
// Epistemic Swarm TUI plugin: previews for the human-only actions. Approvals,
// trust and resume are previewed here, then recorded at a terminal with the
// passphrase-sealed human key (`es approve`, `es trust`, `es factory resume`),
// never through an agent tool or RPC. Only lspInstall still mutates via RPC.
import { userInfo } from "node:os"
import { Plugin } from "@opencode/plugin/tui"
import { registerPanels } from "./panels.tsx"
@@ -19,6 +19,12 @@ export default Plugin.define({
const user = userInfo().username
const toast = (message: string, variant: "success" | "error" | "info" = "info") =>
context.ui.toast.show({ title: "Epistemic Swarm", message, variant })
// Load warnings (e.g. ignored plugin options) are shown once per TUI start.
void call("configState")
.then((view: { warnings: string[] }) => {
for (const warning of view.warnings) toast(warning, "error")
})
.catch(() => undefined)
const confirm = async (preview: Preview, label: string) => {
if (!preview.ok || !preview.token) {
@@ -58,36 +64,44 @@ export default Plugin.define({
],
}))
if (stage !== "frontier" && stage !== "spec") return
const token = await confirm(await call("previewApproval", { stage }), "Approve")
const preview: Preview = await call("previewApproval", { stage })
const token = await confirm(preview, "Approve")
if (!token) return
toast((await call("approve", { stage, user, token })).message, "success")
await context.ui.dialog.alert({
title: preview.title,
message: `${preview.lines.join("\n")}\n\nRun \`es approve ${stage}\` at a terminal with your passphrase (1.1.1: approvals need the sealed human key).`,
})
})
const trust = guarded(async () => {
const token = await confirm(await call("previewTrust"), "Trust")
const preview: Preview = await call("previewTrust")
const token = await confirm(preview, "Trust")
if (!token) return
toast((await call("trust", { user, token })).message, "success")
await context.ui.dialog.alert({
title: preview.title,
message: `${preview.lines.join("\n")}\n\nRun \`es trust\` at a terminal with your passphrase to sign these.`,
})
})
const resume = guarded(async () => {
const preview: Preview = await call("previewResume")
const token = await confirm(preview, "Resume")
if (!token) return
const lines = preview.lines.join("\n")
const drift = preview.lines.some((line) => line.startsWith("Control-file drift"))
let raiseCeilingUSD: number | undefined
// The halt reason names the remedy: a ceiling halt takes a new ceiling,
// a runtime halt restarts the cap. The values still come from the human.
let raise = ""
if (/spend ceiling/i.test(preview.lines[0] ?? "")) {
const raised = await context.ui.dialog.prompt({ title: "New spend ceiling (USD)", placeholder: "e.g. 50" })
raiseCeilingUSD = raised ? Number(raised) : undefined
if (raised) raise = ` --raise-ceiling ${raised}`
}
const extendRuntime = /runtime cap/i.test(preview.lines[0] ?? "")
const result = await call("resume", {
user,
token,
...(drift ? { acceptControlDrift: true } : {}),
...(raiseCeilingUSD ? { raiseCeilingUSD } : {}),
...(extendRuntime ? { extendRuntime: true } : {}),
const extend = /runtime cap/i.test(lines) ? " --extend-runtime" : ""
const flags = `${drift ? " --accept-drift" : ""}${raise}${extend}`
await context.ui.dialog.alert({
title: preview.title,
message: `${preview.lines.join("\n")}\n\nRun \`es factory resume${flags}\` at a terminal with your passphrase.`,
})
toast(result.message, "success")
})
const lspInstall = guarded(async (input) => {
packages/opencode/test/evals.test.ts
+71 -3
View File
@@ -6,7 +6,7 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test"
import { accessSync, constants } from "node:fs"
import path from "node:path"
import { evalWorkspace, runEvalCase } from "@heretek-ai/es-core"
import { type FakeLLM, startFakeLLM } from "@heretek-ai/es-testkit"
import { directiveScript, type FakeLLM, startFakeLLM } from "@heretek-ai/es-testkit"
const binary = (process.env.PATH ?? "")
.split(path.delimiter)
@@ -27,7 +27,8 @@ beforeAll(() => {
})
afterAll(() => llm?.stop())
const workspace = () =>
// `cost` prices the fake model in USD per million tokens, so its turns report spend.
const workspace = (cost?: { input: number; output: number }) =>
evalWorkspace({
pluginDir,
model: "fake/scripted",
@@ -35,7 +36,7 @@ const workspace = () =>
fake: {
package: "@opencode/ai/providers/openai-compatible",
settings: { baseURL: llm.url, apiKey: "test" },
models: { scripted: { limit: { context: 100000, output: 4000 } } },
models: { scripted: { limit: { context: 100000, output: 4000 }, ...(cost ? { cost } : {}) } },
},
},
})
@@ -82,4 +83,71 @@ describe.skipIf(!binary)("the eval runner on the real opencode CLI", () => {
await ws.cleanup()
}
}, 90_000)
test("parallel calls in one model turn are one step", async () => {
llm.setScript(directiveScript)
const ws = await workspace()
try {
const result = await runEvalCase(
{
id: "parallel",
description: "two calls, one turn",
agent: "factory",
// Empty args: quotes in a CLI prompt reach the model escaped.
prompt: "@@PARALLEL@@ @@CALL es_status {}@@ @@CALL es_status {}@@",
checks: { toolsUsed: ["es_status"] },
},
ws,
{ binary: binary!, maxSteps: 2, timeoutMs: 60_000 },
)
expect(result.failures).toEqual([])
expect(result.transcript.tools).toEqual(["es_status", "es_status"])
expect(result.transcript.steps).toBe(2)
} finally {
await ws.cleanup()
}
}, 90_000)
test("a refused tool call does not satisfy toolsUsed", async () => {
llm.setScript(directiveScript)
const ws = await workspace()
try {
// No run exists, so the factory's build start is refused.
const result = await runEvalCase(
{
id: "refused",
description: "the call errors",
agent: "factory",
prompt: "@@CALL es_build_start {}@@",
checks: { toolsUsed: ["es_build_start"] },
},
ws,
{ binary: binary!, maxSteps: 4, timeoutMs: 60_000 },
)
expect(result.transcript.tools).toEqual(["es_build_start"])
expect(result.transcript.completed).toEqual([])
expect(result.failures).toContain("called but did not complete: es_build_start")
} finally {
await ws.cleanup()
}
}, 90_000)
test("a priced model's spend is read from step_finish and the budget kills the run", async () => {
llm.setScript(() => ({ toolCalls: [{ name: "es_status", args: {} }], usage: { input: 10, output: 0 } }))
// $100k per million input tokens: 10 input tokens cost $1 per step.
const ws = await workspace({ input: 100_000, output: 0 })
try {
const result = await runEvalCase(
{ id: "spend", description: "keeps spending", agent: "factory", prompt: "Status?" },
ws,
{ binary: binary!, maxSteps: 10, timeoutMs: 60_000, budgetUSD: 1.5 },
)
expect(result.overBudget).toBe(true)
expect(result.capped).toBe(false)
expect(result.transcript.costUSD).toBeGreaterThan(1.5)
expect(result.failures[0]).toStartWith("spend budget reached")
} finally {
await ws.cleanup()
}
}, 90_000)
})
packages/opencode/test/fires-gates.test.ts
+20 -9
View File
@@ -20,12 +20,15 @@ import {
factoryLayout,
gateRunner,
gradeHarvestFire,
type HumanSigner,
readFrontier,
readHarvestResult,
recordApproval,
researchSourcesDir,
SourceCache,
sealHumanKey,
stringifyFrontmatter,
unlockHumanKey,
} from "@heretek-ai/es-core"
import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit"
@@ -40,6 +43,8 @@ const write = async (root: string, file: string, text: string) => {
async function bootHost(prefix: string) {
const state = await mkdtemp(path.join(tmpdir(), prefix))
await sealHumanKey("test-passphrase-1234", state)
const signer = await unlockHumanKey("test-passphrase-1234", state)
const h = await boot({
git: true,
script: directiveScript,
@@ -47,15 +52,16 @@ async function bootHost(prefix: string) {
files: { "README.md": "# fire suite\n" },
})
const factory = () => new Factory(h.directory, { gates: gateRunner({ stateDir: state }), stateDir: state })
return { state, h, factory }
return { state, h, factory, signer }
}
describe("grill-fires", () => {
let h: Harness
let state: string
let factory: () => Factory
let signer: HumanSigner
beforeAll(async () => {
;({ h, state, factory } = await bootHost("es-grill-fires-"))
;({ h, state, factory, signer } = await bootHost("es-grill-fires-"))
}, 60_000)
afterAll(async () => {
await h?.close()
@@ -149,9 +155,9 @@ describe("grill-fires", () => {
expect(fourth.tools[0]?.text).toContain("frozen")
// A granted frontier approval starts RESEARCH; the fact gates its completion.
await recordApproval(h.directory, { stage: "frontier", channel: "cli", approvedBy: "tester", stateDir: state })
await recordApproval(h.directory, { stage: "frontier", channel: "cli", approvedBy: "tester", signer })
await factory().beginResearch("human:tester")
const cache = new SourceCache(researchSourcesDir(h.directory))
const cache = new SourceCache(researchSourcesDir(h.directory), state)
const source = await cache.put({
url: "https://example.test/handoff",
text: "Session handoff writes a file the next agent reads.",
@@ -181,8 +187,9 @@ describe("factory-gate", () => {
let h: Harness
let state: string
let factory: () => Factory
let signer: HumanSigner
beforeAll(async () => {
;({ h, state, factory } = await bootHost("es-factory-gate-"))
;({ h, state, factory, signer } = await bootHost("es-factory-gate-"))
}, 60_000)
afterAll(async () => {
await h?.close()
@@ -207,9 +214,9 @@ describe("factory-gate", () => {
settled: true,
nodes: [{ id: "scope", question: "Which harness first?", status: "settled", answer: "opencode", round: 1 }],
})
await recordApproval(h.directory, { stage: "frontier", channel: "cli", approvedBy: "tester", stateDir: state })
await recordApproval(h.directory, { stage: "frontier", channel: "cli", approvedBy: "tester", signer })
await factory().beginResearch("human:tester")
const cache = new SourceCache(researchSourcesDir(h.directory))
const cache = new SourceCache(researchSourcesDir(h.directory), state)
const source = await cache.put({
url: "https://example.test/handoff",
text: "Session handoff writes a file the next agent reads.",
@@ -246,9 +253,13 @@ describe("factory-gate", () => {
expect(existsSync(path.join(h.directory, ".factory/worktrees"))).toBe(false)
// The human approves; the scripted factory seat can now build.
await recordApproval(h.directory, { stage: "spec", channel: "cli", approvedBy: "tester", stateDir: state })
await recordApproval(h.directory, { stage: "spec", channel: "cli", approvedBy: "tester", signer })
const started = await h.run(call("es_build_start"), { agent: "factory" })
expect(started.tools[0]?.status).toBe("completed")
// [status, text]: the text names the refusal reason.
expect([started.tools[0]?.status, started.tools[0]?.text]).toEqual([
"completed",
expect.stringContaining("worktree:"),
])
expect(existsSync(path.join(h.directory, ".factory/worktrees/alpha"))).toBe(true)
// Release is a RELEASE-stage action, not a build-stage one.
packages/opencode/test/fires.test.ts
+16
View File
@@ -105,6 +105,22 @@ describe("audit-fires", () => {
expect(hallucinated.tools[0]?.status).toBe("error")
expect(await readAuditRecords(h.directory, "audit-01")).toEqual([])
// Issue #38: single letters between ellipses used to "witness" anything.
const lettered = await h.run(
call("es_audit_verdict", {
audit: "audit-01",
verdict: "fail",
findings: [{ ...caught[0], excerpt: "e ... e ... e ... e ... e ... e ... e" }],
notes: "blind",
}),
{ agent: "es-auditor-antithesis" },
)
expect([lettered.tools[0]?.status, lettered.tools[0]?.text]).toEqual([
"error",
expect.stringContaining("not found verbatim"),
])
expect(await readAuditRecords(h.directory, "audit-01")).toEqual([])
const thesis = await h.run(
call("es_audit_verdict", {
audit: "audit-01",
packages/opencode/test/hooks.test.ts
+23 -1
View File
@@ -4,6 +4,15 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test"
import { mkdtemp, rm } from "node:fs/promises"
import { tmpdir } from "node:os"
import path from "node:path"
import {
commandSetHash,
HookEngine,
type HumanSigner,
loadGatesConfig,
sealHumanKey,
trustProject,
unlockHumanKey,
} from "@heretek-ai/es-core"
import { allText, boot, directiveScript, type Harness, lastAgentRequest, systemText } from "@heretek-ai/es-testkit"
import { EsRpc } from "../src/rpc-def.ts"
@@ -111,10 +120,13 @@ const extraHooks = {
let state: string
let h: Harness
let signer: HumanSigner
const where = () => ({ location: h.location })
beforeAll(async () => {
state = await mkdtemp(path.join(tmpdir(), "es-hookbridge-state-"))
await sealHumanKey("test-passphrase-1234", state)
signer = await unlockHumanKey("test-passphrase-1234", state)
h = await boot({
git: true,
script: (request, index) => {
@@ -147,7 +159,17 @@ describe("hook bridge on the real host", () => {
const rpc = (h.opencode as any).rpc(EsRpc)
const preview = await rpc.previewTrust({}, where())
expect(preview.lines.join("\n")).toContain("hook PreToolUse(Write)")
await rpc.trust({ user: "tester", token: preview.token }, where())
expect(typeof rpc.trust).not.toBe("function")
const { config } = await loadGatesConfig(h.directory)
const gates = await commandSetHash(h.directory, config.commands)
const hooks = (await HookEngine.create({ root: h.directory, stateDir: state, audit: false })).status()
await trustProject(h.directory, gates.hash, gates.lines, { stateDir: state, signer })
if (hooks.projectHash)
await trustProject(h.directory, hooks.projectHash, hooks.projectLines, {
stateDir: state,
kind: "hooks",
signer,
})
const after = await h.run(call("write", { path: "b.txt", content: "b" }))
expect(after.tools[0]?.status).toBe("completed")
expect(await Bun.file(path.join(h.directory, "PRE_RAN")).exists()).toBe(true)
packages/opencode/test/options.test.ts
+73
View File
@@ -0,0 +1,73 @@
// Plugin options go through the strict config schema: every config key works
// from opencode.json, and unknown keys (such as 0.7-era options) are ignored
// with one visible warning instead of vanishing silently.
import { afterAll, beforeAll, describe, expect, test } from "bun:test"
import { mkdtemp, rm } from "node:fs/promises"
import { tmpdir } from "node:os"
import path from "node:path"
import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit"
import { EsRpc } from "../src/rpc-def.ts"
import { parseOptions } from "../src/runtime.ts"
const pluginDir = path.resolve(import.meta.dir, "..")
describe("parseOptions", () => {
test("keeps every config key, separates stateDir, and reports unknown keys", () => {
const parsed = parseOptions({
stateDir: "/tmp/state",
pr: "off",
domainPack: "quant",
licenseWhitelist: ["MIT"],
research: { depth: 3 },
embeddings: { url: "http://localhost:1/v1/embeddings", model: "e" },
search_engine: "duckduckgo",
max_iterations: 2,
mode: "research",
})
expect(parsed.stateDir).toBe("/tmp/state")
expect(parsed.overrides).toEqual({
pr: "off",
domainPack: "quant",
licenseWhitelist: ["MIT"],
research: { depth: 3 },
embeddings: { url: "http://localhost:1/v1/embeddings", model: "e" },
})
expect(parsed.unknown).toEqual(["max_iterations", "mode", "search_engine"])
})
test("no options is an empty, warning-free parse", () => {
expect(parseOptions(undefined)).toEqual({ overrides: {}, unknown: [] })
})
})
describe("plugin options on the real host", () => {
let h: Harness
let state: string
beforeAll(async () => {
state = await mkdtemp(path.join(tmpdir(), "es-options-state-"))
h = await boot({
git: true,
script: directiveScript,
plugins: [
{
path: pluginDir,
options: { stateDir: state, pr: "off", domainPack: "quant", search_engine: "duckduckgo", max_iterations: 2 },
},
],
files: { "README.md": "# demo\n" },
})
}, 60_000)
afterAll(async () => {
await h?.close()
await rm(state, { recursive: true, force: true })
})
test("a config key set as a plugin option takes effect; unknown keys come back as one warning", async () => {
const rpc = (h.opencode as any).rpc(EsRpc)
const view = await rpc.configState({}, { location: h.location })
expect(JSON.parse(view.config).domainPack).toBe("quant")
expect(view.warnings).toHaveLength(1)
expect(view.warnings[0]).toContain("max_iterations, search_engine")
expect(view.warnings[0]).toContain("docs/CONFIG.md")
})
})
packages/opencode/test/plugin.test.ts
+111 -14
View File
@@ -4,7 +4,24 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test"
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"
import { tmpdir } from "node:os"
import path from "node:path"
import { researchSourcesDir, SourceCache } from "@heretek-ai/es-core"
import {
bwrapAvailable,
commandSetHash,
Factory,
factoryLayout,
gateRunner,
HookEngine,
type HumanSigner,
loadGatesConfig,
pendingApprovals,
recordApproval,
researchSourcesDir,
SourceCache,
sealHumanKey,
trustProject,
unlockHumanKey,
writeJson,
} from "@heretek-ai/es-core"
import { boot, directiveScript, type Harness, lastAgentRequest, systemText } from "@heretek-ai/es-testkit"
import { EsRpc } from "../src/rpc-def.ts"
@@ -12,8 +29,11 @@ const pluginDir = path.resolve(import.meta.dir, "..")
const call = (name: string, args: Record<string, unknown> = {}) => `@@CALL ${name} ${JSON.stringify(args)}@@`
let state: string
let signer: HumanSigner
beforeAll(async () => {
state = await mkdtemp(path.join(tmpdir(), "es-plugin-state-"))
await sealHumanKey("test-passphrase-1234", state)
signer = await unlockHumanKey("test-passphrase-1234", state)
})
afterAll(() => rm(state, { recursive: true, force: true }))
@@ -173,6 +193,16 @@ describe("integrity on the real host", () => {
expect(systemText(programmer)).not.toContain("<id>grill</id>")
})
test("seats are not offered Code Mode; the user's own agent still is (issue #50)", async () => {
const offered = async (agent: string) => {
await h.run("hello", { agent })
return (lastAgentRequest(h.llm.requests)?.tools ?? []).map((tool) => tool.function.name)
}
expect(await offered("build")).toContain("execute")
for (const agent of ["factory", "grill", "brainstormer", "harvester", "es-research-alpha", "es-programmer"])
expect([agent, (await offered(agent)).includes("execute")]).toEqual([agent, false])
})
test("the factory invokes a hidden seat by id; compaction keeps the factory state", async () => {
const { sessionID } = await h.run(
`delegate ${call("subagent", { agent: "es-manager", description: "plan", prompt: "summarise the roadmap" })}`,
@@ -191,20 +221,54 @@ describe("integrity on the real host", () => {
expect(systemText(compaction)).toContain("<factory-state>")
})
test("a QA seat cannot write, and its shell writes never escape (sandbox or allowlist)", async () => {
test("a QA seat cannot write, and its shell writes never escape (sandboxed, or refused without bwrap)", async () => {
const { tools } = await h.run(
`qa ${call("write", { path: "src/x.ts", content: "x" })} ${call("shell", { command: "touch QA_TOUCHED && echo touched" })}`,
`qa ${call("write", { path: "src/x.ts", content: "x" })} ${call("shell", { command: "touch QA_TOUCHED; echo tried" })}`,
{ agent: "es-qa-functional" },
)
expect(tools[0]?.status).toBe("error")
// With bwrap the command runs and the write vanishes; without it the
// allowlist refuses the command outright. Either way nothing escapes.
// With bwrap the command runs and the write fails or vanishes; without it
// factory seats get no shell at all (1.1.1). Either way nothing escapes.
const shell = tools[1]
if (shell?.status === "completed") expect(shell.text).toContain("touched")
else expect(shell?.text).toMatch(/allowlist|read-only seat/i)
if (bwrapAvailable()) expect(shell?.text).toContain("tried")
else expect(shell?.text).toMatch(/bubblewrap/i)
expect(await exists(path.join(h.directory, "QA_TOUCHED"))).toBe(false)
})
test.skipIf(!bwrapAvailable())(
"the 1.1.0 audit's shell attacks fail in the sandbox even when they evade the text rules",
async () => {
await mkdir(path.join(h.directory, ".factory/research/sources"), { recursive: true })
await writeFile(path.join(state, "key"), "PLANTED-SIGNING-KEY\n")
const sh = async (agent: string, command: string) =>
(await h.run(`x ${call("shell", { command })}`, { agent })).tools[0]
// P1/P5: variable indirection hides the factory dir from any text rule.
for (const agent of ["build", "es-programmer"]) {
const forged = await sh(agent, 'F=.fac; cd "${F}tory" && echo forged > research/sources/forged.md; echo done')
expect([agent, forged?.status]).toEqual([agent, "completed"])
expect(await exists(path.join(h.directory, ".factory/research/sources/forged.md"))).toBe(false)
}
// P3: the programmer reads the signing key through a glob.
const leak = await sh(
"es-programmer",
`D=${path.dirname(state)}; cat "$D"/${path.basename(state)}/k* ; echo done`,
)
expect(leak?.text ?? "").not.toContain("PLANTED-SIGNING-KEY")
// P4: a cached-web research seat reaches a local server.
const server = Bun.serve({ port: 0, hostname: "127.0.0.1", fetch: () => new Response("NET-REACHED") })
try {
const net = await sh(
"es-research-alpha",
`python3 -c "import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:${server.port}/').read())"; echo done`,
)
expect(net?.text ?? "").not.toContain("NET-REACHED")
} finally {
server.stop(true)
}
},
60_000,
)
test("the STOP file stops factory seats and es tools, not the user's own agent", async () => {
await mkdir(path.join(h.directory, ".factory"), { recursive: true })
await writeFile(path.join(h.directory, ".factory/STOP"), "pause")
@@ -215,6 +279,18 @@ describe("integrity on the real host", () => {
expect(user.tools[0]?.status).toBe("completed")
await rm(path.join(h.directory, ".factory/STOP"))
})
test("issue #49: no approval RPC exists for a stolen service password to call", async () => {
// S3 masks service.json in every agent sandbox (sandbox.test.ts); S4
// removed the mutate methods, so even a caller with a valid preview token
// has no RPC to redeem it through. Previews still work for the TUI.
const rpc = rpcFor(h) as any
for (const method of ["approve", "trust", "resume"]) expect(typeof rpc[method]).not.toBe("function")
const approval = await rpc.previewApproval({ stage: "frontier" }, where(h))
expect(typeof approval.title).toBe("string")
expect(Array.isArray(approval.lines)).toBe(true)
expect(((await rpc.previewTrust({}, where(h))).lines ?? []).length).toBeGreaterThan(0)
})
})
describe("a factory run end to end on the real host", () => {
@@ -262,13 +338,19 @@ describe("a factory run end to end on the real host", () => {
const preview = await rpc.previewApproval({ stage: "frontier" }, where(h))
expect(preview.ok).toBe(true)
expect(preview.lines.join("\n")).toContain("$5 USD")
await expect(rpc.approve({ stage: "frontier", user: "tester", token: "bogus" }, where(h))).rejects.toBeDefined()
expect(
(await rpc.approve({ stage: "frontier", user: "tester", token: preview.token }, where(h))).message,
).toContain("Approved frontier")
// S4: approve/trust/resume RPCs removed — TUI previews, terminal signs with the sealed key.
expect(typeof (rpc as any).approve).not.toBe("function")
await recordApproval(h.directory, { stage: "frontier", channel: "cli", signer })
await writeJson(
factoryLayout(h.directory).pending,
(await pendingApprovals(h.directory)).filter((item) => item.stage !== "frontier"),
)
const fac = new Factory(h.directory, { gates: gateRunner({ stateDir: state }), stateDir: state })
if (!(await fac.read())) await fac.begin("human:tester")
if ((await fac.read())?.stage === "GRILL") await fac.beginResearch("human:tester")
expect((await rpc.status({}, where(h))).stage).toBe("RESEARCH")
const cache = new SourceCache(researchSourcesDir(h.directory))
const cache = new SourceCache(researchSourcesDir(h.directory), state)
const source = await cache.put({
url: "https://example.test/greet",
text: "A greeting module exports greet(name) and returns a string.",
@@ -305,7 +387,12 @@ describe("a factory run end to end on the real host", () => {
const early = await h.run(call("es_build_start"), { agent: "factory" })
expect(early.tools[0]?.text).toContain("no spec approval")
const spec = await rpc.previewApproval({ stage: "spec" }, where(h))
await rpc.approve({ stage: "spec", user: "tester", token: spec.token }, where(h))
expect(spec.ok).toBe(true)
await recordApproval(h.directory, { stage: "spec", channel: "cli", signer })
await writeJson(
factoryLayout(h.directory).pending,
(await pendingApprovals(h.directory)).filter((item) => item.stage !== "spec"),
)
const build = await h.run(call("es_build_start"), { agent: "factory" })
expect(build.tools[0]?.status).toBe("completed")
expect(build.tools[0]?.text).toContain(`worktree:.factory/worktrees/${phase}`)
@@ -317,7 +404,17 @@ describe("a factory run end to end on the real host", () => {
const rpc = rpcFor(h)
const preview = await rpc.previewTrust({}, where(h))
expect(preview.lines.join("\n")).toContain("bun test")
expect((await rpc.trust({ user: "tester", token: preview.token }, where(h))).message).toContain("Trusted")
expect(typeof (rpc as any).trust).not.toBe("function")
const { config } = await loadGatesConfig(h.directory)
const gates = await commandSetHash(h.directory, config.commands)
const hooks = (await HookEngine.create({ root: h.directory, stateDir: state, audit: false })).status()
await trustProject(h.directory, gates.hash, gates.lines, { stateDir: state, signer })
if (hooks.projectHash)
await trustProject(h.directory, hooks.projectHash, hooks.projectLines, {
stateDir: state,
kind: "hooks",
signer,
})
}, 60_000)
test("programmer: red test first, implement in the worktree, gated complete", async () => {
packages/opencode/test/research.test.ts
+57 -3
View File
@@ -1,7 +1,7 @@
// Research stack on the real host: the configured provider is the host's
// websearch (results cached), research seats fetch and audit cited evidence.
import { afterAll, beforeAll, describe, expect, test } from "bun:test"
import { mkdtemp, rm, writeFile } from "node:fs/promises"
import { mkdir, mkdtemp, rm, stat, writeFile } from "node:fs/promises"
import { tmpdir } from "node:os"
import path from "node:path"
import { boot, directiveScript, type Harness } from "@heretek-ai/es-testkit"
@@ -55,9 +55,63 @@ afterAll(async () => {
await rm(state, { recursive: true, force: true })
})
const exists = (file: string) =>
stat(file).then(
() => true,
() => false,
)
describe("host web caching is for factory seats in a factory project only", () => {
let bare: Harness
beforeAll(async () => {
bare = await boot({
git: true,
script: directiveScript,
plugins: [{ path: pluginDir, options: { stateDir: state, pr: "off" } }],
files: { "README.md": "# bare\n" },
})
}, 60_000)
afterAll(() => bare?.close())
const web = (agent: string) =>
bare.run(
`${call("websearch", { query: "factory gates" })} ${call("webfetch", { url: `http://127.0.0.1:${server.port}/page`, format: "text" })}`,
{ agent },
)
test("without .factory/, neither the user's agent nor a seat caches, and no .factory/ appears", async () => {
for (const agent of ["build", "factory"]) {
const { tools } = await web(agent)
expect(tools.map((tool) => `${agent}:${tool.name}:${tool.status}`)).toEqual([
`${agent}:websearch:completed`,
`${agent}:webfetch:completed`,
])
expect(tools[0]?.text).toContain("gates keep agents honest")
expect(tools[1]?.text).toContain("Mechanical gates catch regressions")
for (const tool of tools) expect(tool.text).not.toContain("sha256:")
}
expect(await exists(path.join(bare.directory, ".factory"))).toBe(false)
}, 60_000)
test("in a factory project the user's agent is still untouched; a seat's results are cached and citable", async () => {
await mkdir(path.join(bare.directory, ".factory"), { recursive: true })
const user = await web("build")
for (const tool of user.tools) expect(tool.text).not.toContain("sha256:")
expect(await exists(path.join(bare.directory, ".factory/research/sources"))).toBe(false)
const seat = await web("factory")
expect(seat.tools[0]?.text).toContain("cached sha256:")
expect(seat.tools[0]?.text).toContain("gates keep agents honest")
const fetched = /\[cached as sha256:([0-9a-f]{64})/.exec(seat.tools[1]?.text ?? "")?.[1]
expect(fetched).toBeDefined()
expect(await exists(path.join(bare.directory, `.factory/research/sources/${fetched}.md`))).toBe(true)
}, 60_000)
})
describe("research on the real host", () => {
test("the host websearch tool runs through our cached provider", async () => {
const { tools } = await h.run(call("websearch", { query: "factory gates" }))
test("the host websearch tool runs through our cached provider for a factory seat", async () => {
await mkdir(path.join(h.directory, ".factory"), { recursive: true })
const { tools } = await h.run(call("websearch", { query: "factory gates" }), { agent: "factory" })
expect(tools[0]?.status).toBe("completed")
expect(tools[0]?.text).toContain("cached sha256:")
expect(tools[0]?.text).toContain("gates keep agents honest")
packages/opencode/test/tui.test.ts
+22 -10
View File
@@ -1,5 +1,5 @@
// The TUI plugin loads through the host's runtime Solid transform and only
// redeems an approval token after the human confirms the preview dialog.
// The TUI plugin loads through the host's runtime Solid transform and previews
// an approval, then points at the terminal (`es approve`) after confirm.
import { expect, test } from "bun:test"
import { Plugin, PluginContextProvider, usePlugin } from "@opencode/plugin/tui"
import { ensureRuntimePluginSupport } from "@opentui/solid/runtime-plugin-support/configure"
@@ -9,6 +9,7 @@ ensureRuntimePluginSupport({ additional: { "@opencode/plugin/tui": { Plugin, Plu
function fakeContext(confirmAnswer: boolean) {
const calls: Array<[string, any]> = []
const toasts: string[] = []
const alerts: string[] = []
const layers: any[] = []
const slots: any[] = []
const opened: string[] = []
@@ -29,7 +30,8 @@ function fakeContext(confirmAnswer: boolean) {
problems: [],
token: "tok-1",
}
if (method === "approve") return { message: "Approved frontier as tester." }
if (method === "configState")
return { config: "{}", sources: [], warnings: ["Ignored unknown plugin option: mode."] }
return {}
}
},
@@ -47,7 +49,9 @@ function fakeContext(confirmAnswer: boolean) {
toast: { show: (toast: any) => toasts.push(toast.message) },
dialog: {
confirm: async () => confirmAnswer,
alert: async () => {},
alert: async (input: any) => {
alerts.push(typeof input === "string" ? input : (input?.message ?? JSON.stringify(input)))
},
select: async () => "frontier",
prompt: async () => undefined,
},
@@ -67,13 +71,18 @@ function fakeContext(confirmAnswer: boolean) {
},
},
}
return { context, calls, toasts, layers, slots, opened }
return { context, calls, toasts, alerts, layers, slots, opened }
}
test("slash commands register; approval goes preview → confirm → approve(token)", async () => {
test("slash commands register; approval goes preview → confirm → terminal alert", async () => {
const plugin = (await import("../src/tui.tsx")).default as any
const confirmed = fakeContext(true)
plugin.setup(confirmed.context)
// Load warnings are fetched once at startup and shown as a toast.
await Bun.sleep(0)
expect(confirmed.calls.map(([method]) => method)).toEqual(["configState"])
expect(confirmed.toasts).toContain("Ignored unknown plugin option: mode.")
confirmed.calls.length = 0
const commands = confirmed.layers[0].commands
expect(commands.map((command: any) => command.slash.name)).toEqual([
"es-approve",
@@ -87,14 +96,17 @@ test("slash commands register; approval goes preview → confirm → approve(tok
"es-brainstorm",
])
await commands[0].run()
expect(confirmed.calls.map(([method]) => method)).toEqual(["status", "previewApproval", "approve"])
expect(confirmed.calls[2]![1]).toMatchObject({ stage: "frontier", token: "tok-1" })
expect(confirmed.toasts).toContain("Approved frontier as tester.")
expect(confirmed.calls.map(([method]) => method)).toEqual(["status", "previewApproval"])
expect(confirmed.alerts.join("\n")).toContain("es approve frontier")
const cancelled = fakeContext(false)
plugin.setup(cancelled.context)
await cancelled.layers[0].commands[0].run()
expect(cancelled.calls.map(([method]) => method)).toEqual(["status", "previewApproval"])
expect(cancelled.calls.map(([method]) => method).filter((method) => method !== "configState")).toEqual([
"status",
"previewApproval",
])
expect(cancelled.alerts).toHaveLength(0)
})
test("four panels claim session.panel and their commands open them", async () => {
schemas/approval.schema.json
+25 -6
View File
@@ -4,7 +4,7 @@
"properties": {
"version": {
"type": "number",
"const": 1
"const": 2
},
"stage": {
"type": "string",
@@ -90,10 +90,29 @@
"notes": {
"type": "string"
},
"mac": {
"type": "string",
"minLength": 64,
"maxLength": 64
"signature": {
"type": "object",
"properties": {
"alg": {
"type": "string",
"const": "ed25519"
},
"keyId": {
"type": "string",
"minLength": 16,
"maxLength": 16
},
"sig": {
"type": "string",
"minLength": 1
}
},
"required": [
"alg",
"keyId",
"sig"
],
"additionalProperties": false
}
},
"required": [
@@ -105,7 +124,7 @@
"approvedAt",
"channel",
"auditHead",
"mac"
"signature"
],
"additionalProperties": false
}
schemas/brief.schema.json
+3 -3
View File
@@ -247,19 +247,19 @@
"properties": {
"alg": {
"type": "string",
"const": "hmac-sha256"
"const": "ed25519"
},
"keyId": {
"type": "string"
},
"mac": {
"sig": {
"type": "string"
}
},
"required": [
"alg",
"keyId",
"mac"
"sig"
],
"additionalProperties": false
}
schemas/record-signature.schema.json
+25
View File
@@ -0,0 +1,25 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"alg": {
"type": "string",
"const": "ed25519"
},
"keyId": {
"type": "string",
"minLength": 16,
"maxLength": 16
},
"sig": {
"type": "string",
"minLength": 1
}
},
"required": [
"alg",
"keyId",
"sig"
],
"additionalProperties": false
}
schemas/waiver.schema.json
+25 -6
View File
@@ -4,7 +4,7 @@
"properties": {
"version": {
"type": "number",
"const": 1
"const": 2
},
"id": {
"type": "string",
@@ -71,10 +71,29 @@
}
]
},
"mac": {
"type": "string",
"minLength": 64,
"maxLength": 64
"signature": {
"type": "object",
"properties": {
"alg": {
"type": "string",
"const": "ed25519"
},
"keyId": {
"type": "string",
"minLength": 16,
"maxLength": 16
},
"sig": {
"type": "string",
"minLength": 1
}
},
"required": [
"alg",
"keyId",
"sig"
],
"additionalProperties": false
}
},
"required": [
@@ -88,7 +107,7 @@
"expiresAt",
"channel",
"auditHead",
"mac"
"signature"
],
"additionalProperties": false
}
scripts/docs.ts
+2 -1
View File
@@ -135,7 +135,8 @@ const configDoc = [
"",
"The canonical Epistemic Swarm config (`packages/core/src/schema/config.ts`), layered:",
"user-global (`~/.config/epistemic-swarm/config.json`) → project (`.factory/config.json`) →",
"plugin options in `opencode.json`. Unknown keys are rejected. Generated: do not edit by hand.",
"plugin options in `opencode.json`. An unknown key in a config file is rejected; an unknown",
"plugin option is ignored with a warning (TUI toast and `/config`). Generated: do not edit by hand.",
"",
"| field | type | default | description |",
"| --- | --- | --- | --- |",
scripts/e2e-report.ts
+181
View File
@@ -0,0 +1,181 @@
// Deterministic debug/feedback report for the E2E eval run (`bun run evals`).
// Reads an evals/results/<stamp>.json (or the newest one) plus environment
// context (model, opencode version, git SHA, plugin version) and writes a
// human- and agent-readable markdown report next to it. Pure and offline: no
// model calls, no secrets touched, safe to run with `if: always()` in CI.
import { readdir, readFile, writeFile } from "node:fs/promises"
import path from "node:path"
const root = path.resolve(import.meta.dir, "..")
const resultsDir = path.join(root, "evals/results")
interface EvalRow {
id: string
agent?: string
description?: string
pass: boolean
exitCode?: number | null
capped?: boolean
timedOut?: boolean
overBudget?: boolean
maxSteps?: number
steps?: number
tools?: readonly string[]
completed?: readonly string[]
costUSD?: number
errors?: readonly string[]
textExcerpt?: string
failures?: readonly string[]
/** Where the failed case's workspace, events and stderr were kept. */
kept?: string
}
interface EvalFile {
at?: string
model?: string
stepCeiling?: number
budgetUSD?: number
spentUSD?: number
results?: EvalRow[]
}
function usage(): never {
console.error("usage: bun scripts/e2e-report.ts [results-json] [opencode-version] [git-sha]")
process.exit(2)
}
const arg = process.argv[2]
const opencodeVersion =
process.argv[3] ?? process.env.ES_OPENCODE_VERSION ?? (await safeCommand("opencode", ["--version"]))
const gitSha = process.argv[4] ?? process.env.GITHUB_SHA ?? (await safeCommand("git", ["rev-parse", "--short", "HEAD"]))
async function safeCommand(binary: string, args: string[]): Promise<string> {
try {
const child = Bun.spawn([binary, ...args], { stdout: "pipe", stderr: "ignore" })
const out = await new Response(child.stdout).text()
await child.exited
return out.trim().slice(0, 80) || "unknown"
} catch {
return "unknown"
}
}
async function pickResultsFile(): Promise<string> {
if (arg) return path.isAbsolute(arg) ? arg : path.join(root, arg)
// Current `bun run evals` writes flat <stamp>.json files. Timestamped
// subdirectories underneath hold a legacy schema (no top-level `results`);
// accept those only if they match the current shape.
const flat = (await readdir(resultsDir).catch(() => [] as string[]))
.filter((file) => file.endsWith(".json"))
.sort()
const flatLatest = flat.at(-1)
if (flatLatest) return path.join(resultsDir, flatLatest)
const nested: string[] = []
for (const entry of await readdir(resultsDir).catch(() => [] as string[])) {
if (!entry.endsWith(".json")) {
const dir = path.join(resultsDir, entry)
for (const file of await readdir(dir).catch(() => [] as string[])) {
if (file.endsWith(".json")) nested.push(path.join(dir, file))
}
}
}
// Newest first by filename (stamps sort lexicographically); first file with
// the current schema wins.
nested.sort().reverse()
for (const file of nested) {
try {
const data = JSON.parse(await readFile(file, "utf8")) as { results?: unknown }
if (Array.isArray(data.results)) return file
} catch {
// not readable JSON; keep looking
}
}
console.error(`e2e-report: no current-format results JSON in ${resultsDir}; run \`bun run evals\` first.`)
process.exit(1)
}
const resultsPath = await pickResultsFile()
const raw = await readFile(resultsPath, "utf8").catch(() => usage())
const data = JSON.parse(raw) as EvalFile
const rows = data.results ?? []
const model = data.model ?? process.env.ES_EVAL_MODEL ?? "unknown"
const passed = rows.filter((row) => row.pass).length
const pluginPkg = await readFile(path.join(root, "packages/opencode/package.json"), "utf8")
.then((text) => (JSON.parse(text) as { version?: string }).version ?? "unknown")
.catch(() => "unknown")
const corePkg = await readFile(path.join(root, "packages/core/package.json"), "utf8")
.then((text) => (JSON.parse(text) as { version?: string }).version ?? "unknown")
.catch(() => "unknown")
const lines: string[] = []
lines.push(`# E2E feedback report`)
lines.push(``)
lines.push(`- At: ${data.at ?? "unknown"} (source: \`${path.relative(root, resultsPath)}\`)`)
lines.push(`- Model: \`${model}\``)
lines.push(`- opencode: \`${opencodeVersion}\` (plugin \`@heretek-ai/epistemic-swarm@${pluginPkg}\`, core \`@${corePkg}\`)`)
lines.push(`- Git: \`${gitSha}\``)
lines.push(`- Step caps: per case (ceiling ${data.stepCeiling ?? "unknown"})`)
lines.push(
`- Spend: $${(data.spentUSD ?? 0).toFixed(4)}${data.budgetUSD === undefined ? " (no budget set)" : ` of $${data.budgetUSD}`}`,
)
lines.push(`- Result: **${passed}/${rows.length} passed**${passed === rows.length ? " ✅" : " ❌"}`)
lines.push(``)
lines.push(`| case | agent | steps | cost | tools | verdict | failures |`)
lines.push(`| ---- | ----- | ----- | ---- | ----- | ------- | -------- |`)
for (const row of rows) {
const failures = (row.failures ?? []).join("; ").slice(0, 160).replaceAll("|", "\\|")
lines.push(
`| \`${row.id}\` | ${row.agent ?? "?"} | ${row.steps ?? "?"}/${row.maxSteps ?? "?"} | $${(row.costUSD ?? 0).toFixed(4)} | ${(row.tools ?? []).join(", ") || "—"} | ${row.pass ? "PASS" : "FAIL"} | ${failures || "—"} |`,
)
}
lines.push(``)
for (const row of rows) {
lines.push(`## ${row.pass ? "PASS" : "FAIL"} ${row.id}`)
lines.push(``)
if (row.description) lines.push(`${row.description}`)
lines.push(``)
lines.push(
`- Agent: \`${row.agent ?? "?"}\`, exit: ${row.exitCode ?? "?"}, capped: ${row.capped ?? "?"}, timed out: ${row.timedOut ?? "?"}, over budget: ${row.overBudget ?? "?"}`,
)
const refused = (row.tools ?? []).filter((tool, index, all) => all.indexOf(tool) === index && !(row.completed ?? []).includes(tool))
if (row.completed && refused.length > 0) lines.push(`- Called but never completed: ${refused.join(", ")}`)
if (row.kept) lines.push(`- Evidence kept at \`${row.kept}/\` (project with \`.factory/\`, \`events.jsonl\`, \`stderr.txt\`)`)
if ((row.errors ?? []).length > 0) {
lines.push(`- Error events:`)
for (const error of row.errors!) lines.push(` - ${error.slice(0, 300)}`)
}
if ((row.failures ?? []).length > 0) {
lines.push(`- Failures:`)
for (const failure of row.failures!) lines.push(` - ${failure.slice(0, 300)}`)
}
const excerpt = (row.textExcerpt ?? "").trim()
if (excerpt) {
lines.push(`- Agent output (first 2000 chars):`)
lines.push(` \`\`\``)
for (const line of excerpt.split("\n").slice(0, 30)) lines.push(` ${line}`)
lines.push(` \`\`\``)
}
lines.push(``)
}
lines.push(`## Reproducing locally`)
lines.push(``)
lines.push(`\`\`\`bash`)
lines.push(`export ES_EVAL_MODEL="${model}"`)
lines.push(`export OPENCODE_API_KEY="<key for the model's provider>" # or \$ES_EVAL_KEY_VAR equivalent`)
lines.push(`export ES_EVAL_MAX_USD=15 # optional spend budget for the whole run`)
lines.push(`npm install --global --ignore-scripts @opencode/[email protected]`)
lines.push(`bun run evals`)
lines.push(`bun scripts/e2e-report.ts # regenerates this file from the newest evals/results/*.json`)
lines.push(`\`\`\``)
lines.push(``)
lines.push(`## What to look at when a case fails`)
lines.push(``)
lines.push(`1. \`exit != 0\` with no error events → CLI/host crash; rerun that case's prompt by hand (see \`evals/cases/<id>.json\`).`)
lines.push(`2. \`capped: true\` → the agent ran past its case's step cap; check \`tools\` for a repeated call cycle, then the kept \`events.jsonl\`.`)
lines.push(`3. \`must not call\` / \`did not call\` → seat scoping or prompt regression in \`packages/core/assets\` or the registry.`)
lines.push(` \`called but did not complete\` → the tool refused; its reason is in the kept \`events.jsonl\` (tool_use with status error).`)
lines.push(`4. \`missing: <text>\` → behaviour drift in the seat's first-turn answer; compare against the case description.`)
const reportPath = resultsPath.replace(/\.json$/, ".md")
await writeFile(reportPath, `${lines.join("\n")}\n`)
console.log(`e2e-report: wrote ${path.relative(root, reportPath)} (${passed}/${rows.length} passed)`)
scripts/evals.ts
+63 -18
View File
@@ -1,15 +1,20 @@
// Opt-in, cost-capped behaviour evals (grill / brainstorm / factory). Never
// part of `bun test`; run by `bun run evals` or the nightly job. Needs the
// Opt-in, cost-capped behaviour evals. Never part of `bun test`; run by
// `bun run evals`, the E2E feedback workflow or the nightly job. Needs the
// opencode CLI and ES_EVAL_MODEL ("provider/model"; the provider's API key in
// its usual env var) and skips cleanly without them. Each case runs one real
// turn in an isolated workspace loading this repo's plugin, killed at the
// step cap (ES_EVAL_MAX_STEPS, default 6; a case may set a lower maxSteps) or
// the timeout (ES_EVAL_TIMEOUT_MS, default 300000).
// turn in an isolated workspace loading this repo's plugin, killed at:
// - its own step cap (`maxSteps`, sized for a model that makes one call per
// step), bounded by the ceiling ES_EVAL_MAX_STEPS (default 16);
// - the timeout (ES_EVAL_TIMEOUT_MS, default 300000);
// - the run's spend budget ES_EVAL_MAX_USD (optional; summed from the stream's
// step_finish cost). Once it is spent, the remaining cases are not run.
// A failed case keeps its workspace, event stream and stderr under
// evals/results/<stamp>/<case>/ for debugging.
import { accessSync, constants } from "node:fs"
import { mkdir, readdir, readFile, writeFile } from "node:fs/promises"
import { cp, mkdir, readdir, readFile, writeFile } from "node:fs/promises"
import path from "node:path"
import { gradeFireCase } from "../packages/core/src/evals/fires.ts"
import { type EvalCase, evalWorkspace, runEvalCase } from "../packages/core/src/evals/run.ts"
import { caseStepCap, type EvalCase, evalWorkspace, runEvalCase } from "../packages/core/src/evals/run.ts"
const root = path.resolve(import.meta.dir, "..")
const casesDir = path.join(root, "evals/cases")
@@ -24,8 +29,17 @@ async function seedFiles(files: Record<string, string> | undefined): Promise<Rec
}
const resultsDir = path.join(root, "evals/results")
const model = process.env.ES_EVAL_MODEL
const maxSteps = Number(process.env.ES_EVAL_MAX_STEPS ?? 6)
const timeoutMs = Number(process.env.ES_EVAL_TIMEOUT_MS ?? 300_000)
const stepCeiling = Number(process.env.ES_EVAL_MAX_STEPS || 16)
const timeoutMs = Number(process.env.ES_EVAL_TIMEOUT_MS || 300_000)
const budgetUSD = process.env.ES_EVAL_MAX_USD ? Number(process.env.ES_EVAL_MAX_USD) : undefined
if (!Number.isInteger(stepCeiling) || stepCeiling <= 0) {
console.error(`evals: ES_EVAL_MAX_STEPS must be a positive integer (got ${process.env.ES_EVAL_MAX_STEPS}).`)
process.exit(2)
}
if (budgetUSD !== undefined && !(Number.isFinite(budgetUSD) && budgetUSD > 0)) {
console.error(`evals: ES_EVAL_MAX_USD must be a positive number (got ${process.env.ES_EVAL_MAX_USD}).`)
process.exit(2)
}
function resolveOnPath(binary: string): string | undefined {
for (const dir of (process.env.PATH ?? "").split(path.delimiter)) {
@@ -46,35 +60,66 @@ if (!binary || !model) {
process.exit(0)
}
const stamp = new Date().toISOString().replace(/[:.]/g, "-")
const files = (await readdir(casesDir)).filter((file) => file.endsWith(".json")).sort()
const results = []
let spentUSD = 0
for (const file of files) {
const testCase = JSON.parse(await readFile(path.join(casesDir, file), "utf8")) as EvalCase & { maxSteps?: number }
const testCase = JSON.parse(await readFile(path.join(casesDir, file), "utf8")) as EvalCase
const maxSteps = caseStepCap(testCase.maxSteps, stepCeiling)
const remaining = budgetUSD === undefined ? undefined : budgetUSD - spentUSD
if (remaining !== undefined && remaining <= 0) {
const failure = `not run: spend budget $${budgetUSD} used up`
results.push({ id: testCase.id, agent: testCase.agent, description: testCase.description, pass: false, maxSteps, failures: [failure] })
console.log(`SKIP ${testCase.id} — ${failure}`)
continue
}
const workspace = await evalWorkspace({
pluginDir: path.join(root, "packages/opencode"),
model,
files: await seedFiles(testCase.files as Record<string, string> | undefined),
})
try {
const ran = await runEvalCase(testCase, workspace, {
binary,
maxSteps: Math.min(testCase.maxSteps ?? maxSteps, maxSteps),
timeoutMs,
})
const ran = await runEvalCase(testCase, workspace, { binary, maxSteps, timeoutMs, budgetUSD: remaining })
spentUSD += ran.transcript.costUSD
// A fire is scored by the same deterministic grader the CI suites use.
const fire = testCase.fire ? await gradeFireCase(testCase.fire, workspace.dir) : undefined
const result = fire
? { ...ran, pass: ran.pass && fire.pass, failures: [...ran.failures, ...fire.failures] }
: ran
// Keep a failed case's evidence: its project (with .factory/), events and stderr.
let kept: string | undefined
if (!result.pass) {
const keep = path.join(resultsDir, stamp, testCase.id)
await mkdir(keep, { recursive: true })
await cp(workspace.dir, path.join(keep, "project"), { recursive: true, verbatimSymlinks: true })
await writeFile(path.join(keep, "events.jsonl"), result.events.map((event) => JSON.stringify(event)).join("\n"))
await writeFile(path.join(keep, "stderr.txt"), result.stderr)
kept = path.relative(root, keep)
}
results.push({
id: result.id,
agent: testCase.agent,
description: testCase.description,
pass: result.pass,
maxSteps,
exitCode: result.exitCode,
capped: result.capped,
timedOut: result.timedOut,
overBudget: result.overBudget,
steps: result.transcript.steps,
tools: result.transcript.tools,
completed: result.transcript.completed,
costUSD: result.transcript.costUSD,
tokens: result.transcript.tokens,
errors: result.transcript.errors,
// The full text can be large; the debug report carries an excerpt.
textExcerpt: result.transcript.text.slice(0, 2000),
failures: result.failures,
...(kept ? { kept } : {}),
})
console.log(
`${result.pass ? "PASS" : "FAIL"} ${result.id} (${result.transcript.steps} step(s))${result.pass ? "" : ` — ${result.failures.join("; ")}`}`,
`${result.pass ? "PASS" : "FAIL"} ${result.id} (${result.transcript.steps}/${maxSteps} step(s), $${result.transcript.costUSD.toFixed(4)})${result.pass ? "" : ` — ${result.failures.join("; ")}`}`,
)
} finally {
await workspace.cleanup()
@@ -82,9 +127,9 @@ for (const file of files) {
}
await mkdir(resultsDir, { recursive: true })
const stamp = new Date().toISOString().replace(/[:.]/g, "-")
await writeFile(
path.join(resultsDir, `${stamp}.json`),
`${JSON.stringify({ at: stamp, model, maxSteps, results }, null, 2)}\n`,
`${JSON.stringify({ at: stamp, model, stepCeiling, budgetUSD, spentUSD, results }, null, 2)}\n`,
)
console.log(`evals: spent $${spentUSD.toFixed(4)}${budgetUSD === undefined ? "" : ` of $${budgetUSD}`}.`)
if (results.some((result) => !result.pass)) process.exit(1)