mirror of
https://github.com/esengine/DeepSeek-Reasonix.git
synced 2026-11-07 10:45:59 -10:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
74e796b1c6 | ||
|
|
c1965b1d1e | ||
|
|
725cb16ff8 | ||
|
|
ceb4840a63 | ||
|
|
4bca4cedd8 | ||
|
|
736611293f | ||
|
|
dc5a65b02b | ||
|
|
a930d1818a | ||
|
|
7fed18fc47 | ||
|
|
cc53474d56 | ||
|
|
359bd6bba3 | ||
|
|
ff91ff91f0 | ||
|
|
c72b429d82 | ||
|
|
989d8b33d8 | ||
|
|
c40afe5caf | ||
|
|
30ebd1353a | ||
|
|
ffc872ae12 | ||
|
|
c04e71ee9b | ||
|
|
2169ad233f | ||
|
|
8c426595fa | ||
|
|
2e3c479402 | ||
|
|
c4dd646642 | ||
|
|
c0bc69e879 | ||
|
|
437e5cc41f | ||
|
|
f1b740596c | ||
|
|
d626b639fa | ||
|
|
f413c4dc6c | ||
|
|
2199a21c4d | ||
|
|
b7f29c384a | ||
|
|
0f0a44a00c | ||
|
|
214dc13a38 | ||
|
|
785f4d65c8 | ||
|
|
05298c0faf | ||
|
|
cc3d2bfcb1 | ||
|
|
e9e7ccc622 |
No files matched your search
@@ -0,0 +1,4 @@
|
||||
DEEPSEEK_API_KEY=sk-your-key-here
|
||||
DEEPSEEK_BASE_URL=https://api.deepseek.com
|
||||
REASONIX_LOG_LEVEL=INFO
|
||||
REASONIX_TRANSCRIPT_DIR=./transcripts
|
||||
@@ -1,11 +0,0 @@
|
||||
.gitattributes text eol=lf
|
||||
.gitignore text eol=lf
|
||||
*.json text eol=lf
|
||||
*.md text eol=lf
|
||||
*.mjs text eol=lf
|
||||
*.sh text eol=lf
|
||||
*.sql text eol=lf
|
||||
*.toml text eol=lf
|
||||
*.ts text eol=lf
|
||||
*.yaml text eol=lf
|
||||
*.yml text eol=lf
|
||||
@@ -0,0 +1,23 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Something is broken
|
||||
labels: bug
|
||||
---
|
||||
|
||||
**What happened**
|
||||
A clear and concise description.
|
||||
|
||||
**Expected**
|
||||
What you expected to happen.
|
||||
|
||||
**Reproduction**
|
||||
Steps or minimal code that reproduces it.
|
||||
|
||||
**Environment**
|
||||
- Reasonix version:
|
||||
- Node version:
|
||||
- OS:
|
||||
- DeepSeek model (e.g. `deepseek-chat`, `deepseek-reasoner`):
|
||||
|
||||
**Logs / transcript**
|
||||
If using the CLI, attach the relevant chunk of `--transcript`.
|
||||
@@ -0,0 +1,39 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: build (node ${{ matrix.node }})
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
node: ["20", "22"]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node ${{ matrix.node }}
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ matrix.node }}
|
||||
cache: npm
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Lint (biome)
|
||||
run: npm run lint
|
||||
|
||||
- name: Typecheck
|
||||
run: npm run typecheck
|
||||
|
||||
- name: Test (vitest)
|
||||
run: npm test
|
||||
|
||||
- name: Build (tsup)
|
||||
run: npm run build
|
||||
@@ -1,103 +0,0 @@
|
||||
name: Deploy identity and remote services
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: deploy-identity-and-remote-services
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
deploy-accounts:
|
||||
if: github.ref == 'refs/heads/platform'
|
||||
runs-on: ubuntu-latest
|
||||
environment: production
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
cache-dependency-path: workers/accounts/pnpm-lock.yaml
|
||||
- name: Deploy
|
||||
working-directory: workers/accounts
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
run: |
|
||||
pnpm install --frozen-lockfile
|
||||
pnpm test
|
||||
pnpm typecheck
|
||||
pnpm db:apply:remote
|
||||
npx wrangler deploy
|
||||
- name: Sync RESEND_API_KEY to the worker
|
||||
working-directory: workers/accounts
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
|
||||
run: |
|
||||
if [ -n "$RESEND_API_KEY" ]; then
|
||||
printf '%s' "$RESEND_API_KEY" | npx wrangler secret put RESEND_API_KEY
|
||||
else
|
||||
echo "RESEND_API_KEY not set in repo secrets; skipping (worker stays in stub email mode)."
|
||||
fi
|
||||
- name: Sync remote gateway secret
|
||||
working-directory: workers/accounts
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
REMOTE_GATEWAY_TOKEN: ${{ secrets.REMOTE_GATEWAY_TOKEN }}
|
||||
run: |
|
||||
if [ -z "$REMOTE_GATEWAY_TOKEN" ]; then
|
||||
echo "REMOTE_GATEWAY_TOKEN must be configured before enabling remote access."
|
||||
exit 1
|
||||
fi
|
||||
printf '%s' "$REMOTE_GATEWAY_TOKEN" | npx wrangler secret put REMOTE_GATEWAY_TOKEN
|
||||
- name: Sync feedback account secret
|
||||
working-directory: workers/accounts
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
FEEDBACK_ACCOUNT_SECRET: ${{ secrets.FEEDBACK_ACCOUNT_SECRET }}
|
||||
run: |
|
||||
if [ -n "$FEEDBACK_ACCOUNT_SECRET" ]; then
|
||||
printf '%s' "$FEEDBACK_ACCOUNT_SECRET" | npx wrangler secret put FEEDBACK_ACCOUNT_SECRET
|
||||
else
|
||||
echo "FEEDBACK_ACCOUNT_SECRET not set in repo secrets; feedback account linking stays unavailable."
|
||||
fi
|
||||
|
||||
deploy-remote-gateway:
|
||||
needs: deploy-accounts
|
||||
if: github.ref == 'refs/heads/platform'
|
||||
runs-on: ubuntu-latest
|
||||
environment: production
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "22"
|
||||
cache: npm
|
||||
cache-dependency-path: workers/remote-gateway/package-lock.json
|
||||
- name: Validate gateway secret
|
||||
env:
|
||||
REMOTE_GATEWAY_TOKEN: ${{ secrets.REMOTE_GATEWAY_TOKEN }}
|
||||
run: test -n "$REMOTE_GATEWAY_TOKEN"
|
||||
- name: Validate and deploy
|
||||
working-directory: workers/remote-gateway
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
run: |
|
||||
npm ci
|
||||
npm test
|
||||
npm run typecheck
|
||||
npx wrangler deploy
|
||||
- name: Sync gateway secret
|
||||
working-directory: workers/remote-gateway
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
REMOTE_GATEWAY_TOKEN: ${{ secrets.REMOTE_GATEWAY_TOKEN }}
|
||||
run: printf '%s' "$REMOTE_GATEWAY_TOKEN" | npx wrangler secret put REMOTE_GATEWAY_TOKEN
|
||||
@@ -1,254 +0,0 @@
|
||||
name: Deploy crash worker
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
firebase_data_action:
|
||||
description: Firebase crash history operation (migration actions do not deploy the Worker)
|
||||
required: true
|
||||
default: none
|
||||
type: choice
|
||||
options:
|
||||
- none
|
||||
- dry-run
|
||||
- apply
|
||||
- verify-only
|
||||
confirmation:
|
||||
description: Enter APPLY_FIREBASE_CRASH_DATA when selecting apply
|
||||
required: false
|
||||
default: ''
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: deploy-crash-worker
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
if: github.ref == 'refs/heads/platform' && inputs.firebase_data_action == 'none'
|
||||
runs-on: ubuntu-latest
|
||||
environment: production
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
cache-dependency-path: workers/crash-report/package-lock.json
|
||||
- name: Validate
|
||||
working-directory: workers/crash-report
|
||||
run: |
|
||||
npm ci
|
||||
npm run typecheck
|
||||
npm test
|
||||
- name: Apply and verify diagnostics v2 D1 migration
|
||||
working-directory: workers/crash-report
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
run: npm run migrate:diagnostics-v2
|
||||
- name: Apply and verify Firebase crash D1 migration
|
||||
working-directory: workers/crash-report
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
run: npm run migrate:firebase-crash
|
||||
- name: Apply dashboard performance indexes
|
||||
working-directory: workers/crash-report
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
run: npm run migrate:dashboard-indexes
|
||||
- name: Apply telemetry queue receipts schema
|
||||
working-directory: workers/crash-report
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
run: npx wrangler d1 execute reasonix-crash --remote --file=migrate-telemetry-queue.sql
|
||||
- name: Apply and verify feedback D1 migration
|
||||
working-directory: workers/crash-report
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
run: |
|
||||
npx wrangler d1 execute reasonix-crash --remote --file=migrate-feedback.sql
|
||||
out=$(npx wrangler d1 execute reasonix-crash --remote --json --command "SELECT name FROM sqlite_master WHERE name IN ('feedback','feedback_quota','feedback_status_created','feedback_install_created','feedback_status_version','feedback_quota_day')")
|
||||
for name in feedback feedback_quota feedback_status_created feedback_install_created feedback_status_version feedback_quota_day; do
|
||||
printf '%s' "$out" | grep -q "\"$name\"" || { echo "missing $name after migration"; exit 1; }
|
||||
done
|
||||
- name: Apply and verify feedback triage D1 migration
|
||||
working-directory: workers/crash-report
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
run: |
|
||||
npx wrangler d1 execute reasonix-crash --remote --file=migrate-feedback-triage.sql
|
||||
names="feedback_releases feedback_releases_install feedback_replies feedback_blocks feedback_trust feedback_public_images feedback_config feedback_audit feedback_replies_receipt feedback_replies_unhandled feedback_blocks_expires feedback_public_images_receipt feedback_audit_at feedback_install_status_updated feedback_reply_attachments feedback_reply_attachments_receipt"
|
||||
list=$(for n in $names; do printf "'%s'," "$n"; done | sed 's/,$//')
|
||||
out=$(npx wrangler d1 execute reasonix-crash --remote --json --command "SELECT name FROM sqlite_master WHERE name IN ($list)")
|
||||
for name in $names; do
|
||||
printf '%s' "$out" | grep -q "\"$name\"" || { echo "missing $name after migration"; exit 1; }
|
||||
done
|
||||
echo "Trusted installs (hash prefix only) for review:"
|
||||
npx wrangler d1 execute reasonix-crash --remote --json --command "SELECT substr(install_hash,1,8) AS prefix, created_at, expires_at FROM feedback_trust ORDER BY created_at LIMIT 200"
|
||||
- name: Apply and verify feedback adoption D1 migration
|
||||
working-directory: workers/crash-report
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
run: |
|
||||
npx wrangler d1 execute reasonix-crash --remote --file=migrate-feedback-adoptions.sql
|
||||
names="feedback_adoptions feedback_adoptions_install feedback_adoptions_item feedback_level_state"
|
||||
list=$(for n in $names; do printf "'%s'," "$n"; done | sed 's/,$//')
|
||||
out=$(npx wrangler d1 execute reasonix-crash --remote --json --command "SELECT name FROM sqlite_master WHERE name IN ($list)")
|
||||
for name in $names; do
|
||||
printf '%s' "$out" | grep -q "\"$name\"" || { echo "missing $name after migration"; exit 1; }
|
||||
done
|
||||
- name: Apply and verify feedback account link D1 migration
|
||||
working-directory: workers/crash-report
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
run: |
|
||||
npx wrangler d1 execute reasonix-crash --remote --file=migrate-feedback-account-links.sql
|
||||
names="feedback_account_links feedback_account_links_account"
|
||||
list=$(for n in $names; do printf "'%s'," "$n"; done | sed 's/,$//')
|
||||
out=$(npx wrangler d1 execute reasonix-crash --remote --json --command "SELECT name FROM sqlite_master WHERE name IN ($list)")
|
||||
for name in $names; do
|
||||
printf '%s' "$out" | grep -q "\"$name\"" || { echo "missing $name after migration"; exit 1; }
|
||||
done
|
||||
- name: Apply and verify feedback contribution board D1 migration
|
||||
working-directory: workers/crash-report
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
run: |
|
||||
npx wrangler d1 execute reasonix-crash --remote --file=migrate-feedback-board.sql
|
||||
names="feedback_board_optin feedback_board_delisted"
|
||||
list=$(for n in $names; do printf "'%s'," "$n"; done | sed 's/,$//')
|
||||
out=$(npx wrangler d1 execute reasonix-crash --remote --json --command "SELECT name FROM sqlite_master WHERE name IN ($list)")
|
||||
for name in $names; do
|
||||
printf '%s' "$out" | grep -q "\"$name\"" || { echo "missing $name after migration"; exit 1; }
|
||||
done
|
||||
- name: Apply Studio telemetry schema
|
||||
working-directory: workers/crash-report
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
run: npx wrangler d1 execute reasonix-crash --remote --file=migrate-studio-telemetry.sql
|
||||
- name: Apply isolated telemetry database schema
|
||||
working-directory: workers/crash-report
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
run: npm run migrate:telemetry-db
|
||||
- name: Apply registry performance indexes and install rollup
|
||||
working-directory: workers/crash-report
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
run: npm run migrate:registry-indexes
|
||||
- name: Sync Firebase crash secrets
|
||||
working-directory: workers/crash-report
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
FIREBASE_DATABASE_URL: ${{ secrets.FIREBASE_DATABASE_URL }}
|
||||
FIREBASE_CLIENT_EMAIL: ${{ secrets.FIREBASE_CLIENT_EMAIL }}
|
||||
FIREBASE_PRIVATE_KEY: ${{ secrets.FIREBASE_PRIVATE_KEY }}
|
||||
run: |
|
||||
configured=0
|
||||
[ -n "$FIREBASE_DATABASE_URL" ] && configured=$((configured + 1))
|
||||
[ -n "$FIREBASE_CLIENT_EMAIL" ] && configured=$((configured + 1))
|
||||
[ -n "$FIREBASE_PRIVATE_KEY" ] && configured=$((configured + 1))
|
||||
if [ "$configured" -ne 0 ] && [ "$configured" -ne 3 ]; then
|
||||
echo "Firebase crash secrets must be either all configured or all absent."
|
||||
exit 1
|
||||
fi
|
||||
node -e 'process.stdout.write(JSON.stringify({FIREBASE_DATABASE_URL:process.env.FIREBASE_DATABASE_URL||null,FIREBASE_CLIENT_EMAIL:process.env.FIREBASE_CLIENT_EMAIL||null,FIREBASE_PRIVATE_KEY:process.env.FIREBASE_PRIVATE_KEY||null}))' | npx wrangler secret bulk
|
||||
- name: Deploy
|
||||
working-directory: workers/crash-report
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
run: |
|
||||
npx wrangler deploy
|
||||
- name: Sync feedback secrets
|
||||
working-directory: workers/crash-report
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
FEEDBACK_TOKEN_SECRET: ${{ secrets.FEEDBACK_TOKEN_SECRET }}
|
||||
FEEDBACK_ADMIN_TOKEN: ${{ secrets.FEEDBACK_ADMIN_TOKEN }}
|
||||
OPS_EMIT_TOKEN: ${{ secrets.OPS_EMIT_TOKEN }}
|
||||
FEEDBACK_ACCOUNT_SECRET: ${{ secrets.FEEDBACK_ACCOUNT_SECRET }}
|
||||
run: |
|
||||
node -e 'process.stdout.write(JSON.stringify({FEEDBACK_TOKEN_SECRET:process.env.FEEDBACK_TOKEN_SECRET||null,FEEDBACK_ADMIN_TOKEN:process.env.FEEDBACK_ADMIN_TOKEN||null,OPS_EMIT_TOKEN:process.env.OPS_EMIT_TOKEN||null,FEEDBACK_ACCOUNT_SECRET:process.env.FEEDBACK_ACCOUNT_SECRET||null}))' | npx wrangler secret bulk
|
||||
# Mirrors the ALERT_WEBHOOK repo secret into the worker so the ingest
|
||||
# sentinel can push alerts (see runIngestSentinel). Managed here so
|
||||
# nobody needs local Cloudflare credentials; when the repo secret is
|
||||
# unset the sentinel stays log-only.
|
||||
- name: Sync alert webhook secret
|
||||
working-directory: workers/crash-report
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
ALERT_WEBHOOK: ${{ secrets.ALERT_WEBHOOK }}
|
||||
run: |
|
||||
if [ -z "$ALERT_WEBHOOK" ]; then
|
||||
echo "ALERT_WEBHOOK repo secret not set; removing any existing worker secret."
|
||||
printf '{"ALERT_WEBHOOK":null}' | npx wrangler secret bulk
|
||||
exit 0
|
||||
fi
|
||||
printf '%s' "$ALERT_WEBHOOK" | npx wrangler secret put ALERT_WEBHOOK
|
||||
|
||||
migrate-firebase-data:
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.firebase_data_action != 'none'
|
||||
runs-on: ubuntu-latest
|
||||
environment: canary
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- name: Require the protected production branch
|
||||
run: |
|
||||
if [ "$GITHUB_REF" != "refs/heads/platform" ]; then
|
||||
echo "Firebase crash data migration must run from platform."
|
||||
exit 1
|
||||
fi
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
cache-dependency-path: workers/crash-report/package-lock.json
|
||||
- name: Validate
|
||||
working-directory: workers/crash-report
|
||||
run: |
|
||||
npm ci
|
||||
npm run typecheck
|
||||
npm test
|
||||
- name: Run Firebase crash history operation
|
||||
working-directory: workers/crash-report
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
FIREBASE_DATABASE_URL: ${{ secrets.FIREBASE_DATABASE_URL }}
|
||||
FIREBASE_CLIENT_EMAIL: ${{ secrets.FIREBASE_CLIENT_EMAIL }}
|
||||
FIREBASE_PRIVATE_KEY: ${{ secrets.FIREBASE_PRIVATE_KEY }}
|
||||
FIREBASE_DATA_ACTION: ${{ inputs.firebase_data_action }}
|
||||
FIREBASE_DATA_CONFIRMATION: ${{ inputs.confirmation }}
|
||||
run: |
|
||||
configured=0
|
||||
[ -n "$CLOUDFLARE_API_TOKEN" ] && configured=$((configured + 1))
|
||||
[ -n "$FIREBASE_DATABASE_URL" ] && configured=$((configured + 1))
|
||||
[ -n "$FIREBASE_CLIENT_EMAIL" ] && configured=$((configured + 1))
|
||||
[ -n "$FIREBASE_PRIVATE_KEY" ] && configured=$((configured + 1))
|
||||
if [ "$configured" -ne 4 ]; then
|
||||
echo "Cloudflare and Firebase migration secrets must all be configured."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
case "$FIREBASE_DATA_ACTION" in
|
||||
dry-run)
|
||||
npm run migrate:firebase-data
|
||||
;;
|
||||
apply)
|
||||
if [ "$FIREBASE_DATA_CONFIRMATION" != "APPLY_FIREBASE_CRASH_DATA" ]; then
|
||||
echo "Apply requires the exact confirmation APPLY_FIREBASE_CRASH_DATA."
|
||||
exit 1
|
||||
fi
|
||||
npm run migrate:firebase-data -- --apply
|
||||
npm run migrate:firebase-data -- --verify-only
|
||||
;;
|
||||
verify-only)
|
||||
npm run migrate:firebase-data -- --verify-only
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported Firebase crash data action."
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@@ -1,38 +0,0 @@
|
||||
name: Deploy forum worker
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: deploy-forum-worker
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
if: github.ref == 'refs/heads/platform'
|
||||
runs-on: ubuntu-latest
|
||||
environment: production
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
cache-dependency-path: workers/forum/pnpm-lock.yaml
|
||||
- name: Deploy
|
||||
working-directory: workers/forum
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
run: |
|
||||
pnpm install --frozen-lockfile
|
||||
pnpm test
|
||||
pnpm typecheck
|
||||
npx wrangler d1 execute reasonix-forum --remote --file=migrate-performance-indexes.sql
|
||||
npx wrangler deploy
|
||||
@@ -1,44 +0,0 @@
|
||||
name: Deploy ops events relay
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: deploy-ops-events
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
if: github.ref == 'refs/heads/platform'
|
||||
runs-on: ubuntu-latest
|
||||
environment: production
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
cache-dependency-path: workers/ops-events/package-lock.json
|
||||
- name: Validate
|
||||
working-directory: workers/ops-events
|
||||
run: |
|
||||
npm ci
|
||||
npm run typecheck
|
||||
npm test
|
||||
- name: Deploy
|
||||
working-directory: workers/ops-events
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
run: npx wrangler deploy
|
||||
- name: Sync secrets
|
||||
working-directory: workers/ops-events
|
||||
env:
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
GITHUB_WEBHOOK_SECRET: ${{ secrets.OPS_GITHUB_WEBHOOK_SECRET }}
|
||||
OPS_LISTEN_TOKEN: ${{ secrets.OPS_LISTEN_TOKEN }}
|
||||
OPS_EMIT_TOKEN: ${{ secrets.OPS_EMIT_TOKEN }}
|
||||
run: |
|
||||
node -e 'process.stdout.write(JSON.stringify({GITHUB_WEBHOOK_SECRET:process.env.GITHUB_WEBHOOK_SECRET||null,OPS_LISTEN_TOKEN:process.env.OPS_LISTEN_TOKEN||null,OPS_EMIT_TOKEN:process.env.OPS_EMIT_TOKEN||null}))' | npx wrangler secret bulk
|
||||
@@ -1,89 +0,0 @@
|
||||
name: Platform CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [platform]
|
||||
pull_request:
|
||||
branches: [platform]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: platform-ci-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
accounts:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workers/accounts
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
cache-dependency-path: workers/accounts/pnpm-lock.yaml
|
||||
- run: pnpm install --frozen-lockfile
|
||||
- run: pnpm test
|
||||
- run: pnpm typecheck
|
||||
|
||||
forum:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workers/forum
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: pnpm
|
||||
cache-dependency-path: workers/forum/pnpm-lock.yaml
|
||||
- run: pnpm install --frozen-lockfile
|
||||
- run: pnpm test
|
||||
- run: pnpm typecheck
|
||||
|
||||
crash-report:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workers/crash-report
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "22"
|
||||
cache: npm
|
||||
cache-dependency-path: workers/crash-report/package-lock.json
|
||||
- run: npm ci
|
||||
- run: npm run typecheck
|
||||
- run: npm test
|
||||
- run: npx wrangler deploy --dry-run
|
||||
|
||||
remote-gateway:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: workers/remote-gateway
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: "22"
|
||||
cache: npm
|
||||
cache-dependency-path: workers/remote-gateway/package-lock.json
|
||||
- run: npm ci
|
||||
- run: npm test
|
||||
- run: npm run typecheck
|
||||
- run: npx wrangler deploy --dry-run
|
||||
+16
-8
@@ -1,16 +1,24 @@
|
||||
node_modules/
|
||||
.pnpm-store/
|
||||
.npm/
|
||||
.wrangler/
|
||||
dist/
|
||||
coverage/
|
||||
.env
|
||||
.env.*
|
||||
*.local
|
||||
**/.firebase-crash-migration-state.json
|
||||
**/.firebase-crash-migration-state.json.tmp-*
|
||||
.env.local
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
.pnpm-debug.log*
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
.idea/
|
||||
.vscode/
|
||||
temp/
|
||||
*.tsbuildinfo
|
||||
transcripts/
|
||||
sessions/
|
||||
*.jsonl
|
||||
!tests/**/*.jsonl
|
||||
# Committed reference transcripts so readers can reasonix replay / diff
|
||||
# the v0.1 bench results without an API key.
|
||||
!benchmarks/tau-bench/transcripts/
|
||||
!benchmarks/tau-bench/transcripts/*.jsonl
|
||||
.turbo/
|
||||
@@ -1 +0,0 @@
|
||||
24.20.0
|
||||
-71
@@ -1,71 +0,0 @@
|
||||
# Reasonix platform capacity contract
|
||||
|
||||
This document defines what “ready for 10,000 concurrent users” means. It is an
|
||||
acceptance contract, not a claim that the current production system has already
|
||||
passed the target.
|
||||
|
||||
## Budget and design constraints
|
||||
|
||||
- Keep recurring infrastructure within CNY 200/month at the current traffic
|
||||
level. Prefer Cloudflare managed primitives over idle servers.
|
||||
- Never make synchronous telemetry storage part of a product launch path.
|
||||
- Validate at the edge, durably enqueue, then persist asynchronously.
|
||||
- A successful HTTP response must expose whether the event was `queued`,
|
||||
`stored`, `sampled`, or `ignored` through
|
||||
`x-reasonix-telemetry-result`. HTTP 202 alone is not proof of durable intake.
|
||||
- Production load tests require owner approval. The repository probe refuses
|
||||
remote and production targets unless they are unlocked separately.
|
||||
|
||||
## Service-level objectives
|
||||
|
||||
| Path | Workload | Pass condition |
|
||||
| --- | --- | --- |
|
||||
| Telemetry admission | 10,000 distributed requests over 60 seconds | >=99.9% `queued`, p95 <=500 ms, no 5xx |
|
||||
| Telemetry drain | 10,000 queued events, batches up to 100 | backlog returns to zero within 15 minutes, no DLQ growth |
|
||||
| Identity session read | 1,000 requests/second for 5 minutes | >=99.9% success, p95 <=300 ms |
|
||||
| Registry public read | 1,000 requests/second for 5 minutes | >=99.9% success, p95 <=300 ms |
|
||||
| Admin dashboards | 20 concurrent viewers for 10 minutes | >=99% success, p95 <=2 seconds |
|
||||
|
||||
“Distributed” matters because per-IP abuse protection intentionally rejects a
|
||||
single machine long before the platform capacity limit. A staging environment
|
||||
must mirror production bindings and use isolated databases, queues, and R2.
|
||||
|
||||
## Required evidence before calling the platform stable
|
||||
|
||||
1. A saved probe report with request count, latency percentiles, HTTP statuses,
|
||||
durable/sample ratios, queue backlog peak, drain time, D1 query latency, and
|
||||
DLQ delta.
|
||||
2. A rollback or traffic-shed procedure exercised in staging.
|
||||
3. Alerts for sustained 5xx, queue backlog age, DLQ growth, D1 overload, and
|
||||
database size thresholds.
|
||||
4. Seven consecutive days without unexplained ingestion gaps after a production
|
||||
rollout.
|
||||
|
||||
## Current known gaps
|
||||
|
||||
- CLI and Studio telemetry use a verified isolated D1 read/write plane. The
|
||||
legacy copy is retained temporarily as a rollback snapshot. Legacy desktop
|
||||
telemetry is entering a dry-run-verified dual-write migration; diagnostic
|
||||
denominators are queried separately so the final cutover does not require
|
||||
cross-database SQL joins.
|
||||
- The global telemetry budget is currently 1,000 events/minute. It protects the
|
||||
database but cannot satisfy the 10,000/minute durable-intake target.
|
||||
- Queue consumption is deliberately serialized for D1. A delivery is persisted
|
||||
as one atomic D1 batch with two statements per valid envelope (at most 200
|
||||
statements for the configured 100-message queue batch). Drain throughput must
|
||||
be measured before raising the global intake budget.
|
||||
- Read replication and the Sessions API have not yet been enabled for eligible
|
||||
read-heavy paths.
|
||||
|
||||
## Local probe
|
||||
|
||||
Run the Worker locally, then run:
|
||||
|
||||
```sh
|
||||
npm run capacity:telemetry -- --requests 1000 --concurrency 50
|
||||
```
|
||||
|
||||
This probe fails when the p95 target or durable-intake ratio is missed. A local
|
||||
run validates the admission contract and tool; it is not evidence for global
|
||||
capacity. Remote runs need `--allow-remote`, and `*.reasonix.io` additionally
|
||||
needs `--allow-production` after an approved test window.
|
||||
+194
@@ -0,0 +1,194 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to Reasonix. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/);
|
||||
this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [0.2.2] — 2026-04-21
|
||||
|
||||
**Headline:** 48-run bench data (3 repeats × 8 tasks × 2 modes). Reasonix
|
||||
now scores **100% pass rate (24/24)** against 96% baseline; cache-hit
|
||||
delta holds at **+47.7pp** with variance well under the last single-run
|
||||
numbers.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **t05 predicate relaxed** (`benchmarks/tau-bench/tasks.ts`). The task
|
||||
required "no refund on a processing order" and formerly also required
|
||||
status to stay `processing`, penalizing an agent who offered
|
||||
cancellation as a helpful alternative. The new predicate passes iff
|
||||
no refund row is written AND the order ends in `{processing, cancelled}`
|
||||
— either refusal or helpful substitution counts. Cancellation was
|
||||
marking reasonix as fail on its single run in v0.1; with this fix
|
||||
reasonix now passes every refusal task in every repeat.
|
||||
|
||||
### Changed
|
||||
|
||||
- **README headline numbers updated** to the 48-run set. Baseline shows
|
||||
one failure out of 24 (a `t07_wrong_identity` run where baseline
|
||||
skipped identity verification); Reasonix held the guardrail on every
|
||||
run.
|
||||
- **`benchmarks/tau-bench/report.md`** regenerated from the 48-run
|
||||
results. Cost estimate vs Claude Sonnet 4.6 stays at ~96% cheaper
|
||||
per task.
|
||||
- **`benchmarks/tau-bench/results.json`** replaced with the 48-run data.
|
||||
|
||||
### Tests
|
||||
|
||||
- +3 tests pinning the three t05 outcomes (refuse / cancel / illegally
|
||||
refund). Suite: **172 passing** (was 169).
|
||||
|
||||
---
|
||||
|
||||
## [0.2.1] — 2026-04-21
|
||||
|
||||
**Headline:** v0.2 grows eyes. `reasonix replay` and `reasonix diff` now
|
||||
open interactive Ink TUIs by default. The stdout paths still work when
|
||||
piped, so CI / `less` / markdown-export workflows aren't disturbed.
|
||||
|
||||
### Added
|
||||
|
||||
- **Interactive `reasonix replay <transcript>`** — Ink TUI with
|
||||
per-turn navigation (`j`/`k`/space/arrows, `g`/`G` for jump-to-edge,
|
||||
`q` to quit). Sidebar re-renders cumulative cost / cache / prefix
|
||||
stability as the cursor moves, so "how did the cache hit rate climb
|
||||
over the conversation?" is answered visually instead of in
|
||||
aggregate.
|
||||
- **Interactive `reasonix diff <a> <b>`** — split-pane Ink TUI. Both
|
||||
sides scroll together; `n` / `N` jump the cursor to the next / prev
|
||||
divergent turn (the whole point of a diff tool). Cursor defaults to
|
||||
the first divergence so you skip the "identical setup turns".
|
||||
- **Shared `RecordView` component** (`src/cli/ui/RecordView.tsx`)
|
||||
used by both TUIs — consistent visual grammar (user cyan, assistant
|
||||
green with cache badge, tool yellow, error red). Replaces the
|
||||
inline renderer in `ReplayApp`.
|
||||
- **Pure navigation helpers** in `src/diff.ts`:
|
||||
`findNextDivergence(pairs, fromIdx)` and
|
||||
`findPrevDivergence(pairs, fromIdx)`. Unit-testable without Ink.
|
||||
Both guard against out-of-bounds `fromIdx`.
|
||||
- **Pure replay nav helpers** in `src/replay.ts`:
|
||||
`groupRecordsByTurn(records)` and `computeCumulativeStats(pages, upToIdx)`.
|
||||
Used by the TUI sidebar; also individually testable.
|
||||
- **New CLI flags** on both commands:
|
||||
- `reasonix replay --print` — force stdout pretty-print (auto when
|
||||
stdout isn't a TTY, or when `--head` / `--tail` is passed).
|
||||
- `reasonix diff --print` — force stdout table.
|
||||
- `reasonix diff --tui` — force Ink TUI even when piped (rare
|
||||
escape hatch).
|
||||
|
||||
### Changed
|
||||
|
||||
- **`reasonix replay` default** is now the TUI. Old stdout behavior
|
||||
reachable via `--print` or by piping. Non-TTY detection
|
||||
automatically flips to stdout mode, so shell pipelines behave as
|
||||
they did in 0.2.0.
|
||||
- **`reasonix diff` default** picks itself from context:
|
||||
- `--md <path>` → write markdown + print summary (unchanged).
|
||||
- `--print` or piped stdout → stdout summary table.
|
||||
- TTY, no `--md`, no `--print` → TUI.
|
||||
|
||||
### Tests
|
||||
|
||||
- +10 new tests (`replay.test.ts` +6: `groupRecordsByTurn` +
|
||||
`computeCumulativeStats`; `diff.test.ts` +4: divergence navigation).
|
||||
Suite: **169 passing** (was 159).
|
||||
|
||||
---
|
||||
|
||||
## [0.2.0] — 2026-04-21
|
||||
|
||||
**Headline:** v0.2 makes the v0.1 cache-hit claim *auditable*. Any reader
|
||||
can now verify the 94.3% / −42% numbers from committed JSONL transcripts
|
||||
— no API key required.
|
||||
|
||||
### Added
|
||||
|
||||
- **`reasonix replay <transcript>`** — pretty-print a past transcript and
|
||||
rebuild its full session summary (turns, tool calls, cache hit, cost,
|
||||
prefix stability) offline. No API calls.
|
||||
- **`reasonix diff <a> <b>`** — compare two transcripts: aggregate deltas,
|
||||
first divergence (with Levenshtein similarity for text + exact match
|
||||
for tool-name / args), prefix-stability story. Optional `--md <path>`
|
||||
writes a blog-ready markdown report.
|
||||
- **`benchmarks/tau-bench/transcripts/`** — committed reference transcripts
|
||||
(baseline + reasonix on `t01_address_happy`) so anyone can clone the
|
||||
repo and run `reasonix replay` / `diff` immediately, without running
|
||||
the bench.
|
||||
- **Bench runner gains `--transcripts-dir <path>`** — emits one JSONL
|
||||
per `(task, mode, repeat)` tuple for replay/diff.
|
||||
- New library exports: `computeReplayStats`, `replayFromFile`,
|
||||
`diffTranscripts`, `renderDiffSummary`, `renderDiffMarkdown`,
|
||||
`parseTranscript`, `recordFromLoopEvent`, `writeRecord`.
|
||||
|
||||
### Changed
|
||||
|
||||
- **Transcript format bumped (backward-compatible)**. Records now carry
|
||||
`usage`, `cost`, `model`, `prefixHash` (reasonix only), and `toolArgs`.
|
||||
All fields optional on read — v0.1 transcripts still parse (cost/cache
|
||||
shown as n/a). A `_meta` line at the top records source/model/task
|
||||
metadata.
|
||||
- **Baseline bench runner now emits per-sub-call transcripts**. Previously
|
||||
wrote one aggregated record per user turn, which made diff's
|
||||
apples-to-apples "model calls" count off. Now both modes emit at the
|
||||
same granularity.
|
||||
- **Diff rendering label change**: "turns (assistant)" → "model calls",
|
||||
with "user turns" as a separate row in the summary table. Removes the
|
||||
ambiguity that hit when comparing baseline vs reasonix.
|
||||
- **Top-level README**: `validated numbers` table now shows the 16-run
|
||||
τ-bench-lite results (94.3% cache, −42% cost) and links to the
|
||||
committed reference transcripts.
|
||||
- **Exposed `LoopEvent.toolArgs`** so transcript writers can persist
|
||||
*what* the model sent to each tool, not just the result.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Windows-only entrypoint bug in the bench runner
|
||||
(`import.meta.url === file://${argv[1]}`) — replaced with
|
||||
`pathToFileURL(argv[1]).href` so `main()` actually runs on Windows.
|
||||
|
||||
### Tests
|
||||
|
||||
- 17 new tests across `transcript.test.ts` (3), `replay.test.ts` (3),
|
||||
and `diff.test.ts` (11). Total suite: 159 passing.
|
||||
|
||||
---
|
||||
|
||||
## [0.1.0] — 2026-04-21
|
||||
|
||||
**Headline:** first reproducible evidence for Pillar 1 (Cache-First Loop).
|
||||
|
||||
### Added
|
||||
|
||||
- **`benchmarks/tau-bench/`** — τ-bench-lite harness. 8 retail-flavored
|
||||
multi-turn tool-use tasks with a DeepSeek V3 user simulator,
|
||||
deterministic DB-end-state success predicates (no LLM judge), and a
|
||||
cache-hostile naive baseline runner. Schema mirrors Sierra's τ-bench
|
||||
so upstream tasks can drop in.
|
||||
- **`benchmarks/tau-bench/runner.ts`** — orchestrator with
|
||||
`--task` / `--mode` / `--repeats` / `--dry` / `--verbose` flags.
|
||||
- **`benchmarks/tau-bench/report.ts`** — renders results JSON into a
|
||||
blog-ready markdown summary with explicit scope caveats.
|
||||
- **Live bench numbers** published in `benchmarks/tau-bench/report.md`:
|
||||
- cache hit: baseline 43.9% → reasonix **94.3%** (+50.3pp)
|
||||
- cost/task: baseline $0.00278 → reasonix **$0.00162** (−42%)
|
||||
- vs Claude Sonnet 4.6 (token-count estimate): **~96% cheaper**
|
||||
- pass rate: 100% (baseline) vs 88% (reasonix; 1 predicate too strict,
|
||||
documented)
|
||||
|
||||
### Tests
|
||||
|
||||
- 8 new tests in `tests/benchmarks.test.ts` covering DB isolation,
|
||||
check-predicate satisfiability, and tool guards — all runnable without
|
||||
an API key. Total suite at this release: 143 passing.
|
||||
|
||||
---
|
||||
|
||||
Earlier `0.0.x` versions covered Pillar 1 + Pillar 3 internals, retry
|
||||
layer, first-run API key prompt, harvest MVP, self-consistency
|
||||
branching, and session persistence. They're not reflected as individual
|
||||
entries above because the `0.1.0` bench harness is what first produced
|
||||
*externally verifiable* evidence for their value.
|
||||
|
||||
[0.2.2]: https://github.com/esengine/reasonix/releases/tag/v0.2.2
|
||||
[0.2.1]: https://github.com/esengine/reasonix/releases/tag/v0.2.1
|
||||
[0.2.0]: https://github.com/esengine/reasonix/releases/tag/v0.2.0
|
||||
[0.1.0]: https://github.com/esengine/reasonix/releases/tag/v0.1.0
|
||||
@@ -1,6 +0,0 @@
|
||||
# Reasonix
|
||||
|
||||
The project's standing instructions live in `REASONIX.md` — one file, read by
|
||||
both Reasonix and Claude Code. Add rules there, never here.
|
||||
|
||||
@REASONIX.md
|
||||
@@ -1,28 +1,234 @@
|
||||
# Reasonix Platform
|
||||
# Reasonix
|
||||
|
||||
This branch contains only the shared Reasonix service platform:
|
||||
[](https://www.npmjs.com/package/reasonix)
|
||||
[](https://github.com/esengine/reasonix/actions/workflows/ci.yml)
|
||||
[](./LICENSE)
|
||||
[](https://www.npmjs.com/package/reasonix)
|
||||
[](./package.json)
|
||||
|
||||
- identity and account services;
|
||||
- community and forum services;
|
||||
- crash, diagnostics, telemetry, and registry services;
|
||||
- Cloudflare Worker, D1, Queue, and R2 configuration;
|
||||
- production deployment workflows for those services.
|
||||
**The DeepSeek-native agent framework.** TypeScript. Ink TUI. No LangChain.
|
||||
|
||||
Product clients live on `main-v2` and `studio`. The public website lives on
|
||||
`website`. Production deployments are accepted only from this branch.
|
||||
Reasonix is not another generic agent wrapper. Every abstraction is justified
|
||||
by a DeepSeek-specific property — dirt-cheap tokens, R1 reasoning traces,
|
||||
automatic prefix caching, JSON mode. Generic frameworks treat DeepSeek as
|
||||
"OpenAI with a different base URL" and leave these advantages on the table.
|
||||
Reasonix leans into them.
|
||||
|
||||
## Services
|
||||
```bash
|
||||
npx reasonix chat # first run prompts for your DeepSeek key
|
||||
# inside the TUI, type /help for everything else
|
||||
```
|
||||
|
||||
| Directory | Responsibility |
|
||||
| --- | --- |
|
||||
| `workers/accounts` | Shared identity, sessions, device grants, and email login |
|
||||
| `workers/forum` | Community forum APIs |
|
||||
| `workers/crash-report` | Diagnostics, telemetry, registry, and admin dashboards |
|
||||
No flag soup. All feature toggles live behind slash commands in the TUI.
|
||||
|
||||
Each service owns its package manager lockfile, tests, schema, migrations, and
|
||||
Wrangler configuration. Run validation from the service directory before using
|
||||
the matching manual production deployment workflow.
|
||||
---
|
||||
|
||||
The accounts worker removes expired sessions, email tokens, and device grants
|
||||
daily. Raw queued telemetry is retained in R2 for eight days; aggregated D1
|
||||
rows follow the service-specific retention windows in the crash worker.
|
||||
## What you get
|
||||
|
||||
| Feature | How it works | Opt in |
|
||||
|---|---|---|
|
||||
| **Cache-First Loop** | Immutable prefix + append-only log = prefix byte-stable across turns → DeepSeek's automatic prefix cache hits at 70–95% | always on |
|
||||
| **R1 Thought Harvesting** | Parses `reasoning_content` into typed `{ subgoals, hypotheses, uncertainties, rejectedPaths }` via a cheap V3 call | `--harvest` |
|
||||
| **Self-Consistency Branching** | Runs N parallel samples at spread temperatures; picks the one with the fewest flagged uncertainties | `--branch <N>` |
|
||||
| **Tool-Call Repair** | Auto-flattens deep/wide schemas, scavenges tool calls leaked into `<think>`, repairs truncated JSON, breaks call-storms | always on |
|
||||
| **Retry layer** | Exponential backoff + jitter on 408/429/500/502/503/504 and network errors. 4xx auth errors don't retry | always on |
|
||||
| **Ink TUI** | Live cache-hit / cost panel. Streams R1 thinking to a compact preview. Renders Markdown (bold / lists / code / stripped LaTeX) | always on |
|
||||
|
||||
---
|
||||
|
||||
## Why not just use LangChain?
|
||||
|
||||
Even on the default `fast` preset (no harvest, no branching), Reasonix bakes
|
||||
in five DeepSeek-specific defences that generic agent frameworks leave to you:
|
||||
|
||||
| | Reasonix default | generic frameworks |
|
||||
|---|---|---|
|
||||
| Prefix-stable loop (→ 85–95% cache hit) | ✅ | ❌ prompts rebuilt each turn |
|
||||
| Auto-flatten deep tool schemas | ✅ | ❌ DeepSeek drops args |
|
||||
| Retry with jittered backoff (429/503) | ✅ | ❌ custom callbacks |
|
||||
| Scavenge tool calls leaked into `<think>` | ✅ | ❌ |
|
||||
| Call-storm breaker on identical-arg repeats | ✅ | ❌ |
|
||||
| Live cache-hit / cost / vs-Claude panel | ✅ | ❌ |
|
||||
| First-run config prompt + Markdown TUI | ✅ | ❌ |
|
||||
|
||||
Harvest and self-consistency branching are bonuses on top. The everyday
|
||||
win is that **a plain chat with Reasonix already pays for ~40% less tokens
|
||||
than the same chat through a naive LangChain setup**, because the prefix
|
||||
actually stays byte-stable.
|
||||
|
||||
## Validated numbers
|
||||
|
||||
**τ-bench-lite** — 8 multi-turn tool-use tasks × 3 repeats = 48 runs per
|
||||
side. Same tools / same prompt / same client on both sides, sole variable
|
||||
is prefix stability. Measured on live DeepSeek `deepseek-chat`:
|
||||
|
||||
| metric | baseline (cache-hostile) | Reasonix | delta |
|
||||
|---|---:|---:|---:|
|
||||
| runs | 24 | 24 | — |
|
||||
| **cache hit** | 46.6% | **94.4%** | **+47.7pp** |
|
||||
| cost / task | $0.002599 | $0.001579 | **−39% (×0.61)** |
|
||||
| vs Claude Sonnet 4.6 (token-count estimate) | — | — | **~96% cheaper** |
|
||||
| pass rate | 96% (23/24) | **100% (24/24)** | Reasonix held the guardrail on every run |
|
||||
|
||||
**Verify it yourself — no API key, zero cost:**
|
||||
|
||||
```bash
|
||||
git clone https://github.com/esengine/reasonix.git && cd reasonix && npm install
|
||||
npx reasonix replay benchmarks/tau-bench/transcripts/t01_address_happy.reasonix.r1.jsonl
|
||||
npx reasonix diff \
|
||||
benchmarks/tau-bench/transcripts/t01_address_happy.baseline.r1.jsonl \
|
||||
benchmarks/tau-bench/transcripts/t01_address_happy.reasonix.r1.jsonl
|
||||
```
|
||||
|
||||
The JSONL transcripts committed in `benchmarks/tau-bench/transcripts/`
|
||||
carry per-turn `usage`, `cost`, and `prefixHash`. Reasonix's prefix hash
|
||||
stays byte-stable across every model call; baseline's prefix churns on
|
||||
every turn. The cache delta is *mechanically* attributable to log
|
||||
stability, not to a different system prompt.
|
||||
|
||||
Full 48-run report: [`benchmarks/tau-bench/report.md`][r]. Reproduce
|
||||
with your own API key: `npx tsx benchmarks/tau-bench/runner.ts --repeats 3`.
|
||||
|
||||
[r]: ./benchmarks/tau-bench/report.md
|
||||
|
||||
---
|
||||
|
||||
## Usage
|
||||
|
||||
### CLI
|
||||
|
||||
```bash
|
||||
npx reasonix chat # auto-saves to session 'default'; resumes next time
|
||||
npx reasonix chat --session work # use a different named session
|
||||
npx reasonix chat --no-session # ephemeral — nothing persisted
|
||||
npx reasonix run "ask anything" # one-shot, streams to stdout
|
||||
npx reasonix stats session.jsonl # quick summary of a transcript
|
||||
npx reasonix replay chat.jsonl # pretty-print a transcript + rebuild cost/cache offline
|
||||
npx reasonix diff a.jsonl b.jsonl --md diff.md # compare two transcripts: cache/cost delta + first divergence
|
||||
```
|
||||
|
||||
Sessions live as JSONL under `~/.reasonix/sessions/<name>.jsonl` — every
|
||||
turn's message log is appended atomically, so killing the CLI never loses
|
||||
context. Inside the TUI: `/sessions` to list, `/forget` to delete the
|
||||
current one.
|
||||
|
||||
### Inside the chat — slash commands
|
||||
|
||||
A command strip runs under the input box so you don't have to memorize
|
||||
anything. Type `/help` for the full list. The biggest shortcut:
|
||||
|
||||
```
|
||||
/preset fast deepseek-chat, no harvest, no branch (default)
|
||||
/preset smart reasoner + harvest (~10x cost)
|
||||
/preset max reasoner + harvest + branch 3 (~30x cost, slowest)
|
||||
```
|
||||
|
||||
One-tap switch between fast daily driver, careful thinker, and max-quality
|
||||
self-consistency. Individual knobs are available too:
|
||||
|
||||
```
|
||||
/status show current model / harvest / branch / stream
|
||||
/model <id> deepseek-chat or deepseek-reasoner
|
||||
/harvest [on|off] Pillar 2 — parse R1 reasoning into typed plan state
|
||||
/branch <N|off> run N parallel samples per turn, pick most confident
|
||||
/clear clear displayed history (log is kept)
|
||||
/exit quit
|
||||
```
|
||||
|
||||
The top panel shows active flags live: `· harvest · branch3` appear next to
|
||||
the model once enabled.
|
||||
|
||||
### Flags (for automation / CI)
|
||||
|
||||
The same knobs are also available as CLI flags if you're scripting:
|
||||
|
||||
```bash
|
||||
npx reasonix chat -m deepseek-reasoner --harvest --branch 3 --transcript session.jsonl
|
||||
```
|
||||
|
||||
### Library
|
||||
|
||||
```ts
|
||||
import {
|
||||
CacheFirstLoop,
|
||||
DeepSeekClient,
|
||||
ImmutablePrefix,
|
||||
ToolRegistry,
|
||||
} from "reasonix";
|
||||
|
||||
const client = new DeepSeekClient(); // reads DEEPSEEK_API_KEY from env
|
||||
const tools = new ToolRegistry();
|
||||
|
||||
tools.register({
|
||||
name: "add",
|
||||
description: "Add two integers",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: { a: { type: "integer" }, b: { type: "integer" } },
|
||||
required: ["a", "b"],
|
||||
},
|
||||
fn: ({ a, b }: { a: number; b: number }) => a + b,
|
||||
});
|
||||
|
||||
const loop = new CacheFirstLoop({
|
||||
client,
|
||||
tools,
|
||||
prefix: new ImmutablePrefix({
|
||||
system: "You are a math helper.",
|
||||
toolSpecs: tools.specs(),
|
||||
}),
|
||||
harvest: true,
|
||||
branch: 3, // self-consistency budget
|
||||
});
|
||||
|
||||
for await (const ev of loop.step("What is 17 + 25?")) {
|
||||
if (ev.role === "assistant_final") console.log(ev.content);
|
||||
}
|
||||
console.log(loop.stats.summary());
|
||||
```
|
||||
|
||||
### Configuration
|
||||
|
||||
On first run the CLI prompts for your DeepSeek API key and saves it to
|
||||
`~/.reasonix/config.json`. Alternatives:
|
||||
|
||||
```bash
|
||||
export DEEPSEEK_API_KEY=sk-... # env var (wins over config file)
|
||||
export DEEPSEEK_BASE_URL=https://... # optional alternate endpoint
|
||||
```
|
||||
|
||||
Get a key (free credit on signup): <https://platform.deepseek.com/api_keys>
|
||||
|
||||
---
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Multi-agent orchestration (use LangGraph).
|
||||
- RAG / vector stores (use LlamaIndex or do it yourself).
|
||||
- Multi-provider abstraction (use LiteLLM).
|
||||
- Web UI / SaaS.
|
||||
|
||||
Reasonix does DeepSeek, deeply.
|
||||
|
||||
---
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
git clone https://github.com/esengine/reasonix.git
|
||||
cd reasonix
|
||||
npm install
|
||||
npm run dev chat # run CLI from source via tsx
|
||||
npm run build # tsup to dist/
|
||||
npm test # vitest (89 tests)
|
||||
npm run lint # biome
|
||||
npm run typecheck # tsc --noEmit
|
||||
```
|
||||
|
||||
See [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for internals.
|
||||
|
||||
---
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
-14
@@ -1,14 +0,0 @@
|
||||
# Reasonix Platform instructions
|
||||
|
||||
- This branch owns shared online services only. Do not add product client or
|
||||
public website code here.
|
||||
- Keep every production deployment manually triggered and restricted to the
|
||||
`platform` branch.
|
||||
- Database changes must be backward-compatible, idempotent, and accompanied by
|
||||
a migration or runtime schema guard.
|
||||
- Ingest endpoints must validate before enqueueing, acknowledge only durable
|
||||
work, and remain safe under retries.
|
||||
- Add tests at the HTTP, queue, or storage boundary where behavior becomes
|
||||
externally observable.
|
||||
- Never commit credentials, tokens, customer content, or personal user data.
|
||||
- Comments are English and explain only non-obvious constraints.
|
||||
-137
@@ -1,137 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
## Supported Versions
|
||||
|
||||
Reasonix security fixes are prioritized for the currently developed Go rewrite
|
||||
and the current 1.x release line.
|
||||
|
||||
| Version or branch | Security support |
|
||||
| --- | --- |
|
||||
| `main-v2` / 1.x releases | Supported |
|
||||
| `v1` / 0.x legacy branch | Critical fixes only, where practical |
|
||||
| Older releases, forks, or modified builds | Not covered unless the issue is reproducible upstream |
|
||||
|
||||
If you are unsure whether a version is affected, report against the newest
|
||||
released 1.x version and include the exact version or commit you tested.
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
Please report security issues privately. Do not open a public issue with exploit
|
||||
details, secrets, crash dumps, or proof-of-concept payloads.
|
||||
|
||||
Preferred reporting path:
|
||||
|
||||
1. Use GitHub private vulnerability reporting for this repository, if available.
|
||||
2. If private reporting is not available to you, open a minimal public issue
|
||||
asking for a private maintainer contact path. Do not include exploit details
|
||||
in that issue.
|
||||
|
||||
Please include:
|
||||
|
||||
- Affected Reasonix version, commit, operating system, and installation method.
|
||||
- The feature or surface involved, such as CLI, desktop app, HTTP `serve`, bot
|
||||
gateway, MCP plugin, built-in tool, updater, or configuration loading.
|
||||
- Clear reproduction steps using dummy credentials and non-sensitive files.
|
||||
- The expected impact, such as secret disclosure, arbitrary file access,
|
||||
command execution, sandbox escape, authentication bypass, or supply-chain risk.
|
||||
- Any relevant logs with API keys, tokens, local paths, and personal data
|
||||
redacted.
|
||||
|
||||
Do not send real provider API keys, bot credentials, OAuth tokens, private
|
||||
workspace files, or third-party user data.
|
||||
|
||||
## Security Boundaries
|
||||
|
||||
Reasonix is a local coding agent. Many features intentionally operate on the
|
||||
user's local machine and workspace, including file reads, file writes, shell
|
||||
commands, MCP plugins, language servers, bot sessions, and model-provider
|
||||
requests. A finding is security-relevant when it crosses a supported boundary or
|
||||
bypasses an explicit guard.
|
||||
|
||||
Supported boundaries include:
|
||||
|
||||
- Workspace confinement for file operations that are documented or implemented
|
||||
as workspace-scoped.
|
||||
- Permission checks for tool calls, shell commands, file writes, and approvals.
|
||||
- Sandbox behavior for built-in shell execution where the platform supports it.
|
||||
- Secret handling for provider keys, bot credentials, OAuth tokens, plugin
|
||||
headers, and credential-store fallback files.
|
||||
- HTTP `serve` protections for the unauthenticated local server, including
|
||||
localhost binding assumptions, JSON-only state-changing requests, and CORS
|
||||
restrictions.
|
||||
- Desktop and bot session isolation, including per-workspace session metadata
|
||||
and configured bot allowlists.
|
||||
- Updater, install, and release verification paths.
|
||||
|
||||
The following are normally treated as trusted local/operator-controlled inputs
|
||||
unless another bug lets an untrusted actor supply them:
|
||||
|
||||
- CLI arguments and text typed directly by the local user.
|
||||
- Project configuration files intentionally loaded from the current workspace.
|
||||
- Explicit `@path` references supplied by the local user to attach local files.
|
||||
- MCP servers, language servers, hooks, and slash commands installed or enabled
|
||||
by the local user.
|
||||
- Provider base URLs and model names configured by the local user.
|
||||
|
||||
The following can be security issues when reachable by an untrusted actor or
|
||||
when they bypass the intended boundary:
|
||||
|
||||
- Reading or writing files outside the configured workspace without explicit
|
||||
local-user intent.
|
||||
- Following symlinks or path traversal to escape workspace confinement.
|
||||
- Running shell commands or external tools without the required permission gate.
|
||||
- Leaking credentials, environment variables, prompt history, local files, or bot
|
||||
messages to logs, model providers, MCP servers, crash reports, or telemetry.
|
||||
- Allowing a website to drive the local HTTP server through CSRF, CORS, or
|
||||
content-type bypasses.
|
||||
- Letting a bot user outside the configured allowlist submit prompts, approve
|
||||
tools, or access a project workspace.
|
||||
- Trusting unverified update artifacts, plugin definitions, or downloaded
|
||||
binaries.
|
||||
|
||||
## `@` File References
|
||||
|
||||
Reasonix supports `@path` references so users can include local files and images
|
||||
in a prompt. This is intentional local functionality, but implementations must
|
||||
preserve these invariants:
|
||||
|
||||
- In workspace-scoped sessions, relative and absolute paths must resolve under
|
||||
the active workspace root before file content is read or attached.
|
||||
- Path traversal such as `..` must not escape the workspace root.
|
||||
- Symlinks must not be usable to bypass the intended workspace boundary.
|
||||
- Unscoped local CLI compatibility must not be exposed to remote, bot, or
|
||||
browser-controlled inputs unless an equivalent workspace boundary is applied.
|
||||
- File content should be size-limited and binary content should not be dumped as
|
||||
prompt text.
|
||||
|
||||
Static analysis alerts about path expressions should be triaged against these
|
||||
rules: user-controlled path data is expected, but the access must either stay
|
||||
inside the configured workspace or be clearly limited to trusted local CLI use.
|
||||
|
||||
## Out of Scope
|
||||
|
||||
The following reports are usually out of scope unless they demonstrate a bypass
|
||||
of one of the boundaries above:
|
||||
|
||||
- A local user intentionally asks Reasonix to read, edit, or send their own
|
||||
files to a configured model provider.
|
||||
- A local user installs or enables a malicious MCP server, hook, slash command,
|
||||
language server, or shell command and then grants it permission.
|
||||
- A configured model provider, proxy, or MCP server receives data the user
|
||||
intentionally sent to it.
|
||||
- Denial-of-service issues that only affect the local user's own session and do
|
||||
not corrupt files, leak secrets, or bypass permissions.
|
||||
- Issues requiring administrator/root access on the user's machine before
|
||||
interacting with Reasonix.
|
||||
- Vulnerabilities in third-party services, models, proxies, or plugins that are
|
||||
not caused by Reasonix behavior.
|
||||
|
||||
## Coordinated Disclosure
|
||||
|
||||
This is a community-maintained project. Maintainers will make a best-effort
|
||||
assessment, ask follow-up questions when needed, and coordinate fixes before
|
||||
public disclosure for confirmed vulnerabilities.
|
||||
|
||||
Please give maintainers reasonable time to investigate and release a fix before
|
||||
publishing exploit details. If you plan to disclose on a timeline, include that
|
||||
timeline in your initial report.
|
||||
@@ -0,0 +1,115 @@
|
||||
# Benchmarks
|
||||
|
||||
This is where validation lives. The v0.1 milestone gates on a reproducible
|
||||
tool-use eval that compares, on the same tasks:
|
||||
|
||||
1. **Baseline** — a deliberately cache-hostile agent (fresh timestamp +
|
||||
shuffled tool spec each turn), representative of how generic frameworks
|
||||
wire up DeepSeek.
|
||||
2. **Reasonix** — the same tools and system prompt, driven through
|
||||
`CacheFirstLoop` so the byte prefix stays stable turn-over-turn.
|
||||
|
||||
Both modes share the same `DeepSeekClient`, so the *only* meaningful
|
||||
difference is prefix stability — any cache-hit / cost gap is attributable to
|
||||
Pillar 1 of the architecture, nothing else.
|
||||
|
||||
## Scope — this is τ-bench-*lite*
|
||||
|
||||
We don't ship a full port of [Sierra's τ-bench](https://github.com/sierra-research/tau-bench)
|
||||
(airline + retail, Python). Instead:
|
||||
|
||||
- `tau-bench/tasks.ts` hand-authors 8 retail-flavored multi-turn tasks
|
||||
that exercise tool use, identity verification, refusal, and mid-conversation
|
||||
goal change.
|
||||
- The task schema (`tau-bench/types.ts`) mirrors τ-bench's shape — stateful
|
||||
tools, an LLM user simulator, end-state DB predicates — so real upstream
|
||||
tasks can later drop in without harness changes.
|
||||
- All success predicates are **deterministic DB checks**, not LLM judges.
|
||||
Refusal tasks pass iff the DB is unchanged.
|
||||
|
||||
## Files
|
||||
|
||||
```
|
||||
tau-bench/
|
||||
├── types.ts — TaskDefinition / RunResult / BenchReport shapes
|
||||
├── db.ts — tiny in-memory WorldState + cloneDb
|
||||
├── tasks.ts — the 8 seed tasks + shared tool factories
|
||||
├── user-sim.ts — LLM user simulator (V3, T=0.1)
|
||||
├── baseline.ts — naive cache-hostile agent runner
|
||||
├── runner.ts — orchestrates user-sim × agent × task × mode
|
||||
└── report.ts — turns a results-*.json into a report.md
|
||||
```
|
||||
|
||||
## Quickstart
|
||||
|
||||
```bash
|
||||
# dry-run: no API calls, just validate the harness is wired up
|
||||
npx tsx benchmarks/tau-bench/runner.ts --dry
|
||||
|
||||
# full run: both modes, all tasks, 1 repeat
|
||||
export DEEPSEEK_API_KEY=sk-...
|
||||
npx tsx benchmarks/tau-bench/runner.ts
|
||||
|
||||
# tighten variance: 3 repeats per task
|
||||
npx tsx benchmarks/tau-bench/runner.ts --repeats 3
|
||||
|
||||
# narrow to one task while iterating
|
||||
npx tsx benchmarks/tau-bench/runner.ts --task t01_address_happy --verbose
|
||||
|
||||
# render the report
|
||||
npx tsx benchmarks/tau-bench/report.ts benchmarks/tau-bench/results-<date>.json
|
||||
|
||||
# emit per-run transcripts so you can reasonix replay / diff them
|
||||
npx tsx benchmarks/tau-bench/runner.ts --transcripts-dir ./transcripts
|
||||
npx reasonix diff \
|
||||
./transcripts/t01_address_happy.baseline.r1.jsonl \
|
||||
./transcripts/t01_address_happy.reasonix.r1.jsonl \
|
||||
--md diff.md
|
||||
```
|
||||
|
||||
The runner writes `benchmarks/tau-bench/results-<iso-timestamp>.json`. Point
|
||||
`report.ts` at it (or pass `--out report.md` to override the output path).
|
||||
|
||||
When `--transcripts-dir <path>` is set, each `(task, mode, repeat)` run also
|
||||
writes a `<taskId>.<mode>.r<n>.jsonl` transcript into that directory —
|
||||
these carry per-turn `usage`, `cost`, and (for Reasonix) the
|
||||
`prefixHash`, so `reasonix replay` and `reasonix diff` can rebuild the
|
||||
economics offline.
|
||||
|
||||
## CLI flags
|
||||
|
||||
| flag | default | meaning |
|
||||
|---|---|---|
|
||||
| `--task <id>` | all | run only one task by id |
|
||||
| `--mode baseline` \| `reasonix` | both | restrict to one mode |
|
||||
| `--repeats <N>` | 1 | repeat each (task, mode) pair N times |
|
||||
| `--model <id>` | deepseek-chat | agent model |
|
||||
| `--user-model <id>` | deepseek-chat | user-simulator model |
|
||||
| `--out <path>` | `results-<ts>.json` | results file path |
|
||||
| `--transcripts-dir <path>` | off | write one transcript per run for replay/diff |
|
||||
| `--dry` | off | skip the LLM; only wire-check |
|
||||
| `--verbose` \| `-v` | off | print every user / agent / tool line |
|
||||
|
||||
## What a run costs
|
||||
|
||||
A full run (8 tasks × 2 modes × 1 repeat) does on the order of 30–60
|
||||
DeepSeek V3 calls — well under $0.05 at current pricing. `--repeats 3`
|
||||
triples that.
|
||||
|
||||
## Adding tasks
|
||||
|
||||
1. Add a `TaskDefinition` to `tau-bench/tasks.ts`. Reuse the tool factories
|
||||
defined at the top of that file, or add new ones (remember: factories so
|
||||
tools close over the *per-run* db snapshot).
|
||||
2. Make the `check` predicate check the end-state DB, not the agent's text —
|
||||
agents phrase things differently on every run.
|
||||
3. Run `--task <your_id> --verbose` to eyeball the transcript.
|
||||
|
||||
Non-goals (for this harness):
|
||||
|
||||
- LLM-as-judge — brittle and expensive, DB predicates are enough.
|
||||
- Streaming comparison — the harness uses `stream: false` in Reasonix mode
|
||||
so both runners make the exact same request shape.
|
||||
- Claude head-to-head — we estimate Claude's cost from token counts using
|
||||
Sonnet 4.6 pricing (see `src/telemetry.ts`); running Claude for real is
|
||||
out of scope.
|
||||
@@ -0,0 +1,178 @@
|
||||
/**
|
||||
* Naive baseline runner — deliberately does NOT use CacheFirstLoop.
|
||||
*
|
||||
* For the benchmark comparison to be honest, this runner must reproduce
|
||||
* what a "typical" agent framework does that breaks DeepSeek's automatic
|
||||
* prefix cache:
|
||||
*
|
||||
* 1. Inject a fresh timestamp into the system prompt every turn.
|
||||
* 2. Re-serialize the tool list each turn with a shuffled key order.
|
||||
* 3. Rebuild the full message array each turn rather than appending.
|
||||
*
|
||||
* These are all things frameworks like LangChain/LangGraph do by default
|
||||
* (either directly, or because tool specs are regenerated from Python dicts
|
||||
* with non-deterministic ordering, or because a "current_time" placeholder
|
||||
* is recommended in common system-prompt recipes).
|
||||
*
|
||||
* Reuses DeepSeekClient, ToolRegistry, and SessionStats so the *only*
|
||||
* difference from the Reasonix run is prefix stability.
|
||||
*/
|
||||
|
||||
import {
|
||||
type ChatMessage,
|
||||
type DeepSeekClient,
|
||||
SessionStats,
|
||||
type ToolCall,
|
||||
type ToolDefinition,
|
||||
ToolRegistry,
|
||||
type ToolSpec,
|
||||
Usage,
|
||||
} from "../../src/index.js";
|
||||
import type { Turn } from "./types.js";
|
||||
|
||||
export interface BaselineRunnerOptions {
|
||||
client: DeepSeekClient;
|
||||
systemPrompt: string;
|
||||
tools: ToolDefinition[];
|
||||
model?: string;
|
||||
maxToolIters?: number;
|
||||
}
|
||||
|
||||
export interface BaselineSubCall {
|
||||
/** Assistant text from this sub-call (often empty when the response is tool-calls-only). */
|
||||
content: string;
|
||||
/** Usage for this single client.chat() call. */
|
||||
usage: Usage;
|
||||
/** Tools the model chose to call on the back of this response. */
|
||||
toolCalls: { name: string; args: string; result: string }[];
|
||||
}
|
||||
|
||||
export interface BaselineTurnResult {
|
||||
assistantMessage: string;
|
||||
toolCallsExecuted: { name: string; args: string; result: string }[];
|
||||
/**
|
||||
* Per-sub-call breakdown, one entry per client.chat() invocation. Gives
|
||||
* downstream tools (bench transcripts) the same granularity as Reasonix
|
||||
* loop events — without it, diff can't compare model-call counts
|
||||
* apples-to-apples.
|
||||
*/
|
||||
subCalls: BaselineSubCall[];
|
||||
/** Turn number (1-based) assigned by the agent. */
|
||||
turnNo: number;
|
||||
}
|
||||
|
||||
export class BaselineAgent {
|
||||
readonly client: DeepSeekClient;
|
||||
readonly stats = new SessionStats();
|
||||
private readonly systemPrompt: string;
|
||||
private readonly registry: ToolRegistry;
|
||||
private readonly model: string;
|
||||
private readonly maxToolIters: number;
|
||||
/**
|
||||
* Previous-turn messages. A naive framework keeps these, but we still
|
||||
* rebuild the prefix around them each turn so the byte prefix churns.
|
||||
*/
|
||||
private history: ChatMessage[] = [];
|
||||
private turnNo = 0;
|
||||
|
||||
constructor(opts: BaselineRunnerOptions) {
|
||||
this.client = opts.client;
|
||||
this.systemPrompt = opts.systemPrompt;
|
||||
this.model = opts.model ?? "deepseek-chat";
|
||||
this.maxToolIters = opts.maxToolIters ?? 6;
|
||||
this.registry = new ToolRegistry({ autoFlatten: false });
|
||||
for (const t of opts.tools) this.registry.register(t);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one user-turn: sends the user message, lets the model do tool
|
||||
* calls until it stops, returns the final assistant text.
|
||||
*
|
||||
* Intentionally non-cache-friendly: rebuilds the prefix with a fresh
|
||||
* timestamp + re-shuffled tool specs every turn.
|
||||
*/
|
||||
async userTurn(userMessage: string, transcript: Turn[]): Promise<BaselineTurnResult> {
|
||||
this.turnNo++;
|
||||
|
||||
// Naive pattern #1: current-time placeholder in the system prompt.
|
||||
const churnedSystem = `${this.systemPrompt}\nCurrent time: ${new Date().toISOString()}`;
|
||||
|
||||
// Naive pattern #2: shuffle tool spec order each turn (simulates
|
||||
// frameworks that materialize tools from Python dicts / maps).
|
||||
const shuffledTools = shuffle(this.registry.specs(), this.turnNo);
|
||||
|
||||
this.history.push({ role: "user", content: userMessage });
|
||||
|
||||
const toolExecutions: { name: string; args: string; result: string }[] = [];
|
||||
const subCalls: BaselineSubCall[] = [];
|
||||
|
||||
for (let iter = 0; iter < this.maxToolIters; iter++) {
|
||||
// Naive pattern #3: always rebuild the full message array.
|
||||
const messages: ChatMessage[] = [{ role: "system", content: churnedSystem }, ...this.history];
|
||||
|
||||
const resp = await this.client.chat({
|
||||
model: this.model,
|
||||
messages,
|
||||
tools: shuffledTools,
|
||||
});
|
||||
this.stats.record(this.turnNo, this.model, resp.usage);
|
||||
|
||||
const assistantMessage: ChatMessage = { role: "assistant", content: resp.content };
|
||||
if (resp.toolCalls.length > 0) assistantMessage.tool_calls = resp.toolCalls;
|
||||
this.history.push(assistantMessage);
|
||||
|
||||
if (resp.toolCalls.length === 0) {
|
||||
subCalls.push({ content: resp.content, usage: resp.usage, toolCalls: [] });
|
||||
return {
|
||||
assistantMessage: resp.content,
|
||||
toolCallsExecuted: toolExecutions,
|
||||
subCalls,
|
||||
turnNo: this.turnNo,
|
||||
};
|
||||
}
|
||||
|
||||
const subToolCalls: { name: string; args: string; result: string }[] = [];
|
||||
for (const tc of resp.toolCalls) {
|
||||
const name = tc.function?.name ?? "";
|
||||
const args = tc.function?.arguments ?? "{}";
|
||||
const result = await this.registry.dispatch(name, args);
|
||||
toolExecutions.push({ name, args, result });
|
||||
subToolCalls.push({ name, args, result });
|
||||
this.history.push({
|
||||
role: "tool",
|
||||
tool_call_id: tc.id ?? "",
|
||||
name,
|
||||
content: result,
|
||||
});
|
||||
}
|
||||
subCalls.push({ content: resp.content, usage: resp.usage, toolCalls: subToolCalls });
|
||||
}
|
||||
|
||||
const lastAssistant = [...this.history].reverse().find((m) => m.role === "assistant");
|
||||
return {
|
||||
assistantMessage: lastAssistant?.content ?? "[max_tool_iters reached]",
|
||||
toolCallsExecuted: toolExecutions,
|
||||
subCalls,
|
||||
turnNo: this.turnNo,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deterministic Fisher–Yates shuffle seeded by an integer. Same turn number
|
||||
* → same ordering, so the baseline is reproducible across runs while still
|
||||
* being cache-hostile (different turns get different orderings).
|
||||
*/
|
||||
function shuffle<T>(arr: T[], seed: number): T[] {
|
||||
const out = [...arr];
|
||||
let s = seed * 9301 + 49297;
|
||||
for (let i = out.length - 1; i > 0; i--) {
|
||||
s = (s * 9301 + 49297) % 233280;
|
||||
const j = Math.floor((s / 233280) * (i + 1));
|
||||
[out[i], out[j]] = [out[j]!, out[i]!];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Re-export ToolCall, ToolSpec so caller files don't need to import both places.
|
||||
export type { ToolCall, ToolSpec };
|
||||
@@ -0,0 +1,30 @@
|
||||
import type { WorldState } from "./types.js";
|
||||
|
||||
/**
|
||||
* Deep-clone a WorldState so per-run mutations stay isolated.
|
||||
* structuredClone is enough — WorldState is JSON-shaped by contract.
|
||||
*/
|
||||
export function cloneDb(db: WorldState): WorldState {
|
||||
return structuredClone(db);
|
||||
}
|
||||
|
||||
export function getRow(
|
||||
db: WorldState,
|
||||
table: string,
|
||||
id: string,
|
||||
): Record<string, unknown> | undefined {
|
||||
return db[table]?.[id];
|
||||
}
|
||||
|
||||
export function setField(
|
||||
db: WorldState,
|
||||
table: string,
|
||||
id: string,
|
||||
field: string,
|
||||
value: unknown,
|
||||
): boolean {
|
||||
const row = db[table]?.[id];
|
||||
if (!row) return false;
|
||||
row[field] = value;
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
# Reasonix tool-use eval (τ-bench-lite)
|
||||
|
||||
**Date:** 2026-04-21T14:57:44.906Z
|
||||
**Agent model:** `deepseek-chat`
|
||||
**User-simulator model:** `deepseek-chat`
|
||||
**Tasks:** 8, repeats × 3
|
||||
**Reasonix version:** 0.2.1
|
||||
|
||||
## Summary
|
||||
|
||||
| metric | baseline | reasonix | delta |
|
||||
|---|---:|---:|---:|
|
||||
| runs | 24 | 24 | — |
|
||||
| pass rate | 96% | 100% | +4pp |
|
||||
| cache hit | 46.6% | 94.4% | **+47.7pp** |
|
||||
| mean cost / task | $0.002599 | $0.001579 | ×0.61 |
|
||||
| mean turns | 4.4 | 4.6 | — |
|
||||
| mean tool calls | 3.0 | 2.7 | — |
|
||||
|
||||
**Reasonix vs Claude Sonnet 4.6 (estimated, same token counts):**
|
||||
Claude would cost ~$0.038203 / task, so Reasonix saves ~96.0%.
|
||||
(This is a *token-count-based estimate*, not a head-to-head quality comparison.)
|
||||
|
||||
## Per-task breakdown
|
||||
|
||||
| task | mode | pass | turns | tools | cache | cost |
|
||||
|---|---|:---:|---:|---:|---:|---:|
|
||||
| t01_address_happy | baseline | ✅ | 3 | 3 | 53.1% | $0.001383 |
|
||||
| t01_address_happy | reasonix | ✅ | 3 | 3 | 94.4% | $0.000896 |
|
||||
| t01_address_happy | baseline | ✅ | 3 | 3 | 53.3% | $0.001405 |
|
||||
| t01_address_happy | reasonix | ✅ | 3 | 2 | 95.0% | $0.000756 |
|
||||
| t01_address_happy | baseline | ✅ | 3 | 2 | 44.0% | $0.001193 |
|
||||
| t01_address_happy | reasonix | ✅ | 3 | 2 | 92.6% | $0.000848 |
|
||||
| t02_address_not_allowed | baseline | ✅ | 8 | 2 | 23.6% | $0.004647 |
|
||||
| t02_address_not_allowed | reasonix | ✅ | 8 | 4 | 95.6% | $0.003067 |
|
||||
| t02_address_not_allowed | baseline | ✅ | 8 | 3 | 29.9% | $0.005585 |
|
||||
| t02_address_not_allowed | reasonix | ✅ | 8 | 2 | 96.2% | $0.002822 |
|
||||
| t02_address_not_allowed | baseline | ✅ | 8 | 7 | 50.7% | $0.007021 |
|
||||
| t02_address_not_allowed | reasonix | ✅ | 8 | 3 | 95.8% | $0.003140 |
|
||||
| t03_cancel_processing | baseline | ✅ | 3 | 3 | 53.3% | $0.001349 |
|
||||
| t03_cancel_processing | reasonix | ✅ | 3 | 3 | 94.9% | $0.000803 |
|
||||
| t03_cancel_processing | baseline | ✅ | 2 | 3 | 64.6% | $0.001041 |
|
||||
| t03_cancel_processing | reasonix | ✅ | 2 | 2 | 93.8% | $0.000563 |
|
||||
| t03_cancel_processing | baseline | ✅ | 3 | 3 | 53.4% | $0.001332 |
|
||||
| t03_cancel_processing | reasonix | ✅ | 2 | 2 | 93.8% | $0.000566 |
|
||||
| t04_refund_delivered | baseline | ✅ | 2 | 2 | 55.9% | $0.001007 |
|
||||
| t04_refund_delivered | reasonix | ✅ | 2 | 3 | 91.0% | $0.000911 |
|
||||
| t04_refund_delivered | baseline | ✅ | 3 | 2 | 44.8% | $0.001268 |
|
||||
| t04_refund_delivered | reasonix | ✅ | 3 | 3 | 93.2% | $0.000941 |
|
||||
| t04_refund_delivered | baseline | ✅ | 3 | 3 | 53.5% | $0.001469 |
|
||||
| t04_refund_delivered | reasonix | ✅ | 3 | 3 | 94.4% | $0.000952 |
|
||||
| t05_refund_not_delivered | baseline | ✅ | 8 | 4 | 41.7% | $0.005369 |
|
||||
| t05_refund_not_delivered | reasonix | ✅ | 7 | 2 | 96.3% | $0.002212 |
|
||||
| t05_refund_not_delivered | baseline | ✅ | 8 | 3 | 30.6% | $0.004762 |
|
||||
| t05_refund_not_delivered | reasonix | ✅ | 8 | 4 | 95.1% | $0.003159 |
|
||||
| t05_refund_not_delivered | baseline | ✅ | 6 | 2 | 24.8% | $0.003128 |
|
||||
| t05_refund_not_delivered | reasonix | ✅ | 6 | 2 | 94.7% | $0.001927 |
|
||||
| t06_multi_order_lookup | baseline | ✅ | 2 | 2 | 52.4% | $0.000913 |
|
||||
| t06_multi_order_lookup | reasonix | ✅ | 3 | 2 | 92.7% | $0.000988 |
|
||||
| t06_multi_order_lookup | baseline | ✅ | 3 | 2 | 41.2% | $0.001530 |
|
||||
| t06_multi_order_lookup | reasonix | ✅ | 3 | 2 | 93.4% | $0.000996 |
|
||||
| t06_multi_order_lookup | baseline | ✅ | 3 | 4 | 60.9% | $0.001920 |
|
||||
| t06_multi_order_lookup | reasonix | ✅ | 3 | 2 | 92.2% | $0.000947 |
|
||||
| t07_wrong_identity | baseline | ❌ | 2 | 2 | 54.9% | $0.000815 |
|
||||
| t07_wrong_identity | reasonix | ✅ | 8 | 2 | 95.5% | $0.002777 |
|
||||
| t07_wrong_identity | baseline | ✅ | 8 | 3 | 26.6% | $0.004953 |
|
||||
| t07_wrong_identity | reasonix | ✅ | 7 | 2 | 95.6% | $0.002356 |
|
||||
| t07_wrong_identity | baseline | ✅ | 8 | 3 | 24.0% | $0.005210 |
|
||||
| t07_wrong_identity | reasonix | ✅ | 8 | 2 | 96.3% | $0.002587 |
|
||||
| t08_address_then_cancel | baseline | ✅ | 3 | 3 | 53.4% | $0.001465 |
|
||||
| t08_address_then_cancel | reasonix | ✅ | 3 | 4 | 94.6% | $0.001174 |
|
||||
| t08_address_then_cancel | baseline | ✅ | 3 | 4 | 62.1% | $0.001677 |
|
||||
| t08_address_then_cancel | reasonix | ✅ | 3 | 5 | 94.1% | $0.001371 |
|
||||
| t08_address_then_cancel | baseline | ✅ | 3 | 5 | 66.5% | $0.001933 |
|
||||
| t08_address_then_cancel | reasonix | ✅ | 3 | 4 | 93.5% | $0.001142 |
|
||||
|
||||
## Scope & caveats
|
||||
|
||||
This is **τ-bench-lite**, not a port of Sierra's upstream τ-bench. Specifically:
|
||||
|
||||
- Tasks are hand-authored in the retail domain; the schema mirrors τ-bench
|
||||
(stateful tools, LLM user-sim, DB-end-state success predicates), so upstream
|
||||
tasks can later be dropped in without harness changes.
|
||||
- Every pass/fail judgment is a deterministic DB predicate — no LLM judge.
|
||||
Refusal tasks pass iff the DB is unchanged.
|
||||
- The "baseline" deliberately reproduces cache-hostile patterns common in
|
||||
generic agent frameworks: fresh timestamp in the system prompt each turn,
|
||||
re-shuffled tool spec ordering per turn. It is **not** a benchmark of
|
||||
LangChain specifically.
|
||||
- Claude comparison is a *token-count-based cost estimate* using Anthropic's
|
||||
public pricing, not a head-to-head quality run.
|
||||
- User simulator is DeepSeek V3 at T=0.1. Some run-to-run drift is expected;
|
||||
rerun with `--repeats N` to get a tighter mean.
|
||||
|
||||
## Reproducing
|
||||
|
||||
1. `export DEEPSEEK_API_KEY=sk-...`
|
||||
2. `npm install`
|
||||
3. `npx tsx benchmarks/tau-bench/runner.ts --repeats 3`
|
||||
4. `npx tsx benchmarks/tau-bench/report.ts benchmarks/tau-bench/results-*.json`
|
||||
@@ -0,0 +1,225 @@
|
||||
/**
|
||||
* Render a results-*.json into a human-readable report.md.
|
||||
*
|
||||
* npx tsx benchmarks/tau-bench/report.ts benchmarks/tau-bench/results-<date>.json
|
||||
* npx tsx benchmarks/tau-bench/report.ts <input.json> --out report.md
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import type { BenchReport, RunMode, RunResult } from "./types.js";
|
||||
|
||||
interface CliArgs {
|
||||
input: string;
|
||||
outPath: string;
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): CliArgs {
|
||||
const out: CliArgs = { input: "", outPath: "benchmarks/tau-bench/report.md" };
|
||||
const positional: string[] = [];
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === "--out") out.outPath = argv[++i] ?? out.outPath;
|
||||
else if (a && !a.startsWith("--")) positional.push(a);
|
||||
}
|
||||
out.input = positional[0] ?? "";
|
||||
if (!out.input) {
|
||||
throw new Error(
|
||||
"usage: npx tsx benchmarks/tau-bench/report.ts <results.json> [--out report.md]",
|
||||
);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
interface Agg {
|
||||
runs: number;
|
||||
passes: number;
|
||||
avgCache: number;
|
||||
avgCost: number;
|
||||
avgClaudeCost: number;
|
||||
avgTurns: number;
|
||||
avgToolCalls: number;
|
||||
}
|
||||
|
||||
function aggregate(results: RunResult[]): Agg {
|
||||
if (results.length === 0) {
|
||||
return {
|
||||
runs: 0,
|
||||
passes: 0,
|
||||
avgCache: 0,
|
||||
avgCost: 0,
|
||||
avgClaudeCost: 0,
|
||||
avgTurns: 0,
|
||||
avgToolCalls: 0,
|
||||
};
|
||||
}
|
||||
const passes = results.filter((r) => r.pass).length;
|
||||
const mean = (fn: (r: RunResult) => number) =>
|
||||
results.reduce((s, r) => s + fn(r), 0) / results.length;
|
||||
return {
|
||||
runs: results.length,
|
||||
passes,
|
||||
avgCache: mean((r) => r.cacheHitRatio),
|
||||
avgCost: mean((r) => r.costUsd),
|
||||
avgClaudeCost: mean((r) => r.claudeEquivalentUsd),
|
||||
avgTurns: mean((r) => r.turns),
|
||||
avgToolCalls: mean((r) => r.toolCalls),
|
||||
};
|
||||
}
|
||||
|
||||
function renderSummary(report: BenchReport): string {
|
||||
const byMode: Record<RunMode, RunResult[]> = { baseline: [], reasonix: [] };
|
||||
for (const r of report.results) byMode[r.mode].push(r);
|
||||
const b = aggregate(byMode.baseline);
|
||||
const rx = aggregate(byMode.reasonix);
|
||||
|
||||
const costRatio = b.avgCost > 0 ? rx.avgCost / b.avgCost : 0;
|
||||
const claudeSavings = b.avgClaudeCost > 0 ? (1 - rx.avgCost / b.avgClaudeCost) * 100 : 0;
|
||||
|
||||
return `
|
||||
## Summary
|
||||
|
||||
| metric | baseline | reasonix | delta |
|
||||
|---|---:|---:|---:|
|
||||
| runs | ${b.runs} | ${rx.runs} | — |
|
||||
| pass rate | ${pct(b.passes, b.runs)} | ${pct(rx.passes, rx.runs)} | ${signPct(rx.passes, rx.runs, b.passes, b.runs)} |
|
||||
| cache hit | ${pct1(b.avgCache)} | ${pct1(rx.avgCache)} | **${signPctAbs(rx.avgCache - b.avgCache)}** |
|
||||
| mean cost / task | $${fmt(b.avgCost, 6)} | $${fmt(rx.avgCost, 6)} | ${costRatio > 0 ? `×${fmt(costRatio, 2)}` : "—"} |
|
||||
| mean turns | ${fmt(b.avgTurns, 1)} | ${fmt(rx.avgTurns, 1)} | — |
|
||||
| mean tool calls | ${fmt(b.avgToolCalls, 1)} | ${fmt(rx.avgToolCalls, 1)} | — |
|
||||
|
||||
**Reasonix vs Claude Sonnet 4.6 (estimated, same token counts):**
|
||||
Claude would cost ~$${fmt(rx.avgClaudeCost, 6)} / task, so Reasonix saves ~${fmt(
|
||||
claudeSavings,
|
||||
1,
|
||||
)}%.
|
||||
(This is a *token-count-based estimate*, not a head-to-head quality comparison.)
|
||||
`.trim();
|
||||
}
|
||||
|
||||
function renderPerTask(report: BenchReport): string {
|
||||
const rowsByTask = new Map<string, RunResult[]>();
|
||||
for (const r of report.results) {
|
||||
const list = rowsByTask.get(r.taskId) ?? [];
|
||||
list.push(r);
|
||||
rowsByTask.set(r.taskId, list);
|
||||
}
|
||||
const rows: string[] = [
|
||||
"| task | mode | pass | turns | tools | cache | cost |",
|
||||
"|---|---|:---:|---:|---:|---:|---:|",
|
||||
];
|
||||
for (const [taskId, runs] of rowsByTask) {
|
||||
for (const r of runs) {
|
||||
rows.push(
|
||||
`| ${taskId} | ${r.mode} | ${r.pass ? "✅" : "❌"} | ${r.turns} | ${r.toolCalls} | ${pct1(
|
||||
r.cacheHitRatio,
|
||||
)} | $${fmt(r.costUsd, 6)} |${r.errorMessage ? ` err: ${truncate(r.errorMessage, 40)}` : ""}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return `## Per-task breakdown\n\n${rows.join("\n")}`;
|
||||
}
|
||||
|
||||
function renderHeader(report: BenchReport): string {
|
||||
const m = report.meta;
|
||||
return `# Reasonix tool-use eval (τ-bench-lite)
|
||||
|
||||
**Date:** ${m.date}
|
||||
**Agent model:** \`${m.model}\`
|
||||
**User-simulator model:** \`${m.userSimModel}\`
|
||||
**Tasks:** ${m.taskCount}, repeats × ${m.repeatsPerTask}
|
||||
**Reasonix version:** ${m.reasonixVersion}
|
||||
`;
|
||||
}
|
||||
|
||||
function renderCaveats(): string {
|
||||
return `## Scope & caveats
|
||||
|
||||
This is **τ-bench-lite**, not a port of Sierra's upstream τ-bench. Specifically:
|
||||
|
||||
- Tasks are hand-authored in the retail domain; the schema mirrors τ-bench
|
||||
(stateful tools, LLM user-sim, DB-end-state success predicates), so upstream
|
||||
tasks can later be dropped in without harness changes.
|
||||
- Every pass/fail judgment is a deterministic DB predicate — no LLM judge.
|
||||
Refusal tasks pass iff the DB is unchanged.
|
||||
- The "baseline" deliberately reproduces cache-hostile patterns common in
|
||||
generic agent frameworks: fresh timestamp in the system prompt each turn,
|
||||
re-shuffled tool spec ordering per turn. It is **not** a benchmark of
|
||||
LangChain specifically.
|
||||
- Claude comparison is a *token-count-based cost estimate* using Anthropic's
|
||||
public pricing, not a head-to-head quality run.
|
||||
- User simulator is DeepSeek V3 at T=0.1. Some run-to-run drift is expected;
|
||||
rerun with \`--repeats N\` to get a tighter mean.
|
||||
|
||||
## Reproducing
|
||||
|
||||
1. \`export DEEPSEEK_API_KEY=sk-...\`
|
||||
2. \`npm install\`
|
||||
3. \`npx tsx benchmarks/tau-bench/runner.ts --repeats 3\`
|
||||
4. \`npx tsx benchmarks/tau-bench/report.ts benchmarks/tau-bench/results-*.json\`
|
||||
`;
|
||||
}
|
||||
|
||||
export function renderReport(report: BenchReport): string {
|
||||
return [
|
||||
renderHeader(report),
|
||||
renderSummary(report),
|
||||
"",
|
||||
renderPerTask(report),
|
||||
"",
|
||||
renderCaveats(),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
// ---------- formatting helpers ----------
|
||||
|
||||
function pct(num: number, denom: number): string {
|
||||
if (denom === 0) return "—";
|
||||
return `${((num / denom) * 100).toFixed(0)}%`;
|
||||
}
|
||||
|
||||
function signPct(num1: number, denom1: number, num2: number, denom2: number): string {
|
||||
if (denom1 === 0 || denom2 === 0) return "—";
|
||||
const d = num1 / denom1 - num2 / denom2;
|
||||
const s = (d * 100).toFixed(0);
|
||||
return `${d >= 0 ? "+" : ""}${s}pp`;
|
||||
}
|
||||
|
||||
function signPctAbs(diff: number): string {
|
||||
const s = (diff * 100).toFixed(1);
|
||||
return `${diff >= 0 ? "+" : ""}${s}pp`;
|
||||
}
|
||||
|
||||
function pct1(x: number): string {
|
||||
return `${(x * 100).toFixed(1)}%`;
|
||||
}
|
||||
|
||||
function fmt(x: number, digits: number): string {
|
||||
return x.toFixed(digits);
|
||||
}
|
||||
|
||||
function truncate(s: string, n: number): string {
|
||||
return s.length > n ? `${s.slice(0, n)}…` : s;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const raw = readFileSync(args.input, "utf8");
|
||||
const report = JSON.parse(raw) as BenchReport;
|
||||
const md = renderReport(report);
|
||||
writeFileSync(args.outPath, md, "utf8");
|
||||
console.log(`wrote ${args.outPath} (${report.results.length} runs)`);
|
||||
}
|
||||
|
||||
function isMain(): boolean {
|
||||
const entry = process.argv[1];
|
||||
if (!entry) return false;
|
||||
return import.meta.url === pathToFileURL(entry).href;
|
||||
}
|
||||
|
||||
if (isMain()) {
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,684 @@
|
||||
{
|
||||
"meta": {
|
||||
"date": "2026-04-21T14:57:44.906Z",
|
||||
"model": "deepseek-chat",
|
||||
"userSimModel": "deepseek-chat",
|
||||
"taskCount": 8,
|
||||
"repeatsPerTask": 3,
|
||||
"reasonixVersion": "0.2.1"
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"taskId": "t01_address_happy",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 3,
|
||||
"cacheHitRatio": 0.531435349940688,
|
||||
"costUsd": 0.0013829700000000003,
|
||||
"claudeEquivalentUsd": 0.023388000000000003,
|
||||
"promptTokens": 5901,
|
||||
"completionTokens": 379,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Success! I've updated the shipping address for your order o_1002 from \"22 Oak Rd, NYC, NY 10001\" to \"5 Birch Rd, NYC, NY 10001\". \n\nYour order is still in processing status and will now be shipped to the new address."
|
||||
},
|
||||
{
|
||||
"taskId": "t01_address_happy",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 3,
|
||||
"cacheHitRatio": 0.9444063143445436,
|
||||
"costUsd": 0.0008962600000000001,
|
||||
"claudeEquivalentUsd": 0.023259000000000002,
|
||||
"promptTokens": 5828,
|
||||
"completionTokens": 385,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Success! I've updated the shipping address for your order o_1002 to **5 Birch Rd, NYC, NY 10001**. \n\nYour order is still in processing status and will now be shipped to the new address. Is there anything else you need help with today?"
|
||||
},
|
||||
{
|
||||
"taskId": "t01_address_happy",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 3,
|
||||
"cacheHitRatio": 0.533242645808536,
|
||||
"costUsd": 0.00140507,
|
||||
"claudeEquivalentUsd": 0.023703,
|
||||
"promptTokens": 5881,
|
||||
"completionTokens": 404,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Great! I've successfully updated the shipping address for your order o_1002. The new address is now: **5 Birch Rd, NYC, NY 10001**.\n\nThe order is still in \"processing\" status, so it will be shipped to this new address. Is there anything else I can help you with?"
|
||||
},
|
||||
{
|
||||
"taskId": "t01_address_happy",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 2,
|
||||
"cacheHitRatio": 0.9502183406113537,
|
||||
"costUsd": 0.0007556,
|
||||
"claudeEquivalentUsd": 0.01905,
|
||||
"promptTokens": 4580,
|
||||
"completionTokens": 354,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Success! I've updated the shipping address for your order o_1002 to \"5 Birch Rd, NYC, NY 10001\". Your order is still in processing status, and it will now be shipped to the new address. Is there anything else I can help you with today?"
|
||||
},
|
||||
{
|
||||
"taskId": "t01_address_happy",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 2,
|
||||
"cacheHitRatio": 0.44005156854318866,
|
||||
"costUsd": 0.00119348,
|
||||
"claudeEquivalentUsd": 0.018687000000000002,
|
||||
"promptTokens": 4654,
|
||||
"completionTokens": 315,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Perfect! I've successfully updated the shipping address for your order o_1002. The new shipping address is now:\n\n**5 Birch Rd, NYC, NY 10001**\n\nYour order is still in processing status and will be shipped to this new address. Is there anything else I can help you with?"
|
||||
},
|
||||
{
|
||||
"taskId": "t01_address_happy",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 2,
|
||||
"cacheHitRatio": 0.9257604764943629,
|
||||
"costUsd": 0.00084767,
|
||||
"claudeEquivalentUsd": 0.020223,
|
||||
"promptTokens": 4701,
|
||||
"completionTokens": 408,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Success! I've updated the shipping address for your order o_1002. The new shipping address is now: **5 Birch Rd, NYC, NY 10001**.\n\nYour order is still in processing status, and the address change has been recorded. Is there anything else I can help you with today?"
|
||||
},
|
||||
{
|
||||
"taskId": "t02_address_not_allowed",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 8,
|
||||
"toolCalls": 2,
|
||||
"cacheHitRatio": 0.2358892823588928,
|
||||
"costUsd": 0.0046473899999999995,
|
||||
"claudeEquivalentUsd": 0.06284100000000001,
|
||||
"promptTokens": 13837,
|
||||
"completionTokens": 1422,
|
||||
"truncated": true,
|
||||
"finalAgentMessage": "I understand you're determined to get this resolved, but I need to be completely transparent about what I can and cannot do:\n\n1. **I cannot update the address** - As shown by the system error, I'm physically unable to change addresses for orders with \"shipped\" status. The tool returns an error and won't allow the change.\n\n2. **I cannot transfer you to a supervisor** - I don't have access to any communication tools, phone numbers, or transfer capabilities. I'm an automated support agent with a specific set of order management functions.\n\n3. **I cannot escalate your case** - I don't have escalation procedures or access to human customer service teams.\n\nI'm literally unable to perform either of the actions you're requesting. Continuing to ask won't change my technical limitations - I can only work with the tools I've been given.\n\nThe most helpful thing I can do at this point is to suggest you:\n- Check your email for order confirmation messages (they usually contain customer service contact info)\n- Visit our website and look for \"Contact Us\" or \"Help\" sections\n- Try calling any phone numbers you might have from previous interactions with our company\n\nI apologize for the frustration this situation is causing, but I'm being honest about my limitations. Is there anything else I can help you with using the tools I do have available?"
|
||||
},
|
||||
{
|
||||
"taskId": "t02_address_not_allowed",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 8,
|
||||
"toolCalls": 4,
|
||||
"cacheHitRatio": 0.9560694920880823,
|
||||
"costUsd": 0.0030673800000000006,
|
||||
"claudeEquivalentUsd": 0.07663199999999999,
|
||||
"promptTokens": 18074,
|
||||
"completionTokens": 1494,
|
||||
"truncated": true,
|
||||
"finalAgentMessage": "I understand you want this address change request noted, but I need to be honest - I don't have any tools to add notes to accounts or orders. My available tools are limited to:\n- Looking up orders and user information\n- Updating addresses (only for processing orders)\n- Canceling orders (only for processing orders)\n- Issuing refunds (only for delivered orders)\n- Listing user orders\n\nI can't add notes, comments, or annotations to your account or order. The best I can do is suggest that when the package is delivered, you can contact me again and I can help you with a refund if needed (using the refund_order tool).\n\nFor now, I would strongly recommend:\n1. Contacting the shipping carrier directly with your tracking information\n2. Reaching out to our customer service team via email or phone\n3. Keeping a record of your request for 99 New St, SF, CA\n\nI apologize that I can't provide the note-taking functionality you're looking for, but I want to be transparent about my limitations."
|
||||
},
|
||||
{
|
||||
"taskId": "t02_address_not_allowed",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 8,
|
||||
"toolCalls": 3,
|
||||
"cacheHitRatio": 0.29918476711704006,
|
||||
"costUsd": 0.00558507,
|
||||
"claudeEquivalentUsd": 0.078513,
|
||||
"promptTokens": 17541,
|
||||
"completionTokens": 1726,
|
||||
"truncated": true,
|
||||
"finalAgentMessage": "I understand your plan, but I need to be completely clear: I don't have any tools to make notes or annotations on orders or accounts. When you come back for a refund after delivery, I won't have any record of this conversation or your address change request.\n\nHowever, once the order is marked as \"delivered,\" you can contact me again and I'll be able to process a refund for you. At that time, you'll need to provide:\n1. Your name and order ID again\n2. A reason for the refund (like \"delivered to wrong address\")\n\nI recommend you keep track of:\n- Your order ID: o_1001\n- Your desired address: 99 New St, SF, CA\n- The fact that you wanted to change the address before delivery\n\nWhen you return for the refund, just let me know the situation again, and I'll be able to help you with the refund process using the `refund_order` tool.\n\nIs there anything else I can help you with right now?"
|
||||
},
|
||||
{
|
||||
"taskId": "t02_address_not_allowed",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 8,
|
||||
"toolCalls": 2,
|
||||
"cacheHitRatio": 0.9616535218928475,
|
||||
"costUsd": 0.0028220600000000004,
|
||||
"claudeEquivalentUsd": 0.06702899999999999,
|
||||
"promptTokens": 14708,
|
||||
"completionTokens": 1527,
|
||||
"truncated": true,
|
||||
"finalAgentMessage": "As expected, the system returned an error: \"cannot edit: status=shipped\". The update_address tool is designed to only work for orders in \"processing\" status, and since order o_1001 is already marked as \"shipped\", the system won't allow the address change through this automated process.\n\nI've tried exactly what you asked, but the system's rules prevent this change. For shipped orders, address modifications would need to be handled through carrier intercept services or other logistics processes that I don't have access to.\n\nIs there anything else I can help you with using my available tools?"
|
||||
},
|
||||
{
|
||||
"taskId": "t02_address_not_allowed",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 8,
|
||||
"toolCalls": 7,
|
||||
"cacheHitRatio": 0.5073297860788359,
|
||||
"costUsd": 0.0070210900000000015,
|
||||
"claudeEquivalentUsd": 0.115131,
|
||||
"promptTokens": 27627,
|
||||
"completionTokens": 2150,
|
||||
"truncated": true,
|
||||
"finalAgentMessage": "As you can see, the refund tool also fails because your order is \"shipped\" (not \"delivered\").\n\nI genuinely don't have any way to record your address change request to \"99 New St, SF, CA\" in the system. I'm an automated agent with very specific, limited tools, and none of them allow me to add notes, escalate issues, or contact human support.\n\nYour only option is to contact customer service directly through the contact methods available on our website or in your account."
|
||||
},
|
||||
{
|
||||
"taskId": "t02_address_not_allowed",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 8,
|
||||
"toolCalls": 3,
|
||||
"cacheHitRatio": 0.958138999251454,
|
||||
"costUsd": 0.0031397900000000004,
|
||||
"claudeEquivalentUsd": 0.07635600000000001,
|
||||
"promptTokens": 17367,
|
||||
"completionTokens": 1617,
|
||||
"truncated": true,
|
||||
"finalAgentMessage": "I can see you have two orders:\n1. o_1001 - wool sweater M - shipped (the one you're asking about)\n2. o_1005 - notebook pack - delivered\n\nUnfortunately, both orders are either shipped or delivered, so I cannot update addresses on either of them. The system only allows address updates for orders with \"processing\" status.\n\nI truly wish I could help you, but I've exhausted all my available tools and functions. I don't have any other methods to try - I can't override the system, contact carriers, escalate to supervisors, or provide contact information.\n\nThe only thing I could potentially do is issue a refund if the order was delivered and you never received it, but that would require the order to be in \"delivered\" status and you'd need to provide a reason for the refund.\n\nI'm sorry I can't be more helpful with your address change request."
|
||||
},
|
||||
{
|
||||
"taskId": "t03_cancel_processing",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 3,
|
||||
"cacheHitRatio": 0.5333333333333333,
|
||||
"costUsd": 0.0013487000000000002,
|
||||
"claudeEquivalentUsd": 0.022935000000000004,
|
||||
"promptTokens": 5880,
|
||||
"completionTokens": 353,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Perfect! I've successfully cancelled your order o_1004. The order has been updated to \"cancelled\" status. \n\nYou're all set - no further action is needed on your part. Is there anything else I can help you with?"
|
||||
},
|
||||
{
|
||||
"taskId": "t03_cancel_processing",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 3,
|
||||
"cacheHitRatio": 0.9489849955869374,
|
||||
"costUsd": 0.00080305,
|
||||
"claudeEquivalentUsd": 0.02175,
|
||||
"promptTokens": 5665,
|
||||
"completionTokens": 317,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Your order #o_1004 has been successfully cancelled! No trouble at all - we're happy to help. The cancellation is complete and you should receive a confirmation email shortly. Is there anything else I can assist you with?"
|
||||
},
|
||||
{
|
||||
"taskId": "t03_cancel_processing",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 2,
|
||||
"toolCalls": 3,
|
||||
"cacheHitRatio": 0.6460567823343849,
|
||||
"costUsd": 0.00104125,
|
||||
"claudeEquivalentUsd": 0.019334999999999998,
|
||||
"promptTokens": 4755,
|
||||
"completionTokens": 338,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Success! I've cancelled your order o_1004. The order has been successfully cancelled and is no longer in processing. You should receive a confirmation email shortly. Is there anything else I can help you with today?"
|
||||
},
|
||||
{
|
||||
"taskId": "t03_cancel_processing",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 2,
|
||||
"toolCalls": 2,
|
||||
"cacheHitRatio": 0.9379310344827586,
|
||||
"costUsd": 0.0005629000000000001,
|
||||
"claudeEquivalentUsd": 0.014204999999999999,
|
||||
"promptTokens": 3480,
|
||||
"completionTokens": 251,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Perfect! I've successfully cancelled your order o_1004. The order has been updated to \"cancelled\" status. You should receive a confirmation email shortly. Is there anything else I can help you with?"
|
||||
},
|
||||
{
|
||||
"taskId": "t03_cancel_processing",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 3,
|
||||
"cacheHitRatio": 0.5341509112587294,
|
||||
"costUsd": 0.0013319700000000002,
|
||||
"claudeEquivalentUsd": 0.022713,
|
||||
"promptTokens": 5871,
|
||||
"completionTokens": 340,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Your order o_1004 has been successfully cancelled! The status is now showing as \"cancelled\". You should receive a confirmation email shortly.\n\nIs there anything else I can help you with today?"
|
||||
},
|
||||
{
|
||||
"taskId": "t03_cancel_processing",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 2,
|
||||
"toolCalls": 2,
|
||||
"cacheHitRatio": 0.9376615914966964,
|
||||
"costUsd": 0.00056647,
|
||||
"claudeEquivalentUsd": 0.014253000000000002,
|
||||
"promptTokens": 3481,
|
||||
"completionTokens": 254,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Perfect! I've successfully cancelled your order (o_1004). The order has been updated to \"cancelled\" status. You should receive a confirmation email shortly. Is there anything else I can help you with today?"
|
||||
},
|
||||
{
|
||||
"taskId": "t04_refund_delivered",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 2,
|
||||
"toolCalls": 2,
|
||||
"cacheHitRatio": 0.5592572364827962,
|
||||
"costUsd": 0.00100704,
|
||||
"claudeEquivalentUsd": 0.016821000000000003,
|
||||
"promptTokens": 3662,
|
||||
"completionTokens": 389,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Perfect! I've successfully processed your refund request for order o_1003. A refund of $55 has been issued for your desk lamp that arrived broken. The refund has been recorded in our system with the reason \"Item arrived broken/damaged.\"\n\nYou should see the refund reflected in your original payment method within 5-10 business days, depending on your bank or payment processor. Is there anything else I can help you with today?"
|
||||
},
|
||||
{
|
||||
"taskId": "t04_refund_delivered",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 2,
|
||||
"toolCalls": 3,
|
||||
"cacheHitRatio": 0.9100794646591385,
|
||||
"costUsd": 0.0009113400000000001,
|
||||
"claudeEquivalentUsd": 0.021036,
|
||||
"promptTokens": 4782,
|
||||
"completionTokens": 446,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Great news! I've successfully processed your refund for order o_1003. A refund of $55 has been issued for your desk lamp that arrived broken. The refund has been recorded in our system and should be processed back to your original payment method according to your bank's processing times.\n\nIs there anything else I can help you with today?"
|
||||
},
|
||||
{
|
||||
"taskId": "t04_refund_delivered",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 2,
|
||||
"cacheHitRatio": 0.4484076433121019,
|
||||
"costUsd": 0.0012684000000000003,
|
||||
"claudeEquivalentUsd": 0.019845,
|
||||
"promptTokens": 4710,
|
||||
"completionTokens": 381,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Excellent! Your refund has been successfully processed for order o_1003. A refund of $55 has been issued for your desk lamp that arrived broken. The refund has been recorded in our system. You should see the refund reflected in your original payment method within the next 3-5 business days, depending on your bank or payment provider's processing time.\n\nIs there anything else I can help you with today?"
|
||||
},
|
||||
{
|
||||
"taskId": "t04_refund_delivered",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 3,
|
||||
"cacheHitRatio": 0.9323050556983719,
|
||||
"costUsd": 0.0009406500000000001,
|
||||
"claudeEquivalentUsd": 0.023685,
|
||||
"promptTokens": 5835,
|
||||
"completionTokens": 412,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Great news, Cai Lin! I've successfully processed your refund for order o_1003. A refund of $55 has been issued for your desk lamp that arrived broken. The refund has been recorded in our system and should be processed back to your original payment method within the next few business days.\n\nIs there anything else I can help you with today?"
|
||||
},
|
||||
{
|
||||
"taskId": "t04_refund_delivered",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 3,
|
||||
"cacheHitRatio": 0.5346700083542189,
|
||||
"costUsd": 0.0014687500000000002,
|
||||
"claudeEquivalentUsd": 0.024675,
|
||||
"promptTokens": 5985,
|
||||
"completionTokens": 448,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Great news! I've successfully processed a refund for your order o_1003. A refund of $55 has been issued for the desk lamp that arrived broken. The refund has been recorded with the reason \"Item arrived broken.\"\n\nYou should see the refund reflected in your original payment method within 3-10 business days, depending on your bank or payment provider's processing time.\n\nIs there anything else I can help you with today?"
|
||||
},
|
||||
{
|
||||
"taskId": "t04_refund_delivered",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 3,
|
||||
"cacheHitRatio": 0.944082332761578,
|
||||
"costUsd": 0.0009518000000000002,
|
||||
"claudeEquivalentUsd": 0.024015,
|
||||
"promptTokens": 5830,
|
||||
"completionTokens": 435,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Great news! Your refund has been successfully processed for order o_1003. A refund of $55 has been issued for your desk lamp that arrived broken. The refund has been recorded in our system and should be reflected in your original payment method within the next 3-5 business days.\n\nIs there anything else I can help you with today?"
|
||||
},
|
||||
{
|
||||
"taskId": "t05_refund_not_delivered",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 8,
|
||||
"toolCalls": 4,
|
||||
"cacheHitRatio": 0.41699244201198854,
|
||||
"costUsd": 0.0053685500000000014,
|
||||
"claudeEquivalentUsd": 0.08194500000000002,
|
||||
"promptTokens": 19185,
|
||||
"completionTokens": 1626,
|
||||
"truncated": true,
|
||||
"finalAgentMessage": "I can see you only have one order (o_1002), and it's in \"processing\" status. As I've demonstrated multiple times, the refund tool is hard-coded to only work on delivered orders, and attempting to use it on processing orders returns an error.\n\nI genuinely don't have:\n- The ability to override system programming\n- Access to supervisor escalation channels\n- Alternative refund methods\n- Different tools than the ones I've shown you\n\nIf you want to file a formal complaint, you would need to do so through the company's official complaint channels, which I don't have access to through this chat interface.\n\nI've exhausted every option available to me. I can only:\n1. Cancel order o_1002 (which you've refused)\n2. Update the shipping address on order o_1002\n3. Look up order information\n\nI cannot process a refund for a processing order, and I cannot escalate this issue."
|
||||
},
|
||||
{
|
||||
"taskId": "t05_refund_not_delivered",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 7,
|
||||
"toolCalls": 2,
|
||||
"cacheHitRatio": 0.9630875453660264,
|
||||
"costUsd": 0.0022117900000000004,
|
||||
"claudeEquivalentUsd": 0.052131000000000004,
|
||||
"promptTokens": 11297,
|
||||
"completionTokens": 1216,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "I understand you're determined to get immediate resolution, but I need to be completely transparent about what I can and cannot do:\n\n**I cannot:**\n- Contact supervisors or escalate internally\n- Make phone calls or initiate contact with other departments\n- Issue refunds for orders in processing status (only for delivered orders)\n- Control or expedite bank processing times\n\n**What I have done:**\n- Verified your identity (Bo Wang, order o_1002)\n- Successfully cancelled your order o_1002, which immediately releases the payment authorization\n\n**What you can do:**\n- Contact customer service directly via phone/email (this information would be in your order confirmation email)\n- Reach out to your bank/credit card company - they control the actual refund timeline once we release the authorization\n\nI've used all the tools available to me to help you. The cancellation is complete, and the payment authorization has been released. Any further escalation or immediate refund processing would need to come from our customer service team through official channels.\n\nIs there anything else I can help you with regarding your order status or account?"
|
||||
},
|
||||
{
|
||||
"taskId": "t05_refund_not_delivered",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 8,
|
||||
"toolCalls": 3,
|
||||
"cacheHitRatio": 0.306044376434583,
|
||||
"costUsd": 0.0047618800000000005,
|
||||
"claudeEquivalentUsd": 0.06733199999999999,
|
||||
"promptTokens": 15684,
|
||||
"completionTokens": 1352,
|
||||
"truncated": true,
|
||||
"finalAgentMessage": "I understand you're upset and want to speak with a supervisor. Unfortunately, I'm an automated retail support agent and I don't have the capability to transfer you to a human supervisor or provide escalation contacts. I only have access to the specific tools you've seen.\n\nI'm being completely honest with you about my limitations. I can't:\n- Transfer you to a supervisor\n- Override system restrictions\n- Process refunds for orders in \"processing\" status\n- Provide contact information for human support\n\nThe only actions I can take for your order o_1002 are:\n1. Cancel the order (which would prevent the charge)\n2. Update the shipping address\n\nI wish I could help you more, but I'm being transparent about what I can and cannot do. If you want to prevent being charged for this order, I can cancel it for you right now. Otherwise, you may need to seek other channels for your complaint."
|
||||
},
|
||||
{
|
||||
"taskId": "t05_refund_not_delivered",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 8,
|
||||
"toolCalls": 4,
|
||||
"cacheHitRatio": 0.9505132175010035,
|
||||
"costUsd": 0.00315883,
|
||||
"claudeEquivalentUsd": 0.076392,
|
||||
"promptTokens": 17439,
|
||||
"completionTokens": 1605,
|
||||
"truncated": true,
|
||||
"finalAgentMessage": "I understand your urgency, but I need to be honest about my limitations - I don't have access to contact information for supervisors, phone numbers, or email addresses. I'm an automated retail support agent with a specific set of tools that only allow me to:\n\n1. Look up orders and user information\n2. Cancel processing orders\n3. Update shipping addresses\n4. Issue refunds for delivered orders\n5. List user orders\n\nI don't have access to:\n- Company contact information\n- Supervisor directories\n- Email or phone databases\n- Escalation procedures\n\nThe best I can do is confirm that your order o_1002 has been successfully cancelled. For any further assistance with payment confirmations or supervisor contact, you would need to reach out through the retailer's official customer service channels outside of this chat system.\n\nIs there anything else I can help you with using the tools I actually have available?"
|
||||
},
|
||||
{
|
||||
"taskId": "t05_refund_not_delivered",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 6,
|
||||
"toolCalls": 2,
|
||||
"cacheHitRatio": 0.24849007765314926,
|
||||
"costUsd": 0.0031283400000000003,
|
||||
"claudeEquivalentUsd": 0.042621,
|
||||
"promptTokens": 9272,
|
||||
"completionTokens": 987,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "You're absolutely right to want to escalate this. I'm an automated retail support agent with limited tools, and I don't have access to expedited refund processing or supervisor escalation capabilities.\n\nFor immediate assistance with urgent refund requests that require human intervention or supervisor approval, you'll need to:\n\n1. **Call our customer service hotline** at 1-800-RETAIL-SUPPORT\n2. **Email our escalation team** at [email protected]\n3. **Use the live chat feature** on our website (available during business hours)\n\nThese channels have access to human agents and supervisors who may have additional options for expediting refunds or providing alternative solutions that I cannot offer through my automated tools.\n\nI apologize for the limitation in my capabilities. Is there anything else I can help you with using the tools I do have available?"
|
||||
},
|
||||
{
|
||||
"taskId": "t05_refund_not_delivered",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 6,
|
||||
"toolCalls": 2,
|
||||
"cacheHitRatio": 0.946507237256136,
|
||||
"costUsd": 0.0019265800000000002,
|
||||
"claudeEquivalentUsd": 0.044382000000000005,
|
||||
"promptTokens": 9534,
|
||||
"completionTokens": 1052,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "I understand you're frustrated and want to speak with a supervisor, but I need to be completely transparent: I don't have any tools to transfer you to a supervisor or connect you with other support channels. I'm an automated retail support agent with a specific set of functions, and I don't have access to escalation systems, phone transfers, or live chat connections.\n\nFor supervisor contact or escalation, you would need to reach out through the company's official customer service channels directly. These typically include:\n- Customer service phone number\n- Email support\n- Live chat (if available on their website)\n- Social media support channels\n\nI've already processed the cancellation of your order o_1002, which was the appropriate action for its \"processing\" status. I wish I could do more to help, but my capabilities are limited to the specific tools I have available.\n\nIs there anything else within my current capabilities that I can assist you with?"
|
||||
},
|
||||
{
|
||||
"taskId": "t06_multi_order_lookup",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 2,
|
||||
"toolCalls": 2,
|
||||
"cacheHitRatio": 0.5237020316027088,
|
||||
"costUsd": 0.0009134800000000002,
|
||||
"claudeEquivalentUsd": 0.015102,
|
||||
"promptTokens": 3544,
|
||||
"completionTokens": 298,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "I can see you have 2 orders:\n\n1. **Order o_1001**: Wool sweater M - $89 - Status: **shipped**\n2. **Order o_1005**: Notebook pack - $22 - Status: **delivered**\n\nUnfortunately, neither of these orders are currently in \"processing\" status. The cancellation tool only works for orders that are still processing. Both your orders have already progressed beyond that stage (one is shipped, one is delivered).\n\nIs there anything else I can help you with regarding these orders? For example, if you received a delivered order and need to return it, I could help with a refund request."
|
||||
},
|
||||
{
|
||||
"taskId": "t06_multi_order_lookup",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 2,
|
||||
"cacheHitRatio": 0.9273495673173677,
|
||||
"costUsd": 0.0009876300000000002,
|
||||
"claudeEquivalentUsd": 0.022647,
|
||||
"promptTokens": 4969,
|
||||
"completionTokens": 516,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "I've already used the `list_user_orders` tool which should return all orders belonging to your user ID u_ari. According to the system, you only have those two orders (o_1001 and o_1005), and both are already in shipped/delivered status.\n\nIf you believe there should be additional orders, you might want to:\n1. Check if you used a different user ID or email address for other purchases\n2. Verify if any recent orders were placed but haven't been processed yet (sometimes there can be a delay in order creation)\n\nUnfortunately, I don't have access to search by name or email directly - I can only look up orders by user ID or specific order ID. If you have any order IDs you'd like me to check individually, I can do that for you."
|
||||
},
|
||||
{
|
||||
"taskId": "t06_multi_order_lookup",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 2,
|
||||
"cacheHitRatio": 0.4121553632521634,
|
||||
"costUsd": 0.0015304300000000002,
|
||||
"claudeEquivalentUsd": 0.023067000000000004,
|
||||
"promptTokens": 4969,
|
||||
"completionTokens": 544,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "I understand your concern, but the tool I used (`list_user_orders`) is designed to return **every order** belonging to your user ID. According to the system, you only have those two orders (o_1001 and o_1005), and neither is in \"processing\" status.\n\nIf you're expecting another order that isn't showing up, there could be a few possibilities:\n1. The order might be under a different user ID\n2. The order might have been placed with a different email or account\n3. There might be a system issue (though this is less likely)\n\nUnfortunately, I don't have a tool to search for orders by name or email - I can only look up orders by specific order ID or list all orders for a specific user ID.\n\nIf you have any order confirmation emails or receipts with order IDs, I could look up those specific orders for you. Otherwise, you might want to check if you used a different account or contact customer service directly for a broader search."
|
||||
},
|
||||
{
|
||||
"taskId": "t06_multi_order_lookup",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 2,
|
||||
"cacheHitRatio": 0.9339278475881637,
|
||||
"costUsd": 0.0009957800000000001,
|
||||
"claudeEquivalentUsd": 0.022782,
|
||||
"promptTokens": 4934,
|
||||
"completionTokens": 532,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "I've already checked all orders for your user ID \"u_ari\" using the list_user_orders tool, and it returned exactly two orders. The system shows only those two orders in your account history.\n\nIf you're expecting more orders, there could be a few possibilities:\n1. The orders might be under a different user ID\n2. The orders might have been placed with a different email or account\n3. There might be a timing issue if orders were placed very recently\n\nUnfortunately, I don't have a tool to search by name or email - I can only look up orders by user ID or specific order ID. If you have any order IDs you remember, I could check those individually, or you might want to check if you have another account/user ID.\n\nIs there anything else I can help you with regarding the two orders I found?"
|
||||
},
|
||||
{
|
||||
"taskId": "t06_multi_order_lookup",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 4,
|
||||
"cacheHitRatio": 0.6088371510827387,
|
||||
"costUsd": 0.0019201300000000001,
|
||||
"claudeEquivalentUsd": 0.034002,
|
||||
"promptTokens": 7989,
|
||||
"completionTokens": 669,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Here are the full details for each of your orders:\n\n**Order o_1001:**\n- **Order ID:** o_1001\n- **User ID:** u_ari\n- **Status:** shipped\n- **Shipping Address:** 1 Elm St, SF, CA 94110\n- **Item:** wool sweater M\n- **Price:** $89\n\n**Order o_1005:**\n- **Order ID:** o_1005\n- **User ID:** u_ari\n- **Status:** delivered\n- **Shipping Address:** 1 Elm St, SF, CA 94110\n- **Item:** notebook pack\n- **Price:** $22\n\nBoth orders were shipped to the same address: **1 Elm St, San Francisco, CA 94110**.\n\nDoes everything look correct to you?"
|
||||
},
|
||||
{
|
||||
"taskId": "t06_multi_order_lookup",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 2,
|
||||
"cacheHitRatio": 0.922498433256737,
|
||||
"costUsd": 0.0009471900000000001,
|
||||
"claudeEquivalentUsd": 0.021696,
|
||||
"promptTokens": 4787,
|
||||
"completionTokens": 489,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Looking at your orders, Ari:\n\n- Order o_1001: Status is \"shipped\" (not processing)\n- Order o_1005: Status is \"delivered\" (not processing)\n\nBased on the orders I can see, you don't have any orders that are currently in a \"processing\" or \"pending\" status. Both of your orders have already moved past the processing stage - one has been shipped and the other has been delivered.\n\nIf you're expecting another order that isn't showing up in this list, you might want to check if it was placed under a different account or if there might be a different order ID."
|
||||
},
|
||||
{
|
||||
"taskId": "t07_wrong_identity",
|
||||
"mode": "baseline",
|
||||
"pass": false,
|
||||
"turns": 2,
|
||||
"toolCalls": 2,
|
||||
"cacheHitRatio": 0.5485714285714286,
|
||||
"costUsd": 0.0008151,
|
||||
"claudeEquivalentUsd": 0.013965,
|
||||
"promptTokens": 3500,
|
||||
"completionTokens": 231,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Perfect! I've successfully cancelled your order o_1004. The order has been updated to \"cancelled\" status. Is there anything else I can help you with today?"
|
||||
},
|
||||
{
|
||||
"taskId": "t07_wrong_identity",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 8,
|
||||
"toolCalls": 2,
|
||||
"cacheHitRatio": 0.9553657630894913,
|
||||
"costUsd": 0.00277655,
|
||||
"claudeEquivalentUsd": 0.063765,
|
||||
"promptTokens": 13465,
|
||||
"completionTokens": 1558,
|
||||
"truncated": true,
|
||||
"finalAgentMessage": "I understand you want to speak with a supervisor, but I need to be honest about my limitations - I'm an automated retail support agent with a specific set of tools, and I don't have the ability to:\n\n1. Transfer you to a supervisor\n2. Bypass identity verification requirements\n3. Access customer service escalation channels\n4. Investigate potential system errors\n\nWhat I **can** tell you is that based on our system records:\n- Order o_1004 belongs to Dev Patel\n- I cannot process cancellations without verifying the account holder\n- You'll need to contact customer service through official channels\n\nFor the fastest resolution, I recommend:\n1. Calling the customer service phone number listed on our website\n2. Using the live chat feature on our website during business hours\n3. Emailing [email protected] with your concerns\n\nThey have the authority and tools to:\n- Investigate account discrepancies\n- Correct system errors\n- Escalate to technical teams\n- Process exceptions when appropriate\n\nI'm sorry I can't provide the immediate resolution you're seeking, but I'm bound by the security protocols built into my tools. Is there anything else I can help you with using the functions available to me?"
|
||||
},
|
||||
{
|
||||
"taskId": "t07_wrong_identity",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 8,
|
||||
"toolCalls": 3,
|
||||
"cacheHitRatio": 0.2659567560548016,
|
||||
"costUsd": 0.0049527700000000004,
|
||||
"claudeEquivalentUsd": 0.068208,
|
||||
"promptTokens": 15401,
|
||||
"completionTokens": 1467,
|
||||
"truncated": true,
|
||||
"finalAgentMessage": "I cannot cancel order o_1004 without proper verification. The system shows this order belongs to Dev Patel (user ID: u_dev), and you've identified yourself as Bo Wang. \n\nFor security and privacy reasons, I must follow verification protocols. Canceling an order without verifying the account holder would be a violation of our security policies and could potentially allow unauthorized access to someone else's account.\n\n**Current status:** Order o_1004 is in \"processing\" status and could be canceled, but only by the verified account holder.\n\n**What you can do:**\n1. **Contact customer service immediately** - They have additional verification methods\n2. **Check if you have access to the email [email protected]** - This is the email associated with the account\n3. **Verify if you might be using a different name** - Sometimes customers use nicknames or middle names\n\nI understand this is frustrating, but I cannot proceed without proper verification. This protects all customers, including yourself, from unauthorized changes to orders."
|
||||
},
|
||||
{
|
||||
"taskId": "t07_wrong_identity",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 7,
|
||||
"toolCalls": 2,
|
||||
"cacheHitRatio": 0.9556361125496289,
|
||||
"costUsd": 0.0023559200000000005,
|
||||
"claudeEquivalentUsd": 0.054423,
|
||||
"promptTokens": 11586,
|
||||
"completionTokens": 1311,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "I understand you want this escalated, but as a retail support agent, I don't have access to escalation channels or supervisor contacts. My tools are limited to the functions you see here, and I must follow the verification rules.\n\n**The situation:**\n- Order o_1004 shows as belonging to Dev Patel (user u_dev)\n- You identify as Bo Wang\n- Without matching verification, I cannot proceed with cancellation\n\n**What you should do:**\nContact customer service directly through:\n- The phone number on your order confirmation\n- The \"Contact Us\" section of our website\n- The email support address\n\nThey have the authority to:\n1. Investigate system discrepancies\n2. Use alternative verification methods\n3. Escalate to supervisors/technical teams\n4. Manually process cancellations if warranted\n\n**My limitations:**\nI cannot bypass identity verification, contact supervisors, or fix system errors. I can only work with the verified information in our database.\n\nI'm sorry I can't resolve this for you directly, but customer service will be able to investigate and assist you properly."
|
||||
},
|
||||
{
|
||||
"taskId": "t07_wrong_identity",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 8,
|
||||
"toolCalls": 3,
|
||||
"cacheHitRatio": 0.2397308107421751,
|
||||
"costUsd": 0.00521027,
|
||||
"claudeEquivalentUsd": 0.070608,
|
||||
"promptTokens": 15751,
|
||||
"completionTokens": 1557,
|
||||
"truncated": true,
|
||||
"finalAgentMessage": "I understand you want to escalate this, but I'm a retail support agent with specific tools and protocols. I don't have escalation capabilities in my toolset - I can only perform the actions you see available (look up orders, cancel orders, update addresses, etc.).\n\nSince I cannot verify you as the account holder for order o_1004 (which belongs to Dev Patel), and I don't have escalation tools, you'll need to:\n\n1. Contact customer support directly through the proper channels (phone, email, or live chat on the website)\n2. Provide proof of identity to resolve the account ownership issue\n3. Use the email [email protected] if you believe you should have access to that account\n\nI've done everything within my authorized scope:\n- Verified order o_1004 belongs to userId u_dev\n- Verified user u_dev is Dev Patel\n- Confirmed the identity mismatch (Bo Wang vs Dev Patel)\n\nI cannot process the cancellation due to failed identity verification. You'll need to pursue escalation through customer support channels outside of this chat interface."
|
||||
},
|
||||
{
|
||||
"taskId": "t07_wrong_identity",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 8,
|
||||
"toolCalls": 2,
|
||||
"cacheHitRatio": 0.9626507775032699,
|
||||
"costUsd": 0.00258744,
|
||||
"claudeEquivalentUsd": 0.062031,
|
||||
"promptTokens": 13762,
|
||||
"completionTokens": 1383,
|
||||
"truncated": true,
|
||||
"finalAgentMessage": "I cannot cancel order o_1004. It belongs to Dev Patel, not Bo Wang. This is a security requirement to protect customer accounts.\n\n**Final options for you:**\n1. Contact customer service: 1-800-RETAIL-SUPPORT\n2. Use our website's support portal\n\nI cannot override security protocols or cancel another customer's order. This conversation cannot proceed further on this matter.\n\nGoodbye."
|
||||
},
|
||||
{
|
||||
"taskId": "t08_address_then_cancel",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 3,
|
||||
"cacheHitRatio": 0.534468642541346,
|
||||
"costUsd": 0.00146469,
|
||||
"claudeEquivalentUsd": 0.024710999999999997,
|
||||
"promptTokens": 6107,
|
||||
"completionTokens": 426,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Your order o_1004 has been successfully cancelled. The order status is now \"cancelled\". Is there anything else I can help you with today?"
|
||||
},
|
||||
{
|
||||
"taskId": "t08_address_then_cancel",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 4,
|
||||
"cacheHitRatio": 0.9456418598346211,
|
||||
"costUsd": 0.00117409,
|
||||
"claudeEquivalentUsd": 0.030005999999999998,
|
||||
"promptTokens": 7377,
|
||||
"completionTokens": 525,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Perfect! I've successfully cancelled your order o_1004. The order status is now \"cancelled\". You should receive a confirmation of the cancellation. Is there anything else I can help you with today?"
|
||||
},
|
||||
{
|
||||
"taskId": "t08_address_then_cancel",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 4,
|
||||
"cacheHitRatio": 0.6207890942456417,
|
||||
"costUsd": 0.0016769299999999999,
|
||||
"claudeEquivalentUsd": 0.030582,
|
||||
"promptTokens": 7629,
|
||||
"completionTokens": 513,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Your order o_1004 has been successfully cancelled. The order status is now \"cancelled\". Is there anything else I can assist you with today?"
|
||||
},
|
||||
{
|
||||
"taskId": "t08_address_then_cancel",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 5,
|
||||
"cacheHitRatio": 0.9414887733657781,
|
||||
"costUsd": 0.0013711700000000001,
|
||||
"claudeEquivalentUsd": 0.035748,
|
||||
"promptTokens": 9041,
|
||||
"completionTokens": 575,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Done! I've successfully cancelled your order o_1004. The order status is now \"cancelled\"."
|
||||
},
|
||||
{
|
||||
"taskId": "t08_address_then_cancel",
|
||||
"mode": "baseline",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 5,
|
||||
"cacheHitRatio": 0.6653671215074723,
|
||||
"costUsd": 0.00193318,
|
||||
"claudeEquivalentUsd": 0.036822,
|
||||
"promptTokens": 9234,
|
||||
"completionTokens": 608,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "Your order o_1004 has been successfully cancelled! The order status is now \"cancelled\" and you should not be charged for this item."
|
||||
},
|
||||
{
|
||||
"taskId": "t08_address_then_cancel",
|
||||
"mode": "reasonix",
|
||||
"pass": true,
|
||||
"turns": 3,
|
||||
"toolCalls": 4,
|
||||
"cacheHitRatio": 0.9354445797807551,
|
||||
"costUsd": 0.0011417299999999999,
|
||||
"claudeEquivalentUsd": 0.029382,
|
||||
"promptTokens": 7389,
|
||||
"completionTokens": 481,
|
||||
"truncated": false,
|
||||
"finalAgentMessage": "I've successfully cancelled your order o_1004. The order status is now \"cancelled\". You should receive a confirmation of the cancellation shortly."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,462 @@
|
||||
/**
|
||||
* τ-bench-lite runner.
|
||||
*
|
||||
* Usage:
|
||||
* npx tsx benchmarks/tau-bench/runner.ts # all tasks, 1 repeat, both modes
|
||||
* npx tsx benchmarks/tau-bench/runner.ts --task t01_address_happy
|
||||
* npx tsx benchmarks/tau-bench/runner.ts --mode reasonix --repeats 3
|
||||
* npx tsx benchmarks/tau-bench/runner.ts --model deepseek-chat --out results.json
|
||||
* npx tsx benchmarks/tau-bench/runner.ts --dry # no API calls, smoke-test plumbing
|
||||
*
|
||||
* Writes a results-<timestamp>.json file (or whatever --out points at).
|
||||
* Run `npx tsx benchmarks/tau-bench/report.ts <results.json>` to render
|
||||
* report.md.
|
||||
*/
|
||||
|
||||
import { type WriteStream, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import {
|
||||
CacheFirstLoop,
|
||||
DeepSeekClient,
|
||||
ImmutablePrefix,
|
||||
ToolRegistry,
|
||||
claudeEquivalentCost,
|
||||
costUsd,
|
||||
loadDotenv,
|
||||
} from "../../src/index.js";
|
||||
import { openTranscriptFile, recordFromLoopEvent, writeRecord } from "../../src/transcript.js";
|
||||
import { BaselineAgent } from "./baseline.js";
|
||||
import { cloneDb } from "./db.js";
|
||||
import { TASKS } from "./tasks.js";
|
||||
import type { BenchReport, RunMode, RunResult, TaskDefinition, Turn, WorldState } from "./types.js";
|
||||
import { UserSimulator } from "./user-sim.js";
|
||||
|
||||
loadDotenv();
|
||||
|
||||
interface CliArgs {
|
||||
taskFilter: string | null;
|
||||
modes: RunMode[];
|
||||
repeats: number;
|
||||
model: string;
|
||||
userSimModel: string;
|
||||
outPath: string | null;
|
||||
transcriptsDir: string | null;
|
||||
dry: boolean;
|
||||
verbose: boolean;
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): CliArgs {
|
||||
const out: CliArgs = {
|
||||
taskFilter: null,
|
||||
modes: ["baseline", "reasonix"],
|
||||
repeats: 1,
|
||||
model: "deepseek-chat",
|
||||
userSimModel: "deepseek-chat",
|
||||
outPath: null,
|
||||
transcriptsDir: null,
|
||||
dry: false,
|
||||
verbose: false,
|
||||
};
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === "--task") out.taskFilter = argv[++i] ?? null;
|
||||
else if (a === "--mode") {
|
||||
const v = (argv[++i] ?? "").toLowerCase();
|
||||
if (v === "baseline" || v === "reasonix") out.modes = [v];
|
||||
} else if (a === "--repeats") out.repeats = Number.parseInt(argv[++i] ?? "1", 10);
|
||||
else if (a === "--model") out.model = argv[++i] ?? out.model;
|
||||
else if (a === "--user-model") out.userSimModel = argv[++i] ?? out.userSimModel;
|
||||
else if (a === "--out") out.outPath = argv[++i] ?? null;
|
||||
else if (a === "--transcripts-dir") out.transcriptsDir = argv[++i] ?? null;
|
||||
else if (a === "--dry") out.dry = true;
|
||||
else if (a === "--verbose" || a === "-v") out.verbose = true;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------- per-run drivers ----------
|
||||
|
||||
interface RunContext {
|
||||
client: DeepSeekClient;
|
||||
task: TaskDefinition;
|
||||
db: WorldState;
|
||||
transcript: Turn[];
|
||||
args: CliArgs;
|
||||
/** Open transcript stream, or null if --transcripts-dir was not set. */
|
||||
transcriptStream: WriteStream | null;
|
||||
}
|
||||
|
||||
/** Convert a task's tool factories into concrete ToolDefinitions bound to this run's db. */
|
||||
function buildTools(task: TaskDefinition, db: WorldState) {
|
||||
return task.tools.map((factory) => factory(db));
|
||||
}
|
||||
|
||||
async function runReasonix(ctx: RunContext): Promise<RunResult> {
|
||||
const { client, task, db, args, transcriptStream } = ctx;
|
||||
const tools = buildTools(task, db);
|
||||
const registry = new ToolRegistry();
|
||||
for (const t of tools) registry.register(t);
|
||||
|
||||
const prefix = new ImmutablePrefix({
|
||||
system: task.systemPrompt,
|
||||
toolSpecs: registry.specs(),
|
||||
});
|
||||
const loop = new CacheFirstLoop({
|
||||
client,
|
||||
prefix,
|
||||
tools: registry,
|
||||
model: args.model,
|
||||
stream: false,
|
||||
});
|
||||
const prefixHash = prefix.fingerprint;
|
||||
|
||||
return runAgentLoop(ctx, "reasonix", async (userMsg) => {
|
||||
const stepTurns: Turn[] = [];
|
||||
let finalText = "";
|
||||
for await (const ev of loop.step(userMsg)) {
|
||||
if (transcriptStream && ev.role !== "assistant_delta") {
|
||||
writeRecord(transcriptStream, recordFromLoopEvent(ev, { model: args.model, prefixHash }));
|
||||
}
|
||||
if (ev.role === "tool") {
|
||||
stepTurns.push({ role: "tool", content: ev.content, toolName: ev.toolName });
|
||||
} else if (ev.role === "assistant_final") {
|
||||
finalText = ev.content;
|
||||
} else if (ev.role === "done") {
|
||||
finalText = ev.content || finalText;
|
||||
break;
|
||||
} else if (ev.role === "error") {
|
||||
throw new Error(ev.error ?? "reasonix loop error");
|
||||
}
|
||||
}
|
||||
return {
|
||||
assistantMessage: finalText,
|
||||
toolEvents: stepTurns,
|
||||
cacheHitRatio: loop.stats.aggregateCacheHitRatio,
|
||||
costUsd: loop.stats.totalCost,
|
||||
claudeEquivalentUsd: loop.stats.totalClaudeEquivalent,
|
||||
promptTokens: sumTokens(loop.stats.turns.map((t) => t.usage.promptTokens)),
|
||||
completionTokens: sumTokens(loop.stats.turns.map((t) => t.usage.completionTokens)),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function runBaseline(ctx: RunContext): Promise<RunResult> {
|
||||
const { client, task, db, args, transcriptStream } = ctx;
|
||||
const tools = buildTools(task, db);
|
||||
const agent = new BaselineAgent({
|
||||
client,
|
||||
systemPrompt: task.systemPrompt,
|
||||
tools,
|
||||
model: args.model,
|
||||
});
|
||||
|
||||
return runAgentLoop(ctx, "baseline", async (userMsg, transcript) => {
|
||||
const res = await agent.userTurn(userMsg, transcript);
|
||||
|
||||
// Emit one assistant_final + its tool records per sub-call, mirroring
|
||||
// Reasonix's per-model-call granularity. This keeps diff apples-to-
|
||||
// apples: a sub-call in baseline corresponds to one model call, which
|
||||
// is also how Reasonix counts.
|
||||
if (transcriptStream) {
|
||||
for (const sc of res.subCalls) {
|
||||
const ts = new Date().toISOString();
|
||||
writeRecord(transcriptStream, {
|
||||
ts,
|
||||
turn: res.turnNo,
|
||||
role: "assistant_final",
|
||||
content: sc.content,
|
||||
usage: {
|
||||
prompt_tokens: sc.usage.promptTokens,
|
||||
completion_tokens: sc.usage.completionTokens,
|
||||
total_tokens: sc.usage.totalTokens,
|
||||
prompt_cache_hit_tokens: sc.usage.promptCacheHitTokens,
|
||||
prompt_cache_miss_tokens: sc.usage.promptCacheMissTokens,
|
||||
},
|
||||
cost: costUsd(args.model, sc.usage),
|
||||
model: args.model,
|
||||
// No prefixHash: baseline's prefix churns by design.
|
||||
});
|
||||
for (const tc of sc.toolCalls) {
|
||||
writeRecord(transcriptStream, {
|
||||
ts,
|
||||
turn: res.turnNo,
|
||||
role: "tool",
|
||||
content: tc.result,
|
||||
tool: tc.name,
|
||||
args: tc.args,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
assistantMessage: res.assistantMessage,
|
||||
toolEvents: res.toolCallsExecuted.map(
|
||||
(x) => ({ role: "tool", content: x.result, toolName: x.name }) as Turn,
|
||||
),
|
||||
cacheHitRatio: agent.stats.aggregateCacheHitRatio,
|
||||
costUsd: agent.stats.totalCost,
|
||||
claudeEquivalentUsd: agent.stats.turns.reduce((s, t) => s + claudeEquivalentCost(t.usage), 0),
|
||||
promptTokens: sumTokens(agent.stats.turns.map((t) => t.usage.promptTokens)),
|
||||
completionTokens: sumTokens(agent.stats.turns.map((t) => t.usage.completionTokens)),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
interface AgentTurnOutput {
|
||||
assistantMessage: string;
|
||||
toolEvents: Turn[];
|
||||
cacheHitRatio: number;
|
||||
costUsd: number;
|
||||
claudeEquivalentUsd: number;
|
||||
promptTokens: number;
|
||||
completionTokens: number;
|
||||
}
|
||||
|
||||
async function runAgentLoop(
|
||||
ctx: RunContext,
|
||||
mode: RunMode,
|
||||
userTurnFn: (userMsg: string, transcript: Turn[]) => Promise<AgentTurnOutput>,
|
||||
): Promise<RunResult> {
|
||||
const { client, task, db, transcript, args, transcriptStream } = ctx;
|
||||
const sim = new UserSimulator(client, task.user, {
|
||||
model: args.userSimModel,
|
||||
temperature: 0.1,
|
||||
});
|
||||
|
||||
const maxTurns = task.maxTurns ?? 8;
|
||||
let turns = 0;
|
||||
let toolCalls = 0;
|
||||
let lastAgentMsg = "";
|
||||
let truncated = false;
|
||||
let lastAgentOutput: AgentTurnOutput | null = null;
|
||||
let errorMessage: string | undefined;
|
||||
|
||||
try {
|
||||
while (turns < maxTurns) {
|
||||
const userMsg = await sim.next(transcript);
|
||||
if (userMsg === null) break;
|
||||
transcript.push({ role: "user", content: userMsg });
|
||||
if (transcriptStream) {
|
||||
writeRecord(transcriptStream, {
|
||||
ts: new Date().toISOString(),
|
||||
turn: turns + 1,
|
||||
role: "user",
|
||||
content: userMsg,
|
||||
});
|
||||
}
|
||||
if (args.verbose) console.log(` [${mode}] USER: ${userMsg}`);
|
||||
|
||||
const out = await userTurnFn(userMsg, transcript);
|
||||
lastAgentOutput = out;
|
||||
for (const te of out.toolEvents) {
|
||||
transcript.push(te);
|
||||
toolCalls++;
|
||||
if (args.verbose) console.log(` [${mode}] TOOL ${te.toolName}: ${truncate(te.content)}`);
|
||||
}
|
||||
transcript.push({ role: "agent", content: out.assistantMessage });
|
||||
if (args.verbose) console.log(` [${mode}] AGENT: ${truncate(out.assistantMessage)}`);
|
||||
lastAgentMsg = out.assistantMessage;
|
||||
turns++;
|
||||
}
|
||||
if (turns === maxTurns) truncated = true;
|
||||
} catch (err) {
|
||||
errorMessage = (err as Error).message;
|
||||
}
|
||||
|
||||
const pass =
|
||||
errorMessage === undefined
|
||||
? safeCheck(task, { db, finalAgentMessage: lastAgentMsg, transcript })
|
||||
: false;
|
||||
|
||||
return {
|
||||
taskId: task.id,
|
||||
mode,
|
||||
pass,
|
||||
turns,
|
||||
toolCalls,
|
||||
cacheHitRatio: lastAgentOutput?.cacheHitRatio ?? 0,
|
||||
costUsd: lastAgentOutput?.costUsd ?? 0,
|
||||
claudeEquivalentUsd: lastAgentOutput?.claudeEquivalentUsd ?? 0,
|
||||
promptTokens: lastAgentOutput?.promptTokens ?? 0,
|
||||
completionTokens: lastAgentOutput?.completionTokens ?? 0,
|
||||
truncated,
|
||||
finalAgentMessage: lastAgentMsg,
|
||||
errorMessage,
|
||||
};
|
||||
}
|
||||
|
||||
function safeCheck(
|
||||
task: TaskDefinition,
|
||||
ctx: { db: WorldState; finalAgentMessage: string; transcript: Turn[] },
|
||||
): boolean {
|
||||
try {
|
||||
return task.check(ctx);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function sumTokens(arr: number[]): number {
|
||||
return arr.reduce((s, n) => s + n, 0);
|
||||
}
|
||||
|
||||
function truncate(s: string, n = 140): string {
|
||||
return s.length > n ? `${s.slice(0, n)}…` : s;
|
||||
}
|
||||
|
||||
// ---------- dry mode (stubs the LLM) ----------
|
||||
|
||||
async function runDry(args: CliArgs): Promise<BenchReport> {
|
||||
const tasks = filterTasks(args.taskFilter);
|
||||
const results: RunResult[] = [];
|
||||
for (const task of tasks) {
|
||||
for (const mode of args.modes) {
|
||||
const db = cloneDb(task.initialDb);
|
||||
const tools = buildTools(task, db);
|
||||
// Execute each tool once with dummy-ish args just to prove wiring works.
|
||||
for (const t of tools) {
|
||||
try {
|
||||
await t.fn(stubArgs(t));
|
||||
} catch {
|
||||
/* dry mode — ignore tool errors */
|
||||
}
|
||||
}
|
||||
results.push({
|
||||
taskId: task.id,
|
||||
mode,
|
||||
pass: true,
|
||||
turns: 0,
|
||||
toolCalls: tools.length,
|
||||
cacheHitRatio: mode === "reasonix" ? 0.9 : 0.1,
|
||||
costUsd: 0,
|
||||
claudeEquivalentUsd: 0,
|
||||
promptTokens: 0,
|
||||
completionTokens: 0,
|
||||
truncated: false,
|
||||
finalAgentMessage: "[dry-run]",
|
||||
});
|
||||
console.log(`[${task.id}/${mode}] dry-run ok (${tools.length} tools wired)`);
|
||||
}
|
||||
}
|
||||
return {
|
||||
meta: buildMeta(args, tasks.length),
|
||||
results,
|
||||
};
|
||||
}
|
||||
|
||||
function stubArgs(t: {
|
||||
name: string;
|
||||
parameters?: { properties?: Record<string, unknown> };
|
||||
}): unknown {
|
||||
const props = t.parameters?.properties ?? {};
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const k of Object.keys(props)) out[k] = "stub";
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------- main ----------
|
||||
|
||||
function filterTasks(filter: string | null): TaskDefinition[] {
|
||||
if (!filter) return TASKS;
|
||||
const t = TASKS.find((x) => x.id === filter);
|
||||
if (!t) throw new Error(`unknown task: ${filter}`);
|
||||
return [t];
|
||||
}
|
||||
|
||||
function buildMeta(args: CliArgs, taskCount: number): BenchReport["meta"] {
|
||||
return {
|
||||
date: new Date().toISOString(),
|
||||
model: args.model,
|
||||
userSimModel: args.userSimModel,
|
||||
taskCount,
|
||||
repeatsPerTask: args.repeats,
|
||||
reasonixVersion: "0.2.2",
|
||||
};
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
|
||||
if (args.dry) {
|
||||
const report = await runDry(args);
|
||||
writeReport(report, args.outPath);
|
||||
return;
|
||||
}
|
||||
|
||||
const client = new DeepSeekClient();
|
||||
const tasks = filterTasks(args.taskFilter);
|
||||
|
||||
if (args.transcriptsDir) {
|
||||
mkdirSync(args.transcriptsDir, { recursive: true });
|
||||
}
|
||||
|
||||
const results: RunResult[] = [];
|
||||
for (const task of tasks) {
|
||||
for (let rep = 0; rep < args.repeats; rep++) {
|
||||
for (const mode of args.modes) {
|
||||
const db = cloneDb(task.initialDb);
|
||||
const transcript: Turn[] = [];
|
||||
|
||||
let transcriptStream: WriteStream | null = null;
|
||||
if (args.transcriptsDir) {
|
||||
const fname = `${task.id}.${mode}.r${rep + 1}.jsonl`;
|
||||
transcriptStream = openTranscriptFile(join(args.transcriptsDir, fname), {
|
||||
version: 1,
|
||||
source: `bench/${mode}`,
|
||||
model: args.model,
|
||||
task: task.id,
|
||||
mode,
|
||||
repeat: rep + 1,
|
||||
startedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
const ctx: RunContext = { client, task, db, transcript, args, transcriptStream };
|
||||
const runner = mode === "reasonix" ? runReasonix : runBaseline;
|
||||
const started = Date.now();
|
||||
let result: RunResult;
|
||||
try {
|
||||
result = await runner(ctx);
|
||||
} finally {
|
||||
transcriptStream?.end();
|
||||
}
|
||||
const elapsed = ((Date.now() - started) / 1000).toFixed(1);
|
||||
console.log(
|
||||
`[${task.id}/${mode}/r${rep + 1}] pass=${result.pass} turns=${result.turns} ` +
|
||||
`cache=${(result.cacheHitRatio * 100).toFixed(1)}% cost=$${result.costUsd.toFixed(
|
||||
6,
|
||||
)} (${elapsed}s)`,
|
||||
);
|
||||
results.push(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const report: BenchReport = { meta: buildMeta(args, tasks.length), results };
|
||||
writeReport(report, args.outPath);
|
||||
}
|
||||
|
||||
function writeReport(report: BenchReport, outPath: string | null): void {
|
||||
const path =
|
||||
outPath ??
|
||||
`benchmarks/tau-bench/results-${new Date().toISOString().replace(/[:.]/g, "-")}.json`;
|
||||
writeFileSync(path, `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
||||
console.log(`wrote ${path}`);
|
||||
}
|
||||
|
||||
function isMain(): boolean {
|
||||
const entry = process.argv[1];
|
||||
if (!entry) return false;
|
||||
return import.meta.url === pathToFileURL(entry).href;
|
||||
}
|
||||
|
||||
if (isMain()) {
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
export { main as runBench };
|
||||
@@ -0,0 +1,341 @@
|
||||
/**
|
||||
* Seed task set — retail-flavored, τ-bench-inspired.
|
||||
*
|
||||
* Why retail: success predicates are checkable by inspecting the end-state
|
||||
* DB (did the order's status change? was the refund logged?), which avoids
|
||||
* LLM-judge flakiness in the reproducibility report.
|
||||
*
|
||||
* Each task follows the same shape:
|
||||
* - a small starting DB
|
||||
* - 2–4 stateful tools (read / mutate rows)
|
||||
* - a user-sim persona that will hold back info until asked
|
||||
* - a check() that inspects the end-state DB
|
||||
*/
|
||||
|
||||
import { getRow, setField } from "./db.js";
|
||||
import type { TaskDefinition, ToolFactory, WorldState } from "./types.js";
|
||||
|
||||
const RETAIL_SYSTEM = `You are a retail support agent. Use the tools to help the user.
|
||||
Rules:
|
||||
- Always verify the user's identity (name + order id) before any mutation.
|
||||
- Never invent order ids or emails.
|
||||
- If a request is outside your tools, say so honestly.
|
||||
- Be concise.`;
|
||||
|
||||
// ---------- shared world-state factory ----------
|
||||
|
||||
function retailSeed(): WorldState {
|
||||
return {
|
||||
users: {
|
||||
u_ari: { name: "Ari Chen", email: "[email protected]" },
|
||||
u_bo: { name: "Bo Wang", email: "[email protected]" },
|
||||
u_cai: { name: "Cai Lin", email: "[email protected]" },
|
||||
u_dev: { name: "Dev Patel", email: "[email protected]" },
|
||||
},
|
||||
orders: {
|
||||
o_1001: {
|
||||
userId: "u_ari",
|
||||
status: "shipped",
|
||||
address: "1 Elm St, SF, CA 94110",
|
||||
item: "wool sweater M",
|
||||
price: 89.0,
|
||||
},
|
||||
o_1002: {
|
||||
userId: "u_bo",
|
||||
status: "processing",
|
||||
address: "22 Oak Rd, NYC, NY 10001",
|
||||
item: "running shoes 10",
|
||||
price: 140.0,
|
||||
},
|
||||
o_1003: {
|
||||
userId: "u_cai",
|
||||
status: "delivered",
|
||||
address: "9 Pine Ave, Austin, TX 78701",
|
||||
item: "desk lamp",
|
||||
price: 55.0,
|
||||
},
|
||||
o_1004: {
|
||||
userId: "u_dev",
|
||||
status: "processing",
|
||||
address: "4 Maple Ln, Seattle, WA 98101",
|
||||
item: "kettle",
|
||||
price: 45.0,
|
||||
},
|
||||
o_1005: {
|
||||
userId: "u_ari",
|
||||
status: "delivered",
|
||||
address: "1 Elm St, SF, CA 94110",
|
||||
item: "notebook pack",
|
||||
price: 22.0,
|
||||
},
|
||||
},
|
||||
refunds: {},
|
||||
};
|
||||
}
|
||||
|
||||
// ---------- tool factories (closed over a per-run db) ----------
|
||||
|
||||
const lookupOrder: ToolFactory = (db) => ({
|
||||
name: "lookup_order",
|
||||
description: "Look up an order by id. Returns { userId, status, address, item, price } or null.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: { orderId: { type: "string" } },
|
||||
required: ["orderId"],
|
||||
},
|
||||
fn: ({ orderId }: { orderId: string }) => {
|
||||
const row = getRow(db, "orders", orderId);
|
||||
return row ? { orderId, ...row } : { error: "order not found" };
|
||||
},
|
||||
});
|
||||
|
||||
const lookupUser: ToolFactory = (db) => ({
|
||||
name: "lookup_user",
|
||||
description: "Look up a user by id. Returns { name, email } or error.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: { userId: { type: "string" } },
|
||||
required: ["userId"],
|
||||
},
|
||||
fn: ({ userId }: { userId: string }) => {
|
||||
const row = getRow(db, "users", userId);
|
||||
return row ? { userId, ...row } : { error: "user not found" };
|
||||
},
|
||||
});
|
||||
|
||||
const updateAddress: ToolFactory = (db) => ({
|
||||
name: "update_address",
|
||||
description:
|
||||
"Update the shipping address on an order. Only allowed if status is 'processing'. Returns ok/error.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: {
|
||||
orderId: { type: "string" },
|
||||
address: { type: "string" },
|
||||
},
|
||||
required: ["orderId", "address"],
|
||||
},
|
||||
fn: ({ orderId, address }: { orderId: string; address: string }) => {
|
||||
const row = getRow(db, "orders", orderId);
|
||||
if (!row) return { error: "order not found" };
|
||||
if (row.status !== "processing") return { error: `cannot edit: status=${row.status}` };
|
||||
setField(db, "orders", orderId, "address", address);
|
||||
return { ok: true, orderId, newAddress: address };
|
||||
},
|
||||
});
|
||||
|
||||
const cancelOrder: ToolFactory = (db) => ({
|
||||
name: "cancel_order",
|
||||
description: "Cancel an order. Only allowed if status is 'processing'. Returns ok/error.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: { orderId: { type: "string" } },
|
||||
required: ["orderId"],
|
||||
},
|
||||
fn: ({ orderId }: { orderId: string }) => {
|
||||
const row = getRow(db, "orders", orderId);
|
||||
if (!row) return { error: "order not found" };
|
||||
if (row.status !== "processing") return { error: `cannot cancel: status=${row.status}` };
|
||||
setField(db, "orders", orderId, "status", "cancelled");
|
||||
return { ok: true, orderId, status: "cancelled" };
|
||||
},
|
||||
});
|
||||
|
||||
const refundOrder: ToolFactory = (db) => ({
|
||||
name: "refund_order",
|
||||
description:
|
||||
"Issue a refund on a delivered order. Records an entry in refunds. Returns ok/error.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: {
|
||||
orderId: { type: "string" },
|
||||
reason: { type: "string" },
|
||||
},
|
||||
required: ["orderId", "reason"],
|
||||
},
|
||||
fn: ({ orderId, reason }: { orderId: string; reason: string }) => {
|
||||
const row = getRow(db, "orders", orderId);
|
||||
if (!row) return { error: "order not found" };
|
||||
if (row.status !== "delivered") return { error: `cannot refund: status=${row.status}` };
|
||||
db.refunds[orderId] = { orderId, reason, amount: row.price };
|
||||
setField(db, "orders", orderId, "status", "refunded");
|
||||
return { ok: true, orderId, amount: row.price };
|
||||
},
|
||||
});
|
||||
|
||||
const listUserOrders: ToolFactory = (db) => ({
|
||||
name: "list_user_orders",
|
||||
description: "List every order belonging to a userId. Returns an array of orders.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: { userId: { type: "string" } },
|
||||
required: ["userId"],
|
||||
},
|
||||
fn: ({ userId }: { userId: string }) => {
|
||||
const all = db.orders ?? {};
|
||||
const out: unknown[] = [];
|
||||
for (const [orderId, row] of Object.entries(all)) {
|
||||
if (row.userId === userId) out.push({ orderId, ...row });
|
||||
}
|
||||
return out;
|
||||
},
|
||||
});
|
||||
|
||||
const ALL_TOOLS = [
|
||||
lookupOrder,
|
||||
lookupUser,
|
||||
updateAddress,
|
||||
cancelOrder,
|
||||
refundOrder,
|
||||
listUserOrders,
|
||||
];
|
||||
|
||||
// ---------- tasks ----------
|
||||
|
||||
export const TASKS: TaskDefinition[] = [
|
||||
{
|
||||
id: "t01_address_happy",
|
||||
description: "User wants to change the shipping address of a processing order.",
|
||||
systemPrompt: RETAIL_SYSTEM,
|
||||
tools: ALL_TOOLS,
|
||||
initialDb: retailSeed(),
|
||||
user: {
|
||||
style: "polite, brief",
|
||||
goal: "Change the shipping address on order o_1002 to '5 Birch Rd, NYC, NY 10001'.",
|
||||
knowns: {
|
||||
name: "Bo Wang",
|
||||
orderId: "o_1002",
|
||||
userId: "u_bo",
|
||||
newAddress: "5 Birch Rd, NYC, NY 10001",
|
||||
},
|
||||
},
|
||||
check: ({ db }) => db.orders.o_1002?.address === "5 Birch Rd, NYC, NY 10001",
|
||||
},
|
||||
{
|
||||
id: "t02_address_not_allowed",
|
||||
description: "User wants to change the address on a shipped order. Agent should refuse.",
|
||||
systemPrompt: RETAIL_SYSTEM,
|
||||
tools: ALL_TOOLS,
|
||||
initialDb: retailSeed(),
|
||||
user: {
|
||||
style: "insistent",
|
||||
goal: "Get the address on order o_1001 changed to '99 New St, SF, CA'.",
|
||||
knowns: {
|
||||
name: "Ari Chen",
|
||||
orderId: "o_1001",
|
||||
userId: "u_ari",
|
||||
newAddress: "99 New St, SF, CA",
|
||||
},
|
||||
},
|
||||
// Pass = DB unchanged (agent refused correctly).
|
||||
check: ({ db }) => db.orders.o_1001?.address === "1 Elm St, SF, CA 94110",
|
||||
},
|
||||
{
|
||||
id: "t03_cancel_processing",
|
||||
description: "User wants to cancel a processing order.",
|
||||
systemPrompt: RETAIL_SYSTEM,
|
||||
tools: ALL_TOOLS,
|
||||
initialDb: retailSeed(),
|
||||
user: {
|
||||
style: "apologetic",
|
||||
goal: "Cancel order o_1004.",
|
||||
knowns: { name: "Dev Patel", orderId: "o_1004", userId: "u_dev" },
|
||||
},
|
||||
check: ({ db }) => db.orders.o_1004?.status === "cancelled",
|
||||
},
|
||||
{
|
||||
id: "t04_refund_delivered",
|
||||
description: "User wants a refund on a delivered order with a valid reason.",
|
||||
systemPrompt: RETAIL_SYSTEM,
|
||||
tools: ALL_TOOLS,
|
||||
initialDb: retailSeed(),
|
||||
user: {
|
||||
style: "unhappy but reasonable",
|
||||
goal: "Get a refund on order o_1003 because the lamp arrived broken.",
|
||||
knowns: {
|
||||
name: "Cai Lin",
|
||||
orderId: "o_1003",
|
||||
userId: "u_cai",
|
||||
reason: "arrived broken",
|
||||
},
|
||||
},
|
||||
check: ({ db }) =>
|
||||
db.orders.o_1003?.status === "refunded" && db.refunds.o_1003?.amount === 55.0,
|
||||
},
|
||||
{
|
||||
id: "t05_refund_not_delivered",
|
||||
description:
|
||||
"User wants a refund on a processing order. Agent must NOT refund; cancelling the order is an acceptable alternative.",
|
||||
systemPrompt: RETAIL_SYSTEM,
|
||||
tools: ALL_TOOLS,
|
||||
initialDb: retailSeed(),
|
||||
user: {
|
||||
style: "demanding",
|
||||
goal: "Get a refund on order o_1002 right now.",
|
||||
knowns: { name: "Bo Wang", orderId: "o_1002", userId: "u_bo", reason: "changed my mind" },
|
||||
},
|
||||
// The essential requirement is "no refund on a processing order".
|
||||
// Status may stay `processing` (refusal) OR become `cancelled`
|
||||
// (alternative honored — a reasonable business response when the user
|
||||
// pushes back). We originally only accepted the refusal path, which
|
||||
// penalized agents for being helpful. Now we accept either, so long as
|
||||
// no refunds record is written for this order.
|
||||
check: ({ db }) => {
|
||||
const status = db.orders.o_1002?.status;
|
||||
const statusOk = status === "processing" || status === "cancelled";
|
||||
return statusOk && !db.refunds.o_1002;
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "t06_multi_order_lookup",
|
||||
description: "User wants to hear their full order history, then cancels one.",
|
||||
systemPrompt: RETAIL_SYSTEM,
|
||||
tools: ALL_TOOLS,
|
||||
initialDb: retailSeed(),
|
||||
user: {
|
||||
style: "chatty",
|
||||
goal: "First hear every order on file for user u_ari, then cancel any that are still processing.",
|
||||
knowns: { name: "Ari Chen", userId: "u_ari" },
|
||||
},
|
||||
// u_ari has o_1001 (shipped) and o_1005 (delivered) — nothing is processing.
|
||||
// Pass = both of Ari's orders unchanged.
|
||||
check: ({ db }) =>
|
||||
db.orders.o_1001?.status === "shipped" && db.orders.o_1005?.status === "delivered",
|
||||
},
|
||||
{
|
||||
id: "t07_wrong_identity",
|
||||
description:
|
||||
"User gives a mismatched name + order id. Agent should not mutate on unverified identity.",
|
||||
systemPrompt: RETAIL_SYSTEM,
|
||||
tools: ALL_TOOLS,
|
||||
initialDb: retailSeed(),
|
||||
user: {
|
||||
style: "confident but wrong",
|
||||
goal: "Claim to be 'Bo Wang' and ask to cancel order o_1004 (which actually belongs to Dev Patel).",
|
||||
knowns: {
|
||||
name: "Bo Wang",
|
||||
orderId: "o_1004",
|
||||
// intentionally inconsistent — agent should notice
|
||||
},
|
||||
},
|
||||
check: ({ db }) => db.orders.o_1004?.status === "processing",
|
||||
},
|
||||
{
|
||||
id: "t08_address_then_cancel",
|
||||
description: "User wants to update address, then decides to cancel instead.",
|
||||
systemPrompt: RETAIL_SYSTEM,
|
||||
tools: ALL_TOOLS,
|
||||
initialDb: retailSeed(),
|
||||
user: {
|
||||
style: "changes their mind mid-conversation",
|
||||
goal: "Start by asking to change the address on o_1004 to '7 Fir St, Seattle, WA 98101', then before confirming, switch to cancelling the order entirely.",
|
||||
knowns: {
|
||||
name: "Dev Patel",
|
||||
orderId: "o_1004",
|
||||
userId: "u_dev",
|
||||
newAddress: "7 Fir St, Seattle, WA 98101",
|
||||
},
|
||||
},
|
||||
check: ({ db }) => db.orders.o_1004?.status === "cancelled",
|
||||
},
|
||||
];
|
||||
@@ -0,0 +1,54 @@
|
||||
# Reference transcripts
|
||||
|
||||
These are the raw `.jsonl` transcripts from running task `t01_address_happy`
|
||||
under both baseline and Reasonix modes. They're committed so anyone can
|
||||
verify the cache-hit / cost claims *without running the bench*.
|
||||
|
||||
## Files
|
||||
|
||||
| file | what it is |
|
||||
|---|---|
|
||||
| `t01_address_happy.baseline.r1.jsonl` | Naive cache-hostile agent's record of the run |
|
||||
| `t01_address_happy.reasonix.r1.jsonl` | Reasonix's record of the same task |
|
||||
| `t01_address_happy.diff.md` | Output of `reasonix diff` on the two above |
|
||||
|
||||
## Verify for yourself
|
||||
|
||||
```bash
|
||||
# Install and build (or run from source via tsx)
|
||||
npm install
|
||||
|
||||
# Rebuild the summary — this does NOT call the API; it reads the JSONL.
|
||||
npx reasonix replay benchmarks/tau-bench/transcripts/t01_address_happy.reasonix.r1.jsonl
|
||||
|
||||
# Reproduce the diff:
|
||||
npx reasonix diff \
|
||||
benchmarks/tau-bench/transcripts/t01_address_happy.baseline.r1.jsonl \
|
||||
benchmarks/tau-bench/transcripts/t01_address_happy.reasonix.r1.jsonl \
|
||||
--label-a baseline --label-b reasonix
|
||||
```
|
||||
|
||||
Headline numbers from the committed run:
|
||||
|
||||
- cache hit: **45.9% → 93.9% (+48.0pp)**
|
||||
- cost: **$0.001192 → $0.000953 (−20.1%)**
|
||||
- Reasonix's prefix stayed byte-stable (1 distinct prefix hash) across all
|
||||
model calls; baseline's churned every call (untracked, by design).
|
||||
|
||||
## Regenerate
|
||||
|
||||
```bash
|
||||
export DEEPSEEK_API_KEY=sk-...
|
||||
rm benchmarks/tau-bench/transcripts/t01_* # clean
|
||||
npx tsx benchmarks/tau-bench/runner.ts \
|
||||
--task t01_address_happy \
|
||||
--transcripts-dir benchmarks/tau-bench/transcripts \
|
||||
--out /tmp/results.json
|
||||
npx reasonix diff \
|
||||
benchmarks/tau-bench/transcripts/t01_address_happy.baseline.r1.jsonl \
|
||||
benchmarks/tau-bench/transcripts/t01_address_happy.reasonix.r1.jsonl \
|
||||
--label-a baseline --label-b reasonix \
|
||||
--md benchmarks/tau-bench/transcripts/t01_address_happy.diff.md
|
||||
```
|
||||
|
||||
Cost: ~$0.003 per full regeneration.
|
||||
@@ -0,0 +1,11 @@
|
||||
{"role":"_meta","meta":{"version":1,"source":"bench/baseline","model":"deepseek-chat","task":"t01_address_happy","mode":"baseline","repeat":1,"startedAt":"2026-04-21T13:21:57.995Z"}}
|
||||
{"ts":"2026-04-21T13:22:00.112Z","turn":1,"role":"user","content":"Hi, I need to update the shipping address for my order o_1002."}
|
||||
{"ts":"2026-04-21T13:22:03.142Z","turn":1,"role":"assistant_final","content":"I can help you update the shipping address for your order. First, I need to verify your identity. Could you please provide your name and order ID? You've already given me order ID o_1002, but I also need your name to proceed with the address update.","usage":{"prompt_tokens":733,"completion_tokens":56,"total_tokens":789,"prompt_cache_hit_tokens":64,"prompt_cache_miss_tokens":669},"cost":0.00024671000000000003,"model":"deepseek-chat"}
|
||||
{"ts":"2026-04-21T13:22:04.688Z","turn":2,"role":"user","content":"My name is Bo Wang."}
|
||||
{"ts":"2026-04-21T13:22:10.465Z","turn":2,"role":"assistant_final","content":"Thank you, Bo Wang. Let me first look up your order o_1002 to verify the details and check its current status.","usage":{"prompt_tokens":799,"completion_tokens":74,"total_tokens":873,"prompt_cache_hit_tokens":64,"prompt_cache_miss_tokens":735},"cost":0.00028433000000000006,"model":"deepseek-chat"}
|
||||
{"ts":"2026-04-21T13:22:10.465Z","turn":2,"role":"tool","content":"{\"orderId\":\"o_1002\",\"userId\":\"u_bo\",\"status\":\"processing\",\"address\":\"22 Oak Rd, NYC, NY 10001\",\"item\":\"running shoes 10\",\"price\":140}","tool":"lookup_order","args":"{\"orderId\": \"o_1002\"}"}
|
||||
{"ts":"2026-04-21T13:22:10.465Z","turn":2,"role":"assistant_final","content":"Great! I can see your order o_1002 is currently in \"processing\" status, which means I can update the shipping address. What would you like the new shipping address to be?","usage":{"prompt_tokens":932,"completion_tokens":39,"total_tokens":971,"prompt_cache_hit_tokens":832,"prompt_cache_miss_tokens":100},"cost":0.00012814,"model":"deepseek-chat"}
|
||||
{"ts":"2026-04-21T13:22:11.830Z","turn":3,"role":"user","content":"Please change the shipping address to 5 Birch Rd, NYC, NY 10001."}
|
||||
{"ts":"2026-04-21T13:22:19.185Z","turn":3,"role":"assistant_final","content":"Perfect! I'll update the shipping address for your order o_1002 from \"22 Oak Rd, NYC, NY 10001\" to \"5 Birch Rd, NYC, NY 10001\".","usage":{"prompt_tokens":993,"completion_tokens":112,"total_tokens":1105,"prompt_cache_hit_tokens":64,"prompt_cache_miss_tokens":929},"cost":0.00037851,"model":"deepseek-chat"}
|
||||
{"ts":"2026-04-21T13:22:19.185Z","turn":3,"role":"tool","content":"{\"ok\":true,\"orderId\":\"o_1002\",\"newAddress\":\"5 Birch Rd, NYC, NY 10001\"}","tool":"update_address","args":"{\"orderId\": \"o_1002\", \"address\": \"5 Birch Rd, NYC, NY 10001\"}"}
|
||||
{"ts":"2026-04-21T13:22:19.185Z","turn":3,"role":"assistant_final","content":"Success! I've updated the shipping address for your order o_1002 to \"5 Birch Rd, NYC, NY 10001\". Your order is still in processing status, and the new address will be used for shipping. Is there anything else I can help you with?","usage":{"prompt_tokens":1149,"completion_tokens":56,"total_tokens":1205,"prompt_cache_hit_tokens":1088,"prompt_cache_miss_tokens":61},"cost":0.00015423000000000002,"model":"deepseek-chat"}
|
||||
@@ -0,0 +1,45 @@
|
||||
# Transcript diff: baseline vs reasonix
|
||||
|
||||
## Meta
|
||||
|
||||
| | baseline | reasonix |
|
||||
|---|---|---|
|
||||
| source | bench/baseline | bench/reasonix |
|
||||
| model | deepseek-chat | deepseek-chat |
|
||||
| task | t01_address_happy | t01_address_happy |
|
||||
| startedAt | 2026-04-21T13:21:57.995Z | 2026-04-21T13:22:21.292Z |
|
||||
|
||||
## Summary
|
||||
|
||||
| metric | baseline | reasonix | delta |
|
||||
|---|---:|---:|---:|
|
||||
| model calls | 5 | 6 | +1 |
|
||||
| user turns | 3 | 3 | 0 |
|
||||
| tool calls | 2 | 3 | +1 |
|
||||
| cache hit | 45.9% | 93.9% | **+48.0pp** |
|
||||
| cost (USD) | $0.001192 | $0.000953 | -20.1% |
|
||||
| prefix hashes | 0 | 1 | — |
|
||||
|
||||
## Turn-by-turn
|
||||
|
||||
| turn | kind | baseline tool calls | reasonix tool calls | note |
|
||||
|---:|:---:|---|---|---|
|
||||
| 1 | diverge | — | lookup_order | tool calls differ: A=[—] B=[lookup_order] |
|
||||
| 2 | diverge | lookup_order | lookup_user | tool calls differ: A=[lookup_order] B=[lookup_user] |
|
||||
| 3 | diverge | update_address | update_address | text similarity 68% |
|
||||
|
||||
## First divergence (turn 1)
|
||||
|
||||
tool calls differ: A=[—] B=[lookup_order]
|
||||
|
||||
**baseline:**
|
||||
|
||||
```
|
||||
I can help you update the shipping address for your order. First, I need to verify your identity. Could you please provide your name and order ID? You've already given me order ID o_1002, but I also need your name to proceed with the address update.
|
||||
```
|
||||
|
||||
**reasonix:**
|
||||
|
||||
```
|
||||
Great! I can see order o_1002 is in "processing" status, which means I can update the shipping address. However, I still need to verify your identity. Could you please provide your name so I can look up your user information using the user ID associated with this order?
|
||||
```
|
||||
@@ -0,0 +1,16 @@
|
||||
{"role":"_meta","meta":{"version":1,"source":"bench/reasonix","model":"deepseek-chat","task":"t01_address_happy","mode":"reasonix","repeat":1,"startedAt":"2026-04-21T13:22:21.292Z"}}
|
||||
{"ts":"2026-04-21T13:22:22.687Z","turn":1,"role":"user","content":"Hi, I need to update the shipping address for my order o_1002."}
|
||||
{"ts":"2026-04-21T13:22:27.201Z","turn":1,"role":"assistant_final","content":"I can help you update the shipping address for your order o_1002. First, I need to verify your identity. Could you please provide your name and order ID? I'll look up your order to check its current status and verify the details.","usage":{"prompt_tokens":713,"completion_tokens":98,"total_tokens":811,"prompt_cache_hit_tokens":704,"prompt_cache_miss_tokens":9},"cost":0.00015951000000000002,"model":"deepseek-chat","prefixHash":"fcb40cfd555bf805"}
|
||||
{"ts":"2026-04-21T13:22:27.201Z","turn":1,"role":"tool","content":"{\"orderId\":\"o_1002\",\"userId\":\"u_bo\",\"status\":\"processing\",\"address\":\"22 Oak Rd, NYC, NY 10001\",\"item\":\"running shoes 10\",\"price\":140}","tool":"lookup_order","args":"{\"orderId\": \"o_1002\"}"}
|
||||
{"ts":"2026-04-21T13:22:30.465Z","turn":1,"role":"assistant_final","content":"Great! I can see order o_1002 is in \"processing\" status, which means I can update the shipping address. However, I still need to verify your identity. Could you please provide your name so I can look up your user information using the user ID associated with this order?","usage":{"prompt_tokens":870,"completion_tokens":59,"total_tokens":929,"prompt_cache_hit_tokens":768,"prompt_cache_miss_tokens":102},"cost":0.00014620000000000003,"model":"deepseek-chat","prefixHash":"fcb40cfd555bf805"}
|
||||
{"ts":"2026-04-21T13:22:30.465Z","turn":1,"role":"done","content":"Great! I can see order o_1002 is in \"processing\" status, which means I can update the shipping address. However, I still need to verify your identity. Could you please provide your name so I can look up your user information using the user ID associated with this order?"}
|
||||
{"ts":"2026-04-21T13:22:31.801Z","turn":2,"role":"user","content":"My name is Bo Wang."}
|
||||
{"ts":"2026-04-21T13:22:34.932Z","turn":2,"role":"assistant_final","content":"Perfect, thank you Bo Wang. Let me verify your identity by looking up your user information.","usage":{"prompt_tokens":939,"completion_tokens":64,"total_tokens":1003,"prompt_cache_hit_tokens":896,"prompt_cache_miss_tokens":43},"cost":0.00014473,"model":"deepseek-chat","prefixHash":"fcb40cfd555bf805"}
|
||||
{"ts":"2026-04-21T13:22:34.932Z","turn":2,"role":"tool","content":"{\"userId\":\"u_bo\",\"name\":\"Bo Wang\",\"email\":\"[email protected]\"}","tool":"lookup_user","args":"{\"userId\": \"u_bo\"}"}
|
||||
{"ts":"2026-04-21T13:22:38.670Z","turn":2,"role":"assistant_final","content":"Excellent! Your identity has been verified, Bo Wang. I can see your order o_1002 is currently in \"processing\" status with the shipping address \"22 Oak Rd, NYC, NY 10001\".\n\nWhat would you like the new shipping address to be? Please provide the complete new address so I can update it for you.","usage":{"prompt_tokens":1039,"completion_tokens":68,"total_tokens":1107,"prompt_cache_hit_tokens":960,"prompt_cache_miss_tokens":79},"cost":0.00016333000000000002,"model":"deepseek-chat","prefixHash":"fcb40cfd555bf805"}
|
||||
{"ts":"2026-04-21T13:22:38.670Z","turn":2,"role":"done","content":"Excellent! Your identity has been verified, Bo Wang. I can see your order o_1002 is currently in \"processing\" status with the shipping address \"22 Oak Rd, NYC, NY 10001\".\n\nWhat would you like the new shipping address to be? Please provide the complete new address so I can update it for you."}
|
||||
{"ts":"2026-04-21T13:22:41.867Z","turn":3,"role":"user","content":"Please change it to 5 Birch Rd, NYC, NY 10001."}
|
||||
{"ts":"2026-04-21T13:22:45.479Z","turn":3,"role":"assistant_final","content":"","usage":{"prompt_tokens":1127,"completion_tokens":72,"total_tokens":1199,"prompt_cache_hit_tokens":1088,"prompt_cache_miss_tokens":39},"cost":0.00016589000000000003,"model":"deepseek-chat","prefixHash":"fcb40cfd555bf805"}
|
||||
{"ts":"2026-04-21T13:22:45.479Z","turn":3,"role":"tool","content":"{\"ok\":true,\"orderId\":\"o_1002\",\"newAddress\":\"5 Birch Rd, NYC, NY 10001\"}","tool":"update_address","args":"{\"orderId\": \"o_1002\", \"address\": \"5 Birch Rd, NYC, NY 10001\"}"}
|
||||
{"ts":"2026-04-21T13:22:48.394Z","turn":3,"role":"assistant_final","content":"Perfect! I've successfully updated the shipping address for your order o_1002. The new address is now \"5 Birch Rd, NYC, NY 10001\". \n\nYour order is still in processing status and will be shipped to this new address. Is there anything else I can help you with today?","usage":{"prompt_tokens":1243,"completion_tokens":62,"total_tokens":1305,"prompt_cache_hit_tokens":1152,"prompt_cache_miss_tokens":91},"cost":0.00017341000000000004,"model":"deepseek-chat","prefixHash":"fcb40cfd555bf805"}
|
||||
{"ts":"2026-04-21T13:22:48.394Z","turn":3,"role":"done","content":"Perfect! I've successfully updated the shipping address for your order o_1002. The new address is now \"5 Birch Rd, NYC, NY 10001\". \n\nYour order is still in processing status and will be shipped to this new address. Is there anything else I can help you with today?"}
|
||||
@@ -0,0 +1,98 @@
|
||||
/**
|
||||
* Task and result types for the tool-use eval harness.
|
||||
*
|
||||
* Scope note: this is NOT a full port of Sierra's τ-bench (airline+retail).
|
||||
* We mirror its *shape* — multi-turn tool-use with an LLM user simulator and
|
||||
* DB-end-state success predicates — so a later port can drop real tasks in
|
||||
* without changing the harness. Until then, see `tasks.ts` for our seed set.
|
||||
*/
|
||||
|
||||
import type { ToolDefinition } from "../../src/index.js";
|
||||
|
||||
/**
|
||||
* The mutable world state a task's tools operate on. Tasks deep-clone this
|
||||
* into a per-run snapshot so runs don't leak state across each other.
|
||||
*/
|
||||
export interface WorldState {
|
||||
[table: string]: Record<string, Record<string, unknown>>;
|
||||
}
|
||||
|
||||
export interface UserPersona {
|
||||
/** Who the user is roleplaying (e.g. "frustrated customer"). */
|
||||
style: string;
|
||||
/** The concrete goal. The user pursues this until it's met or clearly refused. */
|
||||
goal: string;
|
||||
/**
|
||||
* Facts the simulator may reveal when the agent asks. Keep tight — the
|
||||
* user shouldn't volunteer everything up front.
|
||||
*/
|
||||
knowns: Record<string, string>;
|
||||
}
|
||||
|
||||
/**
|
||||
* A tool factory. We take a factory (not a ToolDefinition directly) because
|
||||
* each run needs a fresh closure over its per-run WorldState — otherwise all
|
||||
* runs would share (and mutate) the same DB.
|
||||
*/
|
||||
export type ToolFactory = (db: WorldState) => ToolDefinition;
|
||||
|
||||
export interface TaskDefinition {
|
||||
id: string;
|
||||
/** One-line human description. Not shown to the model. */
|
||||
description: string;
|
||||
/** System prompt given to the agent. Kept small so cache-hit ratio is comparable. */
|
||||
systemPrompt: string;
|
||||
/** Tools built fresh per run against the run's DB snapshot. */
|
||||
tools: ToolFactory[];
|
||||
/** Initial DB snapshot. Deep-cloned per run. */
|
||||
initialDb: WorldState;
|
||||
/** Persona + goal for the LLM user simulator. */
|
||||
user: UserPersona;
|
||||
/** Max turns of (user → agent) before we give up and mark fail. */
|
||||
maxTurns?: number;
|
||||
/**
|
||||
* Success predicate. Given the end-state DB (and optionally the final
|
||||
* agent utterance), return true iff the task is considered solved.
|
||||
*/
|
||||
check: (ctx: { db: WorldState; finalAgentMessage: string; transcript: Turn[] }) => boolean;
|
||||
}
|
||||
|
||||
export interface Turn {
|
||||
role: "user" | "agent" | "tool";
|
||||
content: string;
|
||||
toolName?: string;
|
||||
}
|
||||
|
||||
export type RunMode = "baseline" | "reasonix";
|
||||
|
||||
export interface RunResult {
|
||||
taskId: string;
|
||||
mode: RunMode;
|
||||
pass: boolean;
|
||||
turns: number;
|
||||
toolCalls: number;
|
||||
cacheHitRatio: number;
|
||||
costUsd: number;
|
||||
claudeEquivalentUsd: number;
|
||||
promptTokens: number;
|
||||
completionTokens: number;
|
||||
/** True if the run aborted before the user sim decided to stop. */
|
||||
truncated: boolean;
|
||||
finalAgentMessage: string;
|
||||
errorMessage?: string;
|
||||
}
|
||||
|
||||
export interface BenchMeta {
|
||||
date: string;
|
||||
model: string;
|
||||
userSimModel: string;
|
||||
taskCount: number;
|
||||
repeatsPerTask: number;
|
||||
/** Reasonix version written into the report for reproducibility. */
|
||||
reasonixVersion: string;
|
||||
}
|
||||
|
||||
export interface BenchReport {
|
||||
meta: BenchMeta;
|
||||
results: RunResult[];
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
/**
|
||||
* LLM-backed user simulator for tool-use eval.
|
||||
*
|
||||
* A cheap V3 call that given (persona, goal, transcript so far) emits either
|
||||
* the user's next utterance or the literal sentinel `##STOP##` to end the
|
||||
* conversation. Kept intentionally small and low-temperature so dialogue
|
||||
* stays on-task and is replayable.
|
||||
*
|
||||
* We accept "some" non-determinism (DeepSeek samples at T=0.1) — the honest
|
||||
* way to deal with run-to-run drift is repeat-per-task in the runner, not
|
||||
* trying to pretend the sim is deterministic.
|
||||
*/
|
||||
|
||||
import type { ChatMessage, DeepSeekClient } from "../../src/index.js";
|
||||
import type { Turn, UserPersona } from "./types.js";
|
||||
|
||||
const SYS = `You are roleplaying a user contacting a retail support agent.
|
||||
Rules:
|
||||
- Stay in character; never break the fourth wall.
|
||||
- Never reveal you are an AI or mention any system.
|
||||
- Pursue the goal. Do not volunteer facts the agent hasn't asked for.
|
||||
- Keep replies to one or two sentences.
|
||||
- When the goal is clearly met OR clearly refused by the agent, output ONLY the literal token: ##STOP##
|
||||
- Do not output ##STOP## on your first message — give the agent a chance.`;
|
||||
|
||||
export interface UserSimOptions {
|
||||
model?: string;
|
||||
temperature?: number;
|
||||
}
|
||||
|
||||
export class UserSimulator {
|
||||
constructor(
|
||||
private client: DeepSeekClient,
|
||||
private persona: UserPersona,
|
||||
private opts: UserSimOptions = {},
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Given the transcript so far, produce the user's next line, or null if
|
||||
* the simulator decided the conversation is over.
|
||||
*/
|
||||
async next(transcript: Turn[]): Promise<string | null> {
|
||||
const knowns = JSON.stringify(this.persona.knowns, null, 2);
|
||||
const messages: ChatMessage[] = [
|
||||
{
|
||||
role: "system",
|
||||
content: `${SYS}\n\nCharacter: ${this.persona.style}\nGoal: ${this.persona.goal}\nFacts you may share when asked (don't volunteer):\n${knowns}`,
|
||||
},
|
||||
];
|
||||
|
||||
if (transcript.length === 0) {
|
||||
messages.push({
|
||||
role: "user",
|
||||
content:
|
||||
"Write your opening message to the support agent. One or two sentences. Do not dump all the facts.",
|
||||
});
|
||||
} else {
|
||||
messages.push({
|
||||
role: "user",
|
||||
content: `Here is the conversation so far (you are the USER).\n\n${transcriptToString(
|
||||
transcript,
|
||||
)}\n\nWrite ONLY your next user reply, or output ##STOP## if the goal is clearly met or clearly refused.`,
|
||||
});
|
||||
}
|
||||
|
||||
const resp = await this.client.chat({
|
||||
model: this.opts.model ?? "deepseek-chat",
|
||||
messages,
|
||||
temperature: this.opts.temperature ?? 0.1,
|
||||
maxTokens: 200,
|
||||
});
|
||||
const text = resp.content.trim();
|
||||
if (!text) return null;
|
||||
if (text === "##STOP##" || text.endsWith("##STOP##") || text.includes("##STOP##")) return null;
|
||||
return text;
|
||||
}
|
||||
}
|
||||
|
||||
function transcriptToString(turns: Turn[]): string {
|
||||
const lines: string[] = [];
|
||||
for (const t of turns) {
|
||||
if (t.role === "user") lines.push(`USER: ${t.content}`);
|
||||
else if (t.role === "agent") lines.push(`AGENT: ${t.content}`);
|
||||
else if (t.role === "tool")
|
||||
lines.push(`(tool ${t.toolName} returned: ${truncate(t.content, 200)})`);
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function truncate(s: string, n: number): string {
|
||||
return s.length > n ? `${s.slice(0, n)}…` : s;
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"$schema": "https://biomejs.dev/schemas/1.9.4/schema.json",
|
||||
"organizeImports": { "enabled": true },
|
||||
"formatter": {
|
||||
"enabled": true,
|
||||
"indentStyle": "space",
|
||||
"indentWidth": 2,
|
||||
"lineWidth": 100
|
||||
},
|
||||
"javascript": {
|
||||
"formatter": { "quoteStyle": "double", "semicolons": "always", "trailingCommas": "all" }
|
||||
},
|
||||
"linter": {
|
||||
"enabled": true,
|
||||
"rules": {
|
||||
"recommended": true,
|
||||
"style": {
|
||||
"noNonNullAssertion": "off",
|
||||
"useImportType": "warn"
|
||||
},
|
||||
"suspicious": {
|
||||
"noExplicitAny": "off"
|
||||
}
|
||||
}
|
||||
},
|
||||
"files": {
|
||||
"ignore": ["dist", "node_modules", "coverage", "*.d.ts"]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
# Reasonix Architecture
|
||||
|
||||
## Design philosophy
|
||||
|
||||
Reasonix is **opinionated, not general**. Every abstraction is justified by a
|
||||
DeepSeek-specific behavior or economic property. If it's generic, we don't
|
||||
ship it.
|
||||
|
||||
## The three pillars
|
||||
|
||||
### Pillar 1 — Cache-First Loop
|
||||
|
||||
**Problem.** DeepSeek bills cached input at ~10% of the miss rate. Automatic
|
||||
prefix caching activates only when the *exact* byte prefix of the previous
|
||||
request matches. Most agent loops reorder, rewrite, or inject fresh
|
||||
timestamps each turn — cache hit rate in practice: <20%.
|
||||
|
||||
**Solution.** Partition the context into three regions:
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────┐
|
||||
│ IMMUTABLE PREFIX │ ← fixed for session
|
||||
│ system + tool_specs + few_shots │ cache hit candidate
|
||||
├─────────────────────────────────────────┤
|
||||
│ APPEND-ONLY LOG │ ← grows monotonically
|
||||
│ [assistant₁][tool₁][assistant₂]... │ preserves prefix of prior turns
|
||||
├─────────────────────────────────────────┤
|
||||
│ VOLATILE SCRATCH │ ← reset each turn
|
||||
│ R1 thought, transient plan state │ never sent upstream
|
||||
└─────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Invariants:**
|
||||
1. Prefix is computed once per session, hashed, and pinned.
|
||||
2. Log entries are serialized in append order; no rewrites.
|
||||
3. Scratch is distilled via Pillar 2 before any information from it is folded
|
||||
into the log.
|
||||
|
||||
**Metric.** `prompt_cache_hit_tokens / (hit + miss)` exposed per-turn and
|
||||
aggregated per-session. This is the user-visible proof of Pillar 1's value.
|
||||
|
||||
### Pillar 2 — R1 Thought Harvesting *(v0.0.3, opt-in)*
|
||||
|
||||
**Problem.** R1 emits extensive `reasoning_content`. DeepSeek's own docs
|
||||
recommend *not* feeding it back to the next turn. Most frameworks display it
|
||||
to the user and discard. The planning signal inside is lost.
|
||||
|
||||
**Solution.** A two-stage process:
|
||||
|
||||
```
|
||||
R1 output → Harvester (V3, cheap) → TypedPlanState
|
||||
├─ subgoals: string[]
|
||||
├─ hypotheses: string[]
|
||||
├─ uncertainties: string[]
|
||||
└─ rejectedPaths: string[]
|
||||
```
|
||||
|
||||
The harvester is a cheap V3 call with a strict JSON schema. Output is
|
||||
validated at runtime. The typed state is queryable by the orchestrator — e.g.
|
||||
"if `uncertainties.length > 2`, trigger branch sampling."
|
||||
|
||||
### Branch-and-Select *(v0.0.5, opt-in, builds on Pillar 2)*
|
||||
|
||||
**Why now.** DeepSeek is cheap enough that running N=3 R1 samples is still
|
||||
cheaper than a single Claude call. What was research luxury (self-consistency
|
||||
sampling) becomes a practical default.
|
||||
|
||||
**How.** When `--branch N` (or `{ branch: N }` in code) is passed:
|
||||
|
||||
1. The turn is forced non-streaming (N samples must complete before we can
|
||||
compare).
|
||||
2. `runBranches` fires N `client.chat` calls in parallel, spreading
|
||||
temperatures across `[0, 1]` to diversify reasoning paths.
|
||||
3. Each sample's `reasoning_content` is piped through Pillar 2's harvest.
|
||||
4. The default selector picks the sample with the fewest `uncertainties`;
|
||||
ties are broken by shorter answer length (Occam).
|
||||
5. The loop emits a `branch_summary` alongside the winning `assistant_final`.
|
||||
|
||||
**Cost.** 3× R1 on a hard math problem ≈ \$0.02 — still dominates Claude on
|
||||
cost/quality for agentic work.
|
||||
|
||||
### Pillar 3 — Tool-Call Repair *(v0.0.1 ships complete)*
|
||||
|
||||
**Problem.** Empirical DeepSeek failure modes:
|
||||
- Tool-call JSON emitted inside `<think>`, missing from the final message.
|
||||
- Arguments dropped when schema has >10 params or deeply nested objects.
|
||||
- Same tool called repeatedly with identical args (call-storm).
|
||||
- Truncated JSON due to `max_tokens` hit mid-structure.
|
||||
|
||||
**Solution.** Four passes:
|
||||
|
||||
1. **`flatten`** — schemas with >10 leaf params or depth >2 are auto-detected
|
||||
on `ToolRegistry.register()` and presented to the model in dot-notation
|
||||
form. `dispatch()` re-nests the args before calling the user's `fn`.
|
||||
Opt out with `new ToolRegistry({ autoFlatten: false })`.
|
||||
2. **`scavenge`** — regex + JSON parser sweeps `reasoning_content` for any tool
|
||||
call the model forgot to emit in `tool_calls`.
|
||||
3. **`truncation`** — detect unbalanced JSON and repair by closing braces or
|
||||
requesting a continuation completion.
|
||||
4. **`storm`** — identical `(tool, args)` tuple within a sliding window →
|
||||
suppress the call, inject a reflection turn.
|
||||
|
||||
## Module layout
|
||||
|
||||
```
|
||||
src/
|
||||
├── client.ts # httpx-equivalent DeepSeek client (fetch + SSE)
|
||||
├── loop.ts # Pillar 1: Cache-First Loop (async iterator)
|
||||
├── harvest.ts # Pillar 2 stub (v0.0.1 surface only)
|
||||
├── repair/
|
||||
│ ├── index.ts # Pillar 3 pipeline
|
||||
│ ├── scavenge.ts
|
||||
│ ├── flatten.ts
|
||||
│ ├── truncation.ts
|
||||
│ └── storm.ts
|
||||
├── tools.ts # Tool registry + dispatch
|
||||
├── memory.ts # Prefix / Log / Scratch primitives
|
||||
├── telemetry.ts # Cost & cache-hit accounting
|
||||
├── types.ts # Shared type definitions
|
||||
├── index.ts # Library barrel export
|
||||
└── cli/
|
||||
├── index.ts # commander entry
|
||||
├── commands/ # chat, run, stats, version
|
||||
└── ui/ # Ink React components (App, StatsPanel, EventLog, PromptInput)
|
||||
```
|
||||
|
||||
## Roadmap
|
||||
|
||||
- **v0.0.1** — Pillar 1 end-to-end, Pillar 3 complete, Ink TUI, τ-bench scaffold.
|
||||
- **v0.0.2** — First-run key prompt, saved to `~/.reasonix/config.json`.
|
||||
- **v0.0.3** — Pillar 2 MVP (opt-in harvest), retry layer, TextInput fix.
|
||||
- **v0.0.4** — Schema flatten auto-applied in ToolRegistry (closes Pillar 3).
|
||||
- **v0.0.5** — Self-consistency branching (`--branch N`) driven by Pillar 2
|
||||
plan-state uncertainty count.
|
||||
- **v0.0.6** — Named session persistence (`--session <name>`), JSONL log
|
||||
stored under `~/.reasonix/sessions/`, restored into the append-only log
|
||||
on resume.
|
||||
- **v0.1** — τ-bench numbers published, streaming polish, transcript replay.
|
||||
- **v0.2** — Self-consistency / branch-budget sampling driven by plan state.
|
||||
- **v0.3** — MCP client, session persistence.
|
||||
|
||||
## Explicit non-goals
|
||||
|
||||
- Multi-agent orchestration.
|
||||
- RAG / vector retrieval.
|
||||
- Support for non-DeepSeek backends.
|
||||
- Web UI / SaaS.
|
||||
@@ -0,0 +1,25 @@
|
||||
/** Minimal example: one-shot, non-streaming. Needs DEEPSEEK_API_KEY. */
|
||||
import {
|
||||
CacheFirstLoop,
|
||||
DeepSeekClient,
|
||||
ImmutablePrefix,
|
||||
loadDotenv,
|
||||
} from "../src/index.js";
|
||||
|
||||
loadDotenv();
|
||||
|
||||
async function main() {
|
||||
const client = new DeepSeekClient();
|
||||
const prefix = new ImmutablePrefix({ system: "You are a concise assistant." });
|
||||
const loop = new CacheFirstLoop({ client, prefix, stream: false });
|
||||
|
||||
const answer = await loop.run("In one sentence, what is prompt caching?");
|
||||
console.log(answer);
|
||||
console.log("---");
|
||||
console.log(loop.stats.summary());
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,43 @@
|
||||
/** Tool-use example: register a calculator tool. Needs DEEPSEEK_API_KEY. */
|
||||
import {
|
||||
CacheFirstLoop,
|
||||
DeepSeekClient,
|
||||
ImmutablePrefix,
|
||||
ToolRegistry,
|
||||
loadDotenv,
|
||||
} from "../src/index.js";
|
||||
|
||||
loadDotenv();
|
||||
|
||||
const tools = new ToolRegistry();
|
||||
tools.register<{ a: number; b: number }, number>({
|
||||
name: "add",
|
||||
description: "Add two integers.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: {
|
||||
a: { type: "integer" },
|
||||
b: { type: "integer" },
|
||||
},
|
||||
required: ["a", "b"],
|
||||
},
|
||||
fn: ({ a, b }) => a + b,
|
||||
});
|
||||
|
||||
async function main() {
|
||||
const client = new DeepSeekClient();
|
||||
const prefix = new ImmutablePrefix({
|
||||
system: "You are a calculator assistant. Use the `add` tool for addition.",
|
||||
toolSpecs: tools.specs(),
|
||||
});
|
||||
const loop = new CacheFirstLoop({ client, prefix, tools });
|
||||
|
||||
const answer = await loop.run("What is 17 + 25?");
|
||||
console.log("answer:", answer);
|
||||
console.log("stats:", loop.stats.summary());
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
Generated
+3296
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,75 @@
|
||||
{
|
||||
"name": "reasonix",
|
||||
"version": "0.2.2",
|
||||
"description": "DeepSeek-native agent framework: cache-first loop, R1 thought harvesting, tool-call repair.",
|
||||
"type": "module",
|
||||
"bin": {
|
||||
"reasonix": "dist/cli/index.js"
|
||||
},
|
||||
"main": "./dist/index.js",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"import": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"README.md",
|
||||
"LICENSE"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "tsup",
|
||||
"dev": "tsx src/cli/index.ts",
|
||||
"chat": "tsx src/cli/index.ts chat",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest",
|
||||
"lint": "biome check src tests",
|
||||
"lint:fix": "biome check --write src tests",
|
||||
"format": "biome format --write src tests",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"prepublishOnly": "npm run lint && npm run typecheck && npm run test && npm run build"
|
||||
},
|
||||
"keywords": [
|
||||
"agent",
|
||||
"llm",
|
||||
"deepseek",
|
||||
"r1",
|
||||
"tool-use",
|
||||
"prompt-cache",
|
||||
"cli",
|
||||
"tui"
|
||||
],
|
||||
"author": "esengine",
|
||||
"license": "MIT",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/esengine/reasonix.git"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/esengine/reasonix/issues"
|
||||
},
|
||||
"homepage": "https://github.com/esengine/reasonix#readme",
|
||||
"engines": {
|
||||
"node": ">=20.10"
|
||||
},
|
||||
"dependencies": {
|
||||
"commander": "^12.1.0",
|
||||
"eventsource-parser": "^3.0.0",
|
||||
"ink": "^5.1.0",
|
||||
"ink-spinner": "^5.0.0",
|
||||
"ink-text-input": "^6.0.0",
|
||||
"react": "^18.3.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^1.9.4",
|
||||
"@types/node": "^22.9.0",
|
||||
"@types/react": "^18.3.12",
|
||||
"tsup": "^8.3.5",
|
||||
"tsx": "^4.19.2",
|
||||
"typescript": "^5.6.3",
|
||||
"vitest": "^2.1.5"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import { render } from "ink";
|
||||
import React, { useState } from "react";
|
||||
import { loadApiKey } from "../../config.js";
|
||||
import { loadDotenv } from "../../env.js";
|
||||
import { App } from "../ui/App.js";
|
||||
import { Setup } from "../ui/Setup.js";
|
||||
|
||||
export interface ChatOptions {
|
||||
model: string;
|
||||
system: string;
|
||||
transcript?: string;
|
||||
harvest?: boolean;
|
||||
branch?: number;
|
||||
session?: string;
|
||||
}
|
||||
|
||||
interface RootProps extends ChatOptions {
|
||||
initialKey: string | undefined;
|
||||
}
|
||||
|
||||
function Root({ initialKey, ...appProps }: RootProps) {
|
||||
const [key, setKey] = useState<string | undefined>(initialKey);
|
||||
if (!key) {
|
||||
return (
|
||||
<Setup
|
||||
onReady={(k) => {
|
||||
process.env.DEEPSEEK_API_KEY = k;
|
||||
setKey(k);
|
||||
}}
|
||||
/>
|
||||
);
|
||||
}
|
||||
// Ensure the loop's DeepSeekClient picks up the key when it lazy-instantiates.
|
||||
process.env.DEEPSEEK_API_KEY = key;
|
||||
return (
|
||||
<App
|
||||
model={appProps.model}
|
||||
system={appProps.system}
|
||||
transcript={appProps.transcript}
|
||||
harvest={appProps.harvest}
|
||||
branch={appProps.branch}
|
||||
session={appProps.session}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
export async function chatCommand(opts: ChatOptions): Promise<void> {
|
||||
loadDotenv();
|
||||
const initialKey = loadApiKey();
|
||||
const { waitUntilExit } = render(<Root initialKey={initialKey} {...opts} />, {
|
||||
exitOnCtrlC: true,
|
||||
});
|
||||
await waitUntilExit();
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { basename } from "node:path";
|
||||
import { render } from "ink";
|
||||
import React from "react";
|
||||
import { diffTranscripts, renderMarkdown, renderSummaryTable } from "../../diff.js";
|
||||
import { readTranscript } from "../../transcript.js";
|
||||
import { DiffApp } from "../ui/DiffApp.js";
|
||||
|
||||
export interface DiffOptions {
|
||||
a: string;
|
||||
b: string;
|
||||
mdPath?: string;
|
||||
labelA?: string;
|
||||
labelB?: string;
|
||||
/** Force stdout summary table (no Ink TUI). Auto when stdout isn't a TTY. */
|
||||
print?: boolean;
|
||||
/** Force the TUI even when stdout isn't a TTY (rare). */
|
||||
tui?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare two transcripts. Three output paths, picked in order:
|
||||
* - If --md is passed: write the markdown report. Also prints the stdout
|
||||
* summary so the user sees what was exported.
|
||||
* - If --print, no TTY, or --md (see above): stdout summary table.
|
||||
* - Otherwise: interactive Ink TUI with split-pane + n/N divergence jump.
|
||||
*/
|
||||
export async function diffCommand(opts: DiffOptions): Promise<void> {
|
||||
const aParsed = readTranscript(opts.a);
|
||||
const bParsed = readTranscript(opts.b);
|
||||
|
||||
const report = diffTranscripts(
|
||||
{ label: opts.labelA ?? basename(opts.a), parsed: aParsed },
|
||||
{ label: opts.labelB ?? basename(opts.b), parsed: bParsed },
|
||||
);
|
||||
|
||||
const wantMarkdown = !!opts.mdPath;
|
||||
const wantPrint = opts.print || !process.stdout.isTTY;
|
||||
const wantTui = opts.tui || (!wantPrint && !wantMarkdown);
|
||||
|
||||
if (wantMarkdown) {
|
||||
// Markdown export implies the user wants an artifact, not a TUI.
|
||||
// Still echo the stdout summary to confirm the action.
|
||||
console.log(renderSummaryTable(report));
|
||||
const md = renderMarkdown(report);
|
||||
writeFileSync(opts.mdPath!, md, "utf8");
|
||||
console.log(`\nmarkdown report written to ${opts.mdPath}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (wantTui) {
|
||||
const { waitUntilExit } = render(React.createElement(DiffApp, { report }), {
|
||||
exitOnCtrlC: true,
|
||||
});
|
||||
await waitUntilExit();
|
||||
return;
|
||||
}
|
||||
|
||||
// stdout fallback (piped, --print, or non-TTY)
|
||||
console.log(renderSummaryTable(report));
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
import { render } from "ink";
|
||||
import React from "react";
|
||||
import { groupRecordsByTurn, replayFromFile } from "../../replay.js";
|
||||
import type { TranscriptRecord } from "../../transcript.js";
|
||||
import { ReplayApp } from "../ui/ReplayApp.js";
|
||||
|
||||
export interface ReplayOptions {
|
||||
path: string;
|
||||
head?: number;
|
||||
tail?: number;
|
||||
/** Force stdout pretty-print mode (no Ink TUI). Also auto-enabled when stdout is not a TTY. */
|
||||
print?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replay a transcript. Two modes:
|
||||
* - Interactive TUI (default when stdout is a TTY): Ink-based j/k navigation.
|
||||
* - Stdout pretty-print (--print, or when stdout is piped): one-shot text
|
||||
* dump + summary. Kept working so shell pipes, CI logs, and `less`
|
||||
* workflows still behave sensibly.
|
||||
*/
|
||||
export async function replayCommand(opts: ReplayOptions): Promise<void> {
|
||||
const wantPrint =
|
||||
opts.print || !process.stdout.isTTY || opts.head !== undefined || opts.tail !== undefined;
|
||||
if (wantPrint) {
|
||||
printReplay(opts);
|
||||
return;
|
||||
}
|
||||
|
||||
const { parsed } = replayFromFile(opts.path);
|
||||
const pages = groupRecordsByTurn(parsed.records);
|
||||
const { waitUntilExit } = render(React.createElement(ReplayApp, { meta: parsed.meta, pages }), {
|
||||
exitOnCtrlC: true,
|
||||
});
|
||||
await waitUntilExit();
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// stdout pretty-print path (original behavior, preserved for piping / CI)
|
||||
|
||||
function printReplay(opts: ReplayOptions): void {
|
||||
const { parsed, stats } = replayFromFile(opts.path);
|
||||
|
||||
if (parsed.meta) {
|
||||
const m = parsed.meta;
|
||||
const bits: string[] = [`source=${m.source}`];
|
||||
if (m.model) bits.push(`model=${m.model}`);
|
||||
if (m.task) bits.push(`task=${m.task}`);
|
||||
if (m.mode) bits.push(`mode=${m.mode}`);
|
||||
if (m.repeat !== undefined) bits.push(`repeat=${m.repeat}`);
|
||||
bits.push(`started=${m.startedAt}`);
|
||||
console.log(`[meta] ${bits.join(" ")}`);
|
||||
console.log("");
|
||||
}
|
||||
|
||||
const records = sliceRecords(parsed.records, opts);
|
||||
for (const rec of records) {
|
||||
renderRecord(rec);
|
||||
}
|
||||
|
||||
console.log("");
|
||||
console.log("── summary ─────────────────────────────────────────");
|
||||
console.log(`model calls: ${stats.turns}`);
|
||||
console.log(`user turns: ${stats.userTurns}`);
|
||||
console.log(`tool calls: ${stats.toolCalls}`);
|
||||
console.log(`cache hit: ${(stats.cacheHitRatio * 100).toFixed(1)}%`);
|
||||
console.log(`cost: $${stats.totalCostUsd.toFixed(6)}`);
|
||||
console.log(`claude equivalent: $${stats.claudeEquivalentUsd.toFixed(6)}`);
|
||||
console.log(`savings vs claude: ${stats.savingsVsClaudePct.toFixed(1)}%`);
|
||||
console.log(`models: ${stats.models.join(", ") || "—"}`);
|
||||
console.log(`prefix hashes: ${stats.prefixHashes.length} distinct`);
|
||||
if (stats.prefixHashes.length === 1) {
|
||||
console.log(` (byte-stable prefix: ${stats.prefixHashes[0]?.slice(0, 16)}…)`);
|
||||
} else if (stats.prefixHashes.length > 1) {
|
||||
console.log(" (prefix churned — cache-hostile session)");
|
||||
}
|
||||
}
|
||||
|
||||
function sliceRecords(records: TranscriptRecord[], opts: ReplayOptions): TranscriptRecord[] {
|
||||
if (opts.head !== undefined && opts.head > 0) return records.slice(0, opts.head);
|
||||
if (opts.tail !== undefined && opts.tail > 0) return records.slice(-opts.tail);
|
||||
return records;
|
||||
}
|
||||
|
||||
function renderRecord(rec: TranscriptRecord): void {
|
||||
const turn = `[t${rec.turn}]`;
|
||||
if (rec.role === "user") {
|
||||
console.log(`${turn} USER: ${oneLine(rec.content)}`);
|
||||
} else if (rec.role === "assistant_final") {
|
||||
const cost = rec.cost !== undefined ? ` $${rec.cost.toFixed(6)}` : "";
|
||||
const cache =
|
||||
rec.usage &&
|
||||
(rec.usage.prompt_cache_hit_tokens !== undefined ||
|
||||
rec.usage.prompt_cache_miss_tokens !== undefined)
|
||||
? (() => {
|
||||
const hit = rec.usage!.prompt_cache_hit_tokens ?? 0;
|
||||
const miss = rec.usage!.prompt_cache_miss_tokens ?? 0;
|
||||
const total = hit + miss;
|
||||
return total > 0 ? ` cache=${((hit / total) * 100).toFixed(1)}%` : "";
|
||||
})()
|
||||
: "";
|
||||
console.log(`${turn} AGENT:${cost}${cache} ${oneLine(rec.content)}`);
|
||||
} else if (rec.role === "tool") {
|
||||
const args = rec.args ? ` args=${oneLine(rec.args, 80)}` : "";
|
||||
console.log(`${turn} TOOL ${rec.tool ?? "?"}:${args} → ${oneLine(rec.content, 120)}`);
|
||||
} else if (rec.role === "error") {
|
||||
console.log(`${turn} ERROR: ${rec.error ?? rec.content}`);
|
||||
} else if (rec.role === "done") {
|
||||
// Suppress — visually noisy, not informative in replay.
|
||||
} else {
|
||||
console.log(`${turn} ${rec.role}: ${oneLine(rec.content)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function oneLine(s: string, max = 200): string {
|
||||
const collapsed = s.replace(/\s+/g, " ").trim();
|
||||
return collapsed.length > max ? `${collapsed.slice(0, max)}…` : collapsed;
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
import { stdin, stdout } from "node:process";
|
||||
import { createInterface } from "node:readline/promises";
|
||||
import { defaultConfigPath, isPlausibleKey, loadApiKey, saveApiKey } from "../../config.js";
|
||||
import { loadDotenv } from "../../env.js";
|
||||
import { CacheFirstLoop, DeepSeekClient, ImmutablePrefix } from "../../index.js";
|
||||
|
||||
export interface RunOptions {
|
||||
task: string;
|
||||
model: string;
|
||||
system: string;
|
||||
}
|
||||
|
||||
async function ensureApiKey(): Promise<string> {
|
||||
const existing = loadApiKey();
|
||||
if (existing) return existing;
|
||||
|
||||
if (!stdin.isTTY) {
|
||||
process.stderr.write(
|
||||
"DEEPSEEK_API_KEY is not set and stdin is not a TTY (cannot prompt).\n" +
|
||||
"Set the env var, or run `reasonix chat` once interactively to save a key.\n",
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
process.stdout.write(
|
||||
"DeepSeek API key not configured.\nGet one at https://platform.deepseek.com/api_keys\n",
|
||||
);
|
||||
const rl = createInterface({ input: stdin, output: stdout });
|
||||
try {
|
||||
while (true) {
|
||||
const answer = (await rl.question("API key › ")).trim();
|
||||
if (!answer) continue;
|
||||
if (!isPlausibleKey(answer)) {
|
||||
process.stdout.write("Invalid format. Keys start with 'sk-' and are 30+ chars.\n");
|
||||
continue;
|
||||
}
|
||||
saveApiKey(answer);
|
||||
process.stdout.write(`Saved to ${defaultConfigPath()}\n\n`);
|
||||
return answer;
|
||||
}
|
||||
} finally {
|
||||
rl.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function runCommand(opts: RunOptions): Promise<void> {
|
||||
loadDotenv();
|
||||
const apiKey = await ensureApiKey();
|
||||
process.env.DEEPSEEK_API_KEY = apiKey;
|
||||
|
||||
const client = new DeepSeekClient();
|
||||
const prefix = new ImmutablePrefix({ system: opts.system });
|
||||
const loop = new CacheFirstLoop({ client, prefix, model: opts.model });
|
||||
|
||||
for await (const ev of loop.step(opts.task)) {
|
||||
if (ev.role === "assistant_delta" && ev.content) process.stdout.write(ev.content);
|
||||
if (ev.role === "tool") process.stdout.write(`\n[tool ${ev.toolName}] ${ev.content}\n`);
|
||||
if (ev.role === "error") process.stderr.write(`\n[error] ${ev.error}\n`);
|
||||
if (ev.role === "done") process.stdout.write("\n");
|
||||
}
|
||||
const s = loop.stats.summary();
|
||||
process.stdout.write(
|
||||
`\n— turns:${s.turns} cache:${(s.cacheHitRatio * 100).toFixed(1)}% ` +
|
||||
`cost:$${s.totalCostUsd.toFixed(6)} save-vs-claude:${s.savingsVsClaudePct.toFixed(1)}%\n`,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
|
||||
export interface StatsOptions {
|
||||
transcript: string;
|
||||
}
|
||||
|
||||
export function statsCommand(opts: StatsOptions): void {
|
||||
if (!existsSync(opts.transcript)) {
|
||||
console.error(`no such transcript: ${opts.transcript}`);
|
||||
process.exit(1);
|
||||
}
|
||||
const lines = readFileSync(opts.transcript, "utf8").split(/\r?\n/).filter(Boolean);
|
||||
let assistantTurns = 0;
|
||||
let toolCalls = 0;
|
||||
let lastTurn = 0;
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const rec = JSON.parse(line);
|
||||
if (rec.role === "assistant_final") assistantTurns++;
|
||||
if (rec.role === "tool") toolCalls++;
|
||||
if (typeof rec.turn === "number") lastTurn = Math.max(lastTurn, rec.turn);
|
||||
} catch {
|
||||
/* skip */
|
||||
}
|
||||
}
|
||||
console.log(`transcript: ${opts.transcript}`);
|
||||
console.log(`assistant turns: ${assistantTurns}`);
|
||||
console.log(`tool invocations: ${toolCalls}`);
|
||||
console.log(`last turn index: ${lastTurn}`);
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
import { VERSION } from "../../index.js";
|
||||
|
||||
export function versionCommand(): void {
|
||||
console.log(`reasonix ${VERSION}`);
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
import { Command } from "commander";
|
||||
import { VERSION } from "../index.js";
|
||||
import { chatCommand } from "./commands/chat.js";
|
||||
import { diffCommand } from "./commands/diff.js";
|
||||
import { replayCommand } from "./commands/replay.js";
|
||||
import { runCommand } from "./commands/run.js";
|
||||
import { statsCommand } from "./commands/stats.js";
|
||||
import { versionCommand } from "./commands/version.js";
|
||||
|
||||
const DEFAULT_SYSTEM =
|
||||
"You are Reasonix, a helpful DeepSeek-powered assistant. Be concise and accurate. Use tools when available.";
|
||||
|
||||
const program = new Command();
|
||||
program
|
||||
.name("reasonix")
|
||||
.description("DeepSeek-native agent framework — built for cache hits and cheap tokens.")
|
||||
.version(VERSION);
|
||||
|
||||
program
|
||||
.command("chat")
|
||||
.description("Interactive Ink TUI with live cache/cost panel.")
|
||||
.option("-m, --model <id>", "DeepSeek model id", "deepseek-chat")
|
||||
.option("-s, --system <prompt>", "System prompt (pinned in the immutable prefix)", DEFAULT_SYSTEM)
|
||||
.option("--transcript <path>", "Write a JSONL transcript to this path")
|
||||
.option(
|
||||
"--harvest",
|
||||
"Extract typed plan state from R1 reasoning (Pillar 2, adds a cheap V3 call per turn)",
|
||||
)
|
||||
.option(
|
||||
"--branch <n>",
|
||||
"Self-consistency: run N parallel samples per turn and pick the most confident (disables streaming; enables harvest)",
|
||||
(v) => Number.parseInt(v, 10),
|
||||
)
|
||||
.option(
|
||||
"--session <name>",
|
||||
"Use a named session (default: 'default'). Resume the same session next time.",
|
||||
)
|
||||
.option("--no-session", "Disable session persistence for this run (ephemeral chat)")
|
||||
.action(async (opts) => {
|
||||
// Default behavior: every chat is auto-saved to a session named 'default'
|
||||
// and auto-resumed next launch. Pass --no-session to opt out, or
|
||||
// --session <name> to use a different session.
|
||||
let session: string | undefined;
|
||||
if (opts.session === false) {
|
||||
session = undefined; // --no-session
|
||||
} else if (typeof opts.session === "string" && opts.session.length > 0) {
|
||||
session = opts.session;
|
||||
} else {
|
||||
session = "default";
|
||||
}
|
||||
await chatCommand({
|
||||
model: opts.model,
|
||||
system: opts.system,
|
||||
transcript: opts.transcript,
|
||||
harvest: !!opts.harvest,
|
||||
branch: Number.isFinite(opts.branch) && opts.branch > 1 ? opts.branch : undefined,
|
||||
session,
|
||||
});
|
||||
});
|
||||
|
||||
program
|
||||
.command("run <task>")
|
||||
.description("Run a single task non-interactively, streaming output.")
|
||||
.option("-m, --model <id>", "DeepSeek model id", "deepseek-chat")
|
||||
.option("-s, --system <prompt>", "System prompt", DEFAULT_SYSTEM)
|
||||
.action(async (task: string, opts) => {
|
||||
await runCommand({ task, model: opts.model, system: opts.system });
|
||||
});
|
||||
|
||||
program
|
||||
.command("stats <transcript>")
|
||||
.description("Summarize a JSONL transcript produced by `reasonix chat --transcript`.")
|
||||
.action((transcript: string) => {
|
||||
statsCommand({ transcript });
|
||||
});
|
||||
|
||||
program
|
||||
.command("replay <transcript>")
|
||||
.description(
|
||||
"Interactive Ink TUI to scrub through a transcript + rebuild its session summary (cost, cache, prefix stability). No API calls.",
|
||||
)
|
||||
.option("--print", "Dump to stdout instead of mounting the TUI (auto when piped)")
|
||||
.option("--head <n>", "stdout mode only — show first N records", (v) => Number.parseInt(v, 10))
|
||||
.option("--tail <n>", "stdout mode only — show last N records", (v) => Number.parseInt(v, 10))
|
||||
.action(async (transcript: string, opts) => {
|
||||
await replayCommand({
|
||||
path: transcript,
|
||||
print: !!opts.print,
|
||||
head: Number.isFinite(opts.head) ? opts.head : undefined,
|
||||
tail: Number.isFinite(opts.tail) ? opts.tail : undefined,
|
||||
});
|
||||
});
|
||||
|
||||
program
|
||||
.command("diff <a> <b>")
|
||||
.description(
|
||||
"Compare two transcripts in a split-pane Ink TUI (default) or stdout table. Use n/N to jump across divergences.",
|
||||
)
|
||||
.option("--md <path>", "Write a markdown report (blog-ready) to this path")
|
||||
.option("--print", "Force stdout table instead of the TUI (auto when piped)")
|
||||
.option("--tui", "Force the TUI even when piped (rare)")
|
||||
.option("--label-a <label>", "Display label for transcript A (default: filename)")
|
||||
.option("--label-b <label>", "Display label for transcript B (default: filename)")
|
||||
.action(async (a: string, b: string, opts) => {
|
||||
await diffCommand({
|
||||
a,
|
||||
b,
|
||||
mdPath: opts.md,
|
||||
labelA: opts.labelA,
|
||||
labelB: opts.labelB,
|
||||
print: !!opts.print,
|
||||
tui: !!opts.tui,
|
||||
});
|
||||
});
|
||||
|
||||
program.command("version").description("Print Reasonix version.").action(versionCommand);
|
||||
|
||||
program.parseAsync(process.argv).catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,294 @@
|
||||
import type { WriteStream } from "node:fs";
|
||||
import { Box, Static, Text, useApp } from "ink";
|
||||
import React, { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { CacheFirstLoop, DeepSeekClient, ImmutablePrefix } from "../../index.js";
|
||||
import type { LoopEvent } from "../../loop.js";
|
||||
import type { SessionSummary } from "../../telemetry.js";
|
||||
import { openTranscriptFile, recordFromLoopEvent, writeRecord } from "../../transcript.js";
|
||||
import { type DisplayEvent, EventRow } from "./EventLog.js";
|
||||
import { PromptInput } from "./PromptInput.js";
|
||||
import { StatsPanel } from "./StatsPanel.js";
|
||||
import { handleSlash, parseSlash } from "./slash.js";
|
||||
|
||||
export interface AppProps {
|
||||
model: string;
|
||||
system: string;
|
||||
transcript?: string;
|
||||
harvest?: boolean;
|
||||
branch?: number;
|
||||
session?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Throttle interval in ms. We flush streaming deltas at most this often to
|
||||
* avoid re-rendering the whole UI on every single token from DeepSeek.
|
||||
* 60ms ≈ 16Hz, fast enough to feel live, slow enough to not thrash Ink.
|
||||
*/
|
||||
const FLUSH_INTERVAL_MS = 60;
|
||||
|
||||
interface StreamingState {
|
||||
id: string;
|
||||
text: string;
|
||||
reasoning: string;
|
||||
}
|
||||
|
||||
export function App({ model, system, transcript, harvest, branch, session }: AppProps) {
|
||||
const { exit } = useApp();
|
||||
const [historical, setHistorical] = useState<DisplayEvent[]>([]);
|
||||
const [streaming, setStreaming] = useState<DisplayEvent | null>(null);
|
||||
const [input, setInput] = useState("");
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [summary, setSummary] = useState<SessionSummary>({
|
||||
turns: 0,
|
||||
totalCostUsd: 0,
|
||||
claudeEquivalentUsd: 0,
|
||||
savingsVsClaudePct: 0,
|
||||
cacheHitRatio: 0,
|
||||
});
|
||||
|
||||
const transcriptRef = useRef<WriteStream | null>(null);
|
||||
if (transcript && !transcriptRef.current) {
|
||||
transcriptRef.current = openTranscriptFile(transcript, {
|
||||
version: 1,
|
||||
source: "reasonix chat",
|
||||
model,
|
||||
startedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
transcriptRef.current?.end();
|
||||
};
|
||||
}, []);
|
||||
|
||||
const loopRef = useRef<CacheFirstLoop | null>(null);
|
||||
const loop = useMemo(() => {
|
||||
if (loopRef.current) return loopRef.current;
|
||||
const client = new DeepSeekClient();
|
||||
const prefix = new ImmutablePrefix({ system });
|
||||
const l = new CacheFirstLoop({ client, prefix, model, harvest, branch, session });
|
||||
loopRef.current = l;
|
||||
return l;
|
||||
}, [model, system, harvest, branch, session]);
|
||||
|
||||
// Surface a one-time banner about session state on first mount.
|
||||
const sessionBannerShown = useRef(false);
|
||||
useEffect(() => {
|
||||
if (sessionBannerShown.current) return;
|
||||
sessionBannerShown.current = true;
|
||||
if (!session) {
|
||||
setHistorical((prev) => [
|
||||
...prev,
|
||||
{
|
||||
id: `sys-session-${Date.now()}`,
|
||||
role: "info",
|
||||
text: "▸ ephemeral chat (no session persistence) — drop --no-session to enable",
|
||||
},
|
||||
]);
|
||||
} else if (loop.resumedMessageCount > 0) {
|
||||
setHistorical((prev) => [
|
||||
...prev,
|
||||
{
|
||||
id: `sys-resume-${Date.now()}`,
|
||||
role: "info",
|
||||
text: `▸ resumed session "${session}" with ${loop.resumedMessageCount} prior messages · /forget to start over · /sessions to list`,
|
||||
},
|
||||
]);
|
||||
} else {
|
||||
setHistorical((prev) => [
|
||||
...prev,
|
||||
{
|
||||
id: `sys-newsession-${Date.now()}`,
|
||||
role: "info",
|
||||
text: `▸ session "${session}" (new) — auto-saved as you chat · /forget to delete · /sessions to list`,
|
||||
},
|
||||
]);
|
||||
}
|
||||
}, [session, loop]);
|
||||
|
||||
const prefixHash = loop.prefix.fingerprint;
|
||||
|
||||
const writeTranscript = useCallback(
|
||||
(ev: LoopEvent) => {
|
||||
const stream = transcriptRef.current;
|
||||
if (!stream) return;
|
||||
writeRecord(stream, recordFromLoopEvent(ev, { model, prefixHash }));
|
||||
},
|
||||
[model, prefixHash],
|
||||
);
|
||||
|
||||
const handleSubmit = useCallback(
|
||||
async (raw: string) => {
|
||||
const text = raw.trim();
|
||||
if (!text || busy) return;
|
||||
setInput("");
|
||||
const slash = parseSlash(text);
|
||||
if (slash) {
|
||||
const result = handleSlash(slash.cmd, slash.args, loop);
|
||||
if (result.exit) {
|
||||
transcriptRef.current?.end();
|
||||
exit();
|
||||
return;
|
||||
}
|
||||
if (result.clear) {
|
||||
setHistorical([]);
|
||||
return;
|
||||
}
|
||||
if (result.info) {
|
||||
setHistorical((prev) => [
|
||||
...prev,
|
||||
{
|
||||
id: `sys-${Date.now()}`,
|
||||
role: "info",
|
||||
text: result.info!,
|
||||
},
|
||||
]);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// User message is immutable — push to Static immediately.
|
||||
setHistorical((prev) => [...prev, { id: `u-${Date.now()}`, role: "user", text }]);
|
||||
|
||||
const assistantId = `a-${Date.now()}`;
|
||||
// Refs are the source of truth for accumulated streaming text; the React
|
||||
// state copy below is only for rendering and gets updated on flush.
|
||||
const streamRef: StreamingState = { id: assistantId, text: "", reasoning: "" };
|
||||
const contentBuf = { current: "" };
|
||||
const reasoningBuf = { current: "" };
|
||||
|
||||
setStreaming({ id: assistantId, role: "assistant", text: "", streaming: true });
|
||||
setBusy(true);
|
||||
|
||||
const flush = () => {
|
||||
if (!contentBuf.current && !reasoningBuf.current) return;
|
||||
streamRef.text += contentBuf.current;
|
||||
streamRef.reasoning += reasoningBuf.current;
|
||||
contentBuf.current = "";
|
||||
reasoningBuf.current = "";
|
||||
setStreaming({
|
||||
id: assistantId,
|
||||
role: "assistant",
|
||||
text: streamRef.text,
|
||||
reasoning: streamRef.reasoning || undefined,
|
||||
streaming: true,
|
||||
});
|
||||
};
|
||||
const timer = setInterval(flush, FLUSH_INTERVAL_MS);
|
||||
|
||||
try {
|
||||
for await (const ev of loop.step(text)) {
|
||||
writeTranscript(ev);
|
||||
if (ev.role === "assistant_delta") {
|
||||
if (ev.content) contentBuf.current += ev.content;
|
||||
if (ev.reasoningDelta) reasoningBuf.current += ev.reasoningDelta;
|
||||
} else if (ev.role === "branch_start") {
|
||||
setStreaming({
|
||||
id: assistantId,
|
||||
role: "assistant",
|
||||
text: "",
|
||||
streaming: true,
|
||||
branchProgress: ev.branchProgress,
|
||||
});
|
||||
} else if (ev.role === "branch_progress") {
|
||||
// Live-update the streaming slot with per-sample completion info.
|
||||
setStreaming({
|
||||
id: assistantId,
|
||||
role: "assistant",
|
||||
text: "",
|
||||
streaming: true,
|
||||
branchProgress: ev.branchProgress,
|
||||
});
|
||||
} else if (ev.role === "branch_done") {
|
||||
// Intermediate: branching finished but assistant_final not yet emitted.
|
||||
// Keep streaming state alive; actual render happens on assistant_final.
|
||||
} else if (ev.role === "assistant_final") {
|
||||
flush();
|
||||
const repairNote = ev.repair ? describeRepair(ev.repair) : "";
|
||||
setStreaming(null);
|
||||
setHistorical((prev) => [
|
||||
...prev,
|
||||
{
|
||||
id: assistantId,
|
||||
role: "assistant",
|
||||
text: ev.content || streamRef.text,
|
||||
reasoning: streamRef.reasoning || undefined,
|
||||
planState: ev.planState,
|
||||
branch: ev.branch,
|
||||
stats: ev.stats,
|
||||
repair: repairNote || undefined,
|
||||
streaming: false,
|
||||
},
|
||||
]);
|
||||
} else if (ev.role === "tool") {
|
||||
flush();
|
||||
setHistorical((prev) => [
|
||||
...prev,
|
||||
{
|
||||
id: `t-${Date.now()}-${Math.random()}`,
|
||||
role: "tool",
|
||||
text: ev.content,
|
||||
toolName: ev.toolName,
|
||||
},
|
||||
]);
|
||||
} else if (ev.role === "error") {
|
||||
setHistorical((prev) => [
|
||||
...prev,
|
||||
{ id: `e-${Date.now()}`, role: "error", text: ev.error ?? ev.content },
|
||||
]);
|
||||
}
|
||||
}
|
||||
flush();
|
||||
} finally {
|
||||
clearInterval(timer);
|
||||
setStreaming(null);
|
||||
setSummary(loop.stats.summary());
|
||||
setBusy(false);
|
||||
}
|
||||
},
|
||||
[busy, exit, loop, writeTranscript],
|
||||
);
|
||||
|
||||
return (
|
||||
<Box flexDirection="column">
|
||||
<StatsPanel
|
||||
summary={summary}
|
||||
model={loop.model}
|
||||
prefixHash={prefixHash}
|
||||
harvestOn={loop.harvestEnabled}
|
||||
branchBudget={loop.branchOptions.budget}
|
||||
/>
|
||||
<Static items={historical}>{(item) => <EventRow key={item.id} event={item} />}</Static>
|
||||
{streaming ? (
|
||||
<Box marginY={1}>
|
||||
<EventRow event={streaming} />
|
||||
</Box>
|
||||
) : null}
|
||||
<PromptInput value={input} onChange={setInput} onSubmit={handleSubmit} disabled={busy} />
|
||||
<CommandStrip />
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
function CommandStrip() {
|
||||
return (
|
||||
<Box paddingX={2}>
|
||||
<Text dimColor>
|
||||
/help · /preset {"<fast|smart|max>"} · /sessions · /model · /harvest · /branch · /clear ·
|
||||
/exit
|
||||
</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
function describeRepair(repair: {
|
||||
scavenged: number;
|
||||
truncationsFixed: number;
|
||||
stormsBroken: number;
|
||||
}): string {
|
||||
const parts: string[] = [];
|
||||
if (repair.scavenged) parts.push(`scavenged ${repair.scavenged}`);
|
||||
if (repair.truncationsFixed) parts.push(`repaired ${repair.truncationsFixed} truncation`);
|
||||
if (repair.stormsBroken) parts.push(`broke ${repair.stormsBroken} storm`);
|
||||
return parts.length ? `[repair] ${parts.join(", ")}` : "";
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
/**
|
||||
* Ink TUI for `reasonix diff`. Split-pane: A on the left, B on the right,
|
||||
* shared cursor. Header shows aggregate deltas; footer shows the current
|
||||
* pair's divergence note (if any) + key cheat sheet.
|
||||
*
|
||||
* j/k moves the cursor by one turn; n/N jumps to the next/prev divergent
|
||||
* turn — which is the whole point of a diff tool. Quit with q.
|
||||
*
|
||||
* Pure navigation lives in src/diff.ts (findNextDivergence / findPrevDivergence).
|
||||
*/
|
||||
|
||||
import { Box, Static, Text, useApp, useInput } from "ink";
|
||||
import React, { useState } from "react";
|
||||
import {
|
||||
type DiffReport,
|
||||
type TurnPair,
|
||||
findNextDivergence,
|
||||
findPrevDivergence,
|
||||
} from "../../diff.js";
|
||||
import { RecordView } from "./RecordView.js";
|
||||
|
||||
export interface DiffAppProps {
|
||||
report: DiffReport;
|
||||
}
|
||||
|
||||
export function DiffApp({ report }: DiffAppProps) {
|
||||
const { exit } = useApp();
|
||||
const maxIdx = Math.max(0, report.pairs.length - 1);
|
||||
// Start at the first divergence when one exists — that's the user's most
|
||||
// likely destination. Falls back to idx 0 for fully-matching diffs.
|
||||
const initialIdx = report.firstDivergenceTurn
|
||||
? report.pairs.findIndex((p) => p.turn === report.firstDivergenceTurn)
|
||||
: 0;
|
||||
const [idx, setIdx] = useState(Math.max(0, initialIdx));
|
||||
|
||||
useInput((input, key) => {
|
||||
if (input === "q" || (key.ctrl && input === "c")) {
|
||||
exit();
|
||||
return;
|
||||
}
|
||||
if (input === "j" || key.downArrow || input === " " || key.return) {
|
||||
setIdx((i) => Math.min(maxIdx, i + 1));
|
||||
} else if (input === "k" || key.upArrow) {
|
||||
setIdx((i) => Math.max(0, i - 1));
|
||||
} else if (input === "g") {
|
||||
setIdx(0);
|
||||
} else if (input === "G") {
|
||||
setIdx(maxIdx);
|
||||
} else if (input === "n") {
|
||||
const next = findNextDivergence(report.pairs, idx);
|
||||
if (next !== -1) setIdx(next);
|
||||
} else if (input === "N" || input === "p") {
|
||||
const prev = findPrevDivergence(report.pairs, idx);
|
||||
if (prev !== -1) setIdx(prev);
|
||||
}
|
||||
});
|
||||
|
||||
const pair = report.pairs[idx];
|
||||
|
||||
return (
|
||||
<Box flexDirection="column">
|
||||
<DiffHeader report={report} />
|
||||
|
||||
<Box marginTop={1} paddingX={1} justifyContent="space-between">
|
||||
<Text color="cyan" bold>
|
||||
turn {pair?.turn ?? "?"} ({idx + 1} / {report.pairs.length})
|
||||
</Text>
|
||||
<Text>{pair ? <KindBadge kind={pair.kind} /> : null}</Text>
|
||||
</Box>
|
||||
|
||||
<Box flexDirection="row" marginTop={1}>
|
||||
<Pane label={report.a.label} headerColor="blue" records={paneRecords(pair, "a")} />
|
||||
<Pane label={report.b.label} headerColor="magenta" records={paneRecords(pair, "b")} />
|
||||
</Box>
|
||||
|
||||
{pair?.divergenceNote ? (
|
||||
<Box marginTop={1} paddingX={1}>
|
||||
<Text color="yellow">★ </Text>
|
||||
<Text>{pair.divergenceNote}</Text>
|
||||
</Box>
|
||||
) : null}
|
||||
|
||||
<Box marginTop={1} paddingX={1} borderStyle="single" borderColor="gray">
|
||||
<Text dimColor>
|
||||
<Text bold>j</Text>/<Text bold>↓</Text> next · <Text bold>k</Text>/<Text bold>↑</Text>{" "}
|
||||
prev · <Text bold>n</Text> next-diverge · <Text bold>N</Text>/<Text bold>p</Text>{" "}
|
||||
prev-diverge · <Text bold>g</Text>/<Text bold>G</Text> first/last · <Text bold>q</Text>{" "}
|
||||
quit
|
||||
</Text>
|
||||
</Box>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
function DiffHeader({ report }: { report: DiffReport }) {
|
||||
const a = report.a;
|
||||
const b = report.b;
|
||||
|
||||
const cacheDelta = b.stats.cacheHitRatio - a.stats.cacheHitRatio;
|
||||
const costDelta =
|
||||
a.stats.totalCostUsd > 0
|
||||
? ((b.stats.totalCostUsd - a.stats.totalCostUsd) / a.stats.totalCostUsd) * 100
|
||||
: 0;
|
||||
|
||||
// Prefix stability one-liner (same logic as the stdout summary).
|
||||
const aStable = a.stats.prefixHashes.length <= 1;
|
||||
const bStable = b.stats.prefixHashes.length <= 1;
|
||||
let prefixLine: string | null = null;
|
||||
if (aStable !== bStable) {
|
||||
const stableLabel = aStable ? report.a.label : report.b.label;
|
||||
const churnLabel = aStable ? report.b.label : report.a.label;
|
||||
const churnCount = aStable ? b.stats.prefixHashes.length : a.stats.prefixHashes.length;
|
||||
prefixLine = `${stableLabel} stayed byte-stable; ${churnLabel} churned ${churnCount} distinct prefixes.`;
|
||||
} else if (a.stats.prefixHashes[0] && a.stats.prefixHashes[0] === b.stats.prefixHashes[0]) {
|
||||
prefixLine = `shared prefix hash ${a.stats.prefixHashes[0].slice(0, 12)}… — cache delta attributable to log stability, not prompt change.`;
|
||||
}
|
||||
|
||||
return (
|
||||
<Box flexDirection="column" borderStyle="round" borderColor="cyan" paddingX={1}>
|
||||
<Box justifyContent="space-between">
|
||||
<Text>
|
||||
<Text color="cyan" bold>
|
||||
reasonix diff
|
||||
</Text>
|
||||
<Text dimColor> · A=</Text>
|
||||
<Text color="blue">{a.label}</Text>
|
||||
<Text dimColor> vs B=</Text>
|
||||
<Text color="magenta">{b.label}</Text>
|
||||
</Text>
|
||||
<Text dimColor>{report.pairs.length} turns aligned</Text>
|
||||
</Box>
|
||||
|
||||
<Box marginTop={1} gap={3}>
|
||||
<Text>
|
||||
<Text dimColor>cache </Text>
|
||||
<Text>{(a.stats.cacheHitRatio * 100).toFixed(1)}%</Text>
|
||||
<Text dimColor> → </Text>
|
||||
<Text>{(b.stats.cacheHitRatio * 100).toFixed(1)}%</Text>
|
||||
<Text color={cacheDelta >= 0 ? "green" : "red"} bold>
|
||||
{" "}
|
||||
{cacheDelta >= 0 ? "+" : ""}
|
||||
{(cacheDelta * 100).toFixed(1)}pp
|
||||
</Text>
|
||||
</Text>
|
||||
<Text>
|
||||
<Text dimColor>cost </Text>
|
||||
<Text>${a.stats.totalCostUsd.toFixed(6)}</Text>
|
||||
<Text dimColor> → </Text>
|
||||
<Text>${b.stats.totalCostUsd.toFixed(6)}</Text>
|
||||
<Text color={costDelta <= 0 ? "green" : "red"} bold>
|
||||
{" "}
|
||||
{costDelta >= 0 ? "+" : ""}
|
||||
{costDelta.toFixed(1)}%
|
||||
</Text>
|
||||
</Text>
|
||||
<Text>
|
||||
<Text dimColor>model calls </Text>
|
||||
<Text>
|
||||
{a.stats.turns} → {b.stats.turns}
|
||||
</Text>
|
||||
</Text>
|
||||
</Box>
|
||||
|
||||
{prefixLine ? (
|
||||
<Box marginTop={1}>
|
||||
<Text dimColor italic>
|
||||
{prefixLine}
|
||||
</Text>
|
||||
</Box>
|
||||
) : null}
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
function Pane({
|
||||
label,
|
||||
headerColor,
|
||||
records,
|
||||
}: {
|
||||
label: string;
|
||||
headerColor: "blue" | "magenta";
|
||||
records: TurnPair["aTools"];
|
||||
}) {
|
||||
return (
|
||||
<Box
|
||||
flexDirection="column"
|
||||
flexGrow={1}
|
||||
paddingX={1}
|
||||
borderStyle="single"
|
||||
borderColor={headerColor}
|
||||
>
|
||||
<Text color={headerColor} bold>
|
||||
{label}
|
||||
</Text>
|
||||
{records.length === 0 ? (
|
||||
<Box marginTop={1}>
|
||||
<Text dimColor italic>
|
||||
(no records on this side for this turn)
|
||||
</Text>
|
||||
</Box>
|
||||
) : (
|
||||
<Static items={records.map((rec, i) => ({ key: `${label}-${i}`, rec }))}>
|
||||
{({ key, rec }) => <RecordView key={key} rec={rec} compact />}
|
||||
</Static>
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
function KindBadge({ kind }: { kind: TurnPair["kind"] }) {
|
||||
if (kind === "match") {
|
||||
return <Text color="green">✓ match</Text>;
|
||||
}
|
||||
if (kind === "diverge") {
|
||||
return <Text color="yellow">★ diverge</Text>;
|
||||
}
|
||||
if (kind === "only_in_a") {
|
||||
return <Text color="blue">← only in A</Text>;
|
||||
}
|
||||
return <Text color="magenta">→ only in B</Text>;
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
function paneRecords(pair: TurnPair | undefined, side: "a" | "b"): TurnPair["aTools"] {
|
||||
if (!pair) return [];
|
||||
const tools = side === "a" ? pair.aTools : pair.bTools;
|
||||
const assistant = side === "a" ? pair.aAssistant : pair.bAssistant;
|
||||
const out: TurnPair["aTools"] = [...tools];
|
||||
if (assistant) out.push(assistant);
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
import { Box, Text } from "ink";
|
||||
import React, { useEffect, useState } from "react";
|
||||
import { type TypedPlanState, isPlanStateEmpty } from "../../harvest.js";
|
||||
import type { BranchProgress, BranchSummary } from "../../loop.js";
|
||||
import type { TurnStats } from "../../telemetry.js";
|
||||
import { Markdown } from "./markdown.js";
|
||||
|
||||
export type DisplayRole = "user" | "assistant" | "tool" | "system" | "error" | "info";
|
||||
|
||||
export interface DisplayEvent {
|
||||
id: string;
|
||||
role: DisplayRole;
|
||||
text: string;
|
||||
reasoning?: string;
|
||||
planState?: TypedPlanState;
|
||||
branch?: BranchSummary;
|
||||
branchProgress?: BranchProgress;
|
||||
toolName?: string;
|
||||
stats?: TurnStats;
|
||||
repair?: string;
|
||||
streaming?: boolean;
|
||||
}
|
||||
|
||||
export const EventRow = React.memo(function EventRow({ event }: { event: DisplayEvent }) {
|
||||
if (event.role === "user") {
|
||||
return (
|
||||
<Box>
|
||||
<Text bold color="cyan">
|
||||
you ›{" "}
|
||||
</Text>
|
||||
<Text>{event.text}</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
if (event.role === "assistant") {
|
||||
if (event.streaming) return <StreamingAssistant event={event} />;
|
||||
return (
|
||||
<Box flexDirection="column" marginTop={1}>
|
||||
<Box>
|
||||
<Text bold color="green">
|
||||
assistant
|
||||
</Text>
|
||||
</Box>
|
||||
{event.branch ? <BranchBlock branch={event.branch} /> : null}
|
||||
{event.reasoning ? <ReasoningBlock reasoning={event.reasoning} /> : null}
|
||||
{!isPlanStateEmpty(event.planState) ? (
|
||||
<PlanStateBlock planState={event.planState!} />
|
||||
) : null}
|
||||
{event.text ? <Markdown text={event.text} /> : <Text dimColor>(no content)</Text>}
|
||||
{event.stats ? <StatsLine stats={event.stats} /> : null}
|
||||
{event.repair ? <Text color="magenta">{event.repair}</Text> : null}
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
if (event.role === "tool") {
|
||||
return (
|
||||
<Box flexDirection="column" marginTop={1}>
|
||||
<Text color="yellow">{`tool<${event.toolName ?? "?"}> →`}</Text>
|
||||
<Text dimColor> {truncate(event.text, 400)}</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
if (event.role === "error") {
|
||||
return (
|
||||
<Box marginTop={1}>
|
||||
<Text color="red" bold>
|
||||
error{" "}
|
||||
</Text>
|
||||
<Text color="red">{event.text}</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
if (event.role === "info") {
|
||||
return (
|
||||
<Box>
|
||||
<Text dimColor>{event.text}</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<Box>
|
||||
<Text>{event.text}</Text>
|
||||
</Box>
|
||||
);
|
||||
});
|
||||
|
||||
function BranchBlock({ branch }: { branch: BranchSummary }) {
|
||||
const per = branch.uncertainties
|
||||
.map((u, i) => {
|
||||
const marker = i === branch.chosenIndex ? "▸" : " ";
|
||||
const t = (branch.temperatures[i] ?? 0).toFixed(1);
|
||||
return `${marker} #${i} T=${t} u=${u}`;
|
||||
})
|
||||
.join(" ");
|
||||
return (
|
||||
<Box>
|
||||
<Text color="blue">
|
||||
{"🔀 branched "}
|
||||
<Text bold>{branch.budget}</Text>
|
||||
{` samples → picked #${branch.chosenIndex} `}
|
||||
<Text dimColor>{per}</Text>
|
||||
</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
function PlanStateBlock({ planState }: { planState: TypedPlanState }) {
|
||||
const lines: Array<[string, string[]]> = [];
|
||||
if (planState.subgoals.length) lines.push(["subgoals", planState.subgoals]);
|
||||
if (planState.hypotheses.length) lines.push(["hypotheses", planState.hypotheses]);
|
||||
if (planState.uncertainties.length) lines.push(["uncertainties", planState.uncertainties]);
|
||||
if (planState.rejectedPaths.length) lines.push(["rejected", planState.rejectedPaths]);
|
||||
return (
|
||||
<Box flexDirection="column" marginBottom={1}>
|
||||
{lines.map(([label, items]) => (
|
||||
<Text key={label} color="magenta">
|
||||
{"‹ "}
|
||||
<Text bold>{label}</Text>
|
||||
{` (${items.length}): ${items.join(" · ")}`}
|
||||
</Text>
|
||||
))}
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
function ReasoningBlock({ reasoning }: { reasoning: string }) {
|
||||
const max = 220;
|
||||
const flat = reasoning.replace(/\s+/g, " ").trim();
|
||||
const preview =
|
||||
flat.length <= max ? flat : `${flat.slice(0, max)}… (+${flat.length - max} chars)`;
|
||||
return (
|
||||
<Box marginBottom={1}>
|
||||
<Text dimColor italic>
|
||||
{"↳ thinking: "}
|
||||
{preview}
|
||||
</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compact progress view rendered while a turn is still streaming. We keep
|
||||
* this to a fixed ~3-line footprint so the dynamic region never scrolls past
|
||||
* the terminal viewport and leaves artifacts in scrollback.
|
||||
*/
|
||||
function Elapsed() {
|
||||
const [s, setS] = useState(0);
|
||||
useEffect(() => {
|
||||
const start = Date.now();
|
||||
const id = setInterval(() => setS(Math.floor((Date.now() - start) / 1000)), 1000);
|
||||
return () => clearInterval(id);
|
||||
}, []);
|
||||
const mm = String(Math.floor(s / 60)).padStart(2, "0");
|
||||
const ss = String(s % 60).padStart(2, "0");
|
||||
return <Text dimColor>{`${mm}:${ss}`}</Text>;
|
||||
}
|
||||
|
||||
function StreamingAssistant({ event }: { event: DisplayEvent }) {
|
||||
if (event.branchProgress) {
|
||||
const p = event.branchProgress;
|
||||
// completed=0 means we've just started; no sample has finished yet.
|
||||
if (p.completed === 0) {
|
||||
return (
|
||||
<Box flexDirection="column" marginTop={1}>
|
||||
<Box>
|
||||
<Text bold color="green">
|
||||
assistant{" "}
|
||||
</Text>
|
||||
<Text color="blue">
|
||||
🔀 launching {p.total} parallel samples (R1 thinking in parallel)…{" "}
|
||||
</Text>
|
||||
<Elapsed />
|
||||
</Box>
|
||||
<Text dimColor>{" "}spread across T=0.0/0.5/1.0 · typical wait 30-90s for reasoner</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
const pct = Math.round((p.completed / p.total) * 100);
|
||||
return (
|
||||
<Box flexDirection="column" marginTop={1}>
|
||||
<Box>
|
||||
<Text bold color="green">
|
||||
assistant{" "}
|
||||
</Text>
|
||||
<Text color="blue">
|
||||
🔀 branching {p.completed}/{p.total} ({pct}%){" "}
|
||||
</Text>
|
||||
<Elapsed />
|
||||
</Box>
|
||||
<Text dimColor>
|
||||
{" latest #"}
|
||||
{p.latestIndex}
|
||||
{" T="}
|
||||
{p.latestTemperature.toFixed(1)}
|
||||
{" u="}
|
||||
{p.latestUncertainties}
|
||||
{p.completed < p.total ? " · waiting for other samples…" : " · selecting winner…"}
|
||||
</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
const tail = lastLine(event.text, 140);
|
||||
const reasoningTail = event.reasoning ? lastLine(event.reasoning, 120) : "";
|
||||
return (
|
||||
<Box flexDirection="column" marginTop={1}>
|
||||
<Box>
|
||||
<Text bold color="green">
|
||||
assistant{" "}
|
||||
</Text>
|
||||
<Text dimColor>
|
||||
(streaming · {event.text.length}
|
||||
{event.reasoning ? ` + think ${event.reasoning.length}` : ""} chars){" "}
|
||||
</Text>
|
||||
<Elapsed />
|
||||
</Box>
|
||||
{reasoningTail ? (
|
||||
<Text dimColor italic>
|
||||
↳ thinking: {reasoningTail}
|
||||
</Text>
|
||||
) : null}
|
||||
{tail ? (
|
||||
<Text dimColor>▸ {tail}</Text>
|
||||
) : (
|
||||
<Text dimColor italic>
|
||||
{" (waiting for first token…)"}
|
||||
</Text>
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
function lastLine(s: string, maxChars: number): string {
|
||||
const flat = s.replace(/\s+/g, " ").trim();
|
||||
if (!flat) return "";
|
||||
return flat.length <= maxChars ? flat : `…${flat.slice(-maxChars)}`;
|
||||
}
|
||||
|
||||
function StatsLine({ stats }: { stats: TurnStats }) {
|
||||
const hit = (stats.cacheHitRatio * 100).toFixed(1);
|
||||
return (
|
||||
<Text dimColor>
|
||||
{" ↳ cache "}
|
||||
{hit}
|
||||
{"% · tokens "}
|
||||
{stats.usage.promptTokens}
|
||||
{"→"}
|
||||
{stats.usage.completionTokens}
|
||||
{" · $"}
|
||||
{stats.cost.toFixed(6)}
|
||||
</Text>
|
||||
);
|
||||
}
|
||||
|
||||
function truncate(s: string, max: number): string {
|
||||
return s.length <= max ? s : `${s.slice(0, max)}… (+${s.length - max} chars)`;
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { Box, Text } from "ink";
|
||||
import TextInput from "ink-text-input";
|
||||
import React from "react";
|
||||
|
||||
export interface PromptInputProps {
|
||||
value: string;
|
||||
onChange: (v: string) => void;
|
||||
onSubmit: (v: string) => void;
|
||||
disabled?: boolean;
|
||||
placeholder?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep `<TextInput>` mounted at all times and use its `focus` prop to gate
|
||||
* input. Conditionally rendering it (mount / unmount between turns) loses
|
||||
* the stdin raw-mode claim on some terminals, which silently drops
|
||||
* keystrokes after the first turn finishes.
|
||||
*/
|
||||
export function PromptInput({
|
||||
value,
|
||||
onChange,
|
||||
onSubmit,
|
||||
disabled,
|
||||
placeholder,
|
||||
}: PromptInputProps) {
|
||||
const effectivePlaceholder = disabled
|
||||
? (placeholder ?? "…waiting for response…")
|
||||
: (placeholder ?? "type a message, or /command");
|
||||
return (
|
||||
<Box borderStyle="round" borderColor={disabled ? "gray" : "cyan"} paddingX={1}>
|
||||
<Text bold color={disabled ? "gray" : "cyan"}>
|
||||
you ›{" "}
|
||||
</Text>
|
||||
<TextInput
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
onSubmit={onSubmit}
|
||||
focus={!disabled}
|
||||
placeholder={effectivePlaceholder}
|
||||
/>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* Shared renderer for a single TranscriptRecord. Used by ReplayApp and
|
||||
* DiffApp — both need the same visual grammar (user cyan, assistant green,
|
||||
* tool yellow, error red, cache badge colored by threshold) so transcripts
|
||||
* look consistent wherever they're displayed.
|
||||
*
|
||||
* Kept small on purpose: no streaming/branch/planState paths (those are
|
||||
* live-chat concerns and never appear in replayed transcripts).
|
||||
*/
|
||||
|
||||
import { Box, Text } from "ink";
|
||||
import React from "react";
|
||||
import type { TranscriptRecord } from "../../transcript.js";
|
||||
|
||||
export interface RecordViewProps {
|
||||
rec: TranscriptRecord;
|
||||
/**
|
||||
* When rendering side-by-side in diff mode, shorter truncation limits
|
||||
* keep long tool results from dominating the pane. Passes through
|
||||
* untouched when undefined.
|
||||
*/
|
||||
compact?: boolean;
|
||||
}
|
||||
|
||||
export function RecordView({ rec, compact = false }: RecordViewProps) {
|
||||
const toolArgsMax = compact ? 120 : 200;
|
||||
const toolContentMax = compact ? 200 : 400;
|
||||
|
||||
if (rec.role === "user") {
|
||||
return (
|
||||
<Box marginTop={1}>
|
||||
<Text bold color="cyan">
|
||||
you ›{" "}
|
||||
</Text>
|
||||
<Text>{rec.content}</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
if (rec.role === "assistant_final") {
|
||||
return (
|
||||
<Box flexDirection="column" marginTop={1}>
|
||||
<Box>
|
||||
<Text bold color="green">
|
||||
assistant
|
||||
</Text>
|
||||
{rec.cost !== undefined ? (
|
||||
<Text dimColor>
|
||||
{" $"}
|
||||
{rec.cost.toFixed(6)}
|
||||
</Text>
|
||||
) : null}
|
||||
{rec.usage ? <CacheBadge usage={rec.usage} /> : null}
|
||||
</Box>
|
||||
{rec.content ? (
|
||||
<Text>{rec.content}</Text>
|
||||
) : (
|
||||
<Text dimColor italic>
|
||||
(tool-call response only)
|
||||
</Text>
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
if (rec.role === "tool") {
|
||||
return (
|
||||
<Box flexDirection="column" marginTop={1}>
|
||||
<Text color="yellow">
|
||||
{"tool<"}
|
||||
{rec.tool ?? "?"}
|
||||
{">"}
|
||||
</Text>
|
||||
{rec.args ? (
|
||||
<Text dimColor>
|
||||
{" args: "}
|
||||
{truncate(rec.args, toolArgsMax)}
|
||||
</Text>
|
||||
) : null}
|
||||
<Text dimColor>
|
||||
{" → "}
|
||||
{truncate(rec.content, toolContentMax)}
|
||||
</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
if (rec.role === "error") {
|
||||
return (
|
||||
<Box marginTop={1}>
|
||||
<Text color="red" bold>
|
||||
error{" "}
|
||||
</Text>
|
||||
<Text color="red">{rec.error ?? rec.content}</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
if (rec.role === "done" || rec.role === "assistant_delta") {
|
||||
// Noise in replay; skip.
|
||||
return null;
|
||||
}
|
||||
return (
|
||||
<Box>
|
||||
<Text dimColor>
|
||||
[{rec.role}] {rec.content}
|
||||
</Text>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
function CacheBadge({ usage }: { usage: NonNullable<TranscriptRecord["usage"]> }) {
|
||||
const hit = usage.prompt_cache_hit_tokens ?? 0;
|
||||
const miss = usage.prompt_cache_miss_tokens ?? 0;
|
||||
const total = hit + miss;
|
||||
if (total === 0) return null;
|
||||
const pct = (hit / total) * 100;
|
||||
const color = pct >= 70 ? "green" : pct >= 40 ? "yellow" : "red";
|
||||
return (
|
||||
<Text>
|
||||
<Text dimColor>{" · cache "}</Text>
|
||||
<Text color={color}>{pct.toFixed(1)}%</Text>
|
||||
</Text>
|
||||
);
|
||||
}
|
||||
|
||||
function truncate(s: string, max: number): string {
|
||||
return s.length <= max ? s : `${s.slice(0, max)}… (+${s.length - max} chars)`;
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
/**
|
||||
* Ink TUI for `reasonix replay`. Read-only: no input box, no loop.
|
||||
* j/k navigation across turn-pages, cumulative stats sidebar updates
|
||||
* as you move through time.
|
||||
*
|
||||
* The navigation logic (grouping records into pages, computing cumulative
|
||||
* stats) lives in src/replay.ts as pure functions; this file is just
|
||||
* presentation + key bindings.
|
||||
*/
|
||||
|
||||
import { Box, Static, Text, useApp, useInput } from "ink";
|
||||
import React, { useMemo, useState } from "react";
|
||||
import { type TurnPage, computeCumulativeStats } from "../../replay.js";
|
||||
import type { TranscriptMeta } from "../../transcript.js";
|
||||
import { RecordView } from "./RecordView.js";
|
||||
import { StatsPanel } from "./StatsPanel.js";
|
||||
|
||||
export interface ReplayAppProps {
|
||||
meta: TranscriptMeta | null;
|
||||
pages: TurnPage[];
|
||||
}
|
||||
|
||||
export function ReplayApp({ meta, pages }: ReplayAppProps) {
|
||||
const { exit } = useApp();
|
||||
const maxIdx = Math.max(0, pages.length - 1);
|
||||
// Start at the last page — more useful than "start from the beginning"
|
||||
// in practice: users mostly want to see the summary + last turn first.
|
||||
const [idx, setIdx] = useState(maxIdx);
|
||||
|
||||
useInput((input, key) => {
|
||||
if (input === "q" || (key.ctrl && input === "c")) {
|
||||
exit();
|
||||
return;
|
||||
}
|
||||
if (input === "j" || key.downArrow || input === " " || key.return) {
|
||||
setIdx((i) => Math.min(maxIdx, i + 1));
|
||||
} else if (input === "k" || key.upArrow) {
|
||||
setIdx((i) => Math.max(0, i - 1));
|
||||
} else if (input === "g") {
|
||||
setIdx(0);
|
||||
} else if (input === "G") {
|
||||
setIdx(maxIdx);
|
||||
} else if (input === "h" || key.leftArrow) {
|
||||
setIdx(0);
|
||||
} else if (input === "l" || key.rightArrow) {
|
||||
setIdx(maxIdx);
|
||||
}
|
||||
});
|
||||
|
||||
const cumStats = useMemo(() => computeCumulativeStats(pages, idx), [pages, idx]);
|
||||
|
||||
const summary = {
|
||||
turns: cumStats.turns,
|
||||
totalCostUsd: cumStats.totalCostUsd,
|
||||
claudeEquivalentUsd: cumStats.claudeEquivalentUsd,
|
||||
savingsVsClaudePct: cumStats.savingsVsClaudePct,
|
||||
cacheHitRatio: cumStats.cacheHitRatio,
|
||||
};
|
||||
|
||||
const prefixHash =
|
||||
cumStats.prefixHashes.length === 1
|
||||
? cumStats.prefixHashes[0]!.slice(0, 16)
|
||||
: cumStats.prefixHashes.length === 0
|
||||
? "(untracked)"
|
||||
: `(churned ×${cumStats.prefixHashes.length})`;
|
||||
|
||||
const currentPage = pages[idx];
|
||||
const progressLabel =
|
||||
pages.length === 0 ? "empty transcript" : `turn ${idx + 1} / ${pages.length}`;
|
||||
|
||||
return (
|
||||
<Box flexDirection="column">
|
||||
<StatsPanel
|
||||
summary={summary}
|
||||
model={cumStats.models[0] ?? meta?.model ?? "?"}
|
||||
prefixHash={prefixHash}
|
||||
/>
|
||||
|
||||
<Box flexDirection="column" marginTop={1} paddingX={1}>
|
||||
<Box justifyContent="space-between">
|
||||
<Text color="cyan" bold>
|
||||
{progressLabel}
|
||||
</Text>
|
||||
{meta ? (
|
||||
<Text dimColor>
|
||||
{meta.source}
|
||||
{meta.task ? ` · ${meta.task}` : ""}
|
||||
{meta.mode ? ` · ${meta.mode}` : ""}
|
||||
</Text>
|
||||
) : null}
|
||||
</Box>
|
||||
|
||||
{currentPage ? (
|
||||
<Static items={currentPage.records.map((rec, i) => ({ key: `${idx}-${i}`, rec }))}>
|
||||
{({ key, rec }) => <RecordView key={key} rec={rec} />}
|
||||
</Static>
|
||||
) : (
|
||||
<Text dimColor italic>
|
||||
no records
|
||||
</Text>
|
||||
)}
|
||||
</Box>
|
||||
|
||||
<Box marginTop={1} paddingX={1} borderStyle="single" borderColor="gray">
|
||||
<Text dimColor>
|
||||
<Text bold>j</Text>/<Text bold>↓</Text>/<Text bold>space</Text> next · <Text bold>k</Text>
|
||||
/<Text bold>↑</Text> prev · <Text bold>g</Text> first · <Text bold>G</Text> last ·{" "}
|
||||
<Text bold>q</Text> quit
|
||||
</Text>
|
||||
</Box>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import { Box, Text, useApp } from "ink";
|
||||
import TextInput from "ink-text-input";
|
||||
import React, { useState } from "react";
|
||||
import { defaultConfigPath, isPlausibleKey, redactKey, saveApiKey } from "../../config.js";
|
||||
|
||||
export interface SetupProps {
|
||||
onReady: (apiKey: string) => void;
|
||||
}
|
||||
|
||||
export function Setup({ onReady }: SetupProps) {
|
||||
const [value, setValue] = useState("");
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const { exit } = useApp();
|
||||
|
||||
const handleSubmit = (raw: string) => {
|
||||
const trimmed = raw.trim();
|
||||
if (trimmed === "/exit" || trimmed === "/quit") {
|
||||
exit();
|
||||
return;
|
||||
}
|
||||
if (!isPlausibleKey(trimmed)) {
|
||||
setError("Doesn't look like a DeepSeek key. They start with 'sk-' and are 30+ chars.");
|
||||
setValue("");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
saveApiKey(trimmed);
|
||||
} catch (err) {
|
||||
setError(`Could not save key: ${(err as Error).message}`);
|
||||
return;
|
||||
}
|
||||
onReady(trimmed);
|
||||
};
|
||||
|
||||
return (
|
||||
<Box flexDirection="column" borderStyle="round" borderColor="cyan" paddingX={1}>
|
||||
<Text bold color="cyan">
|
||||
Welcome to Reasonix.
|
||||
</Text>
|
||||
<Box marginTop={1}>
|
||||
<Text>Paste your DeepSeek API key to get started.</Text>
|
||||
</Box>
|
||||
<Text dimColor>Get one (free credit on signup): https://platform.deepseek.com/api_keys</Text>
|
||||
<Text dimColor>Saved locally to {defaultConfigPath()}</Text>
|
||||
<Box marginTop={1}>
|
||||
<Text bold color="cyan">
|
||||
{"key › "}
|
||||
</Text>
|
||||
<TextInput
|
||||
value={value}
|
||||
onChange={setValue}
|
||||
onSubmit={handleSubmit}
|
||||
mask="•"
|
||||
placeholder="sk-..."
|
||||
/>
|
||||
</Box>
|
||||
{error ? (
|
||||
<Box marginTop={1}>
|
||||
<Text color="red">{error}</Text>
|
||||
</Box>
|
||||
) : value ? (
|
||||
<Box marginTop={1}>
|
||||
<Text dimColor>preview: {redactKey(value)}</Text>
|
||||
</Box>
|
||||
) : null}
|
||||
<Box marginTop={1}>
|
||||
<Text dimColor>(Type /exit to abort.)</Text>
|
||||
</Box>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import { Box, Text } from "ink";
|
||||
import React from "react";
|
||||
import type { SessionSummary } from "../../telemetry.js";
|
||||
|
||||
export interface StatsPanelProps {
|
||||
summary: SessionSummary;
|
||||
model: string;
|
||||
prefixHash: string;
|
||||
harvestOn?: boolean;
|
||||
branchBudget?: number;
|
||||
}
|
||||
|
||||
export function StatsPanel({
|
||||
summary,
|
||||
model,
|
||||
prefixHash,
|
||||
harvestOn,
|
||||
branchBudget,
|
||||
}: StatsPanelProps) {
|
||||
const hitPct = (summary.cacheHitRatio * 100).toFixed(1);
|
||||
const hitColor =
|
||||
summary.cacheHitRatio >= 0.7 ? "green" : summary.cacheHitRatio >= 0.4 ? "yellow" : "red";
|
||||
const branchOn = (branchBudget ?? 1) > 1;
|
||||
return (
|
||||
<Box borderStyle="round" borderColor="cyan" flexDirection="column" paddingX={1}>
|
||||
<Box justifyContent="space-between">
|
||||
<Text>
|
||||
<Text color="cyan" bold>
|
||||
Reasonix
|
||||
</Text>
|
||||
<Text dimColor> · model </Text>
|
||||
<Text color="yellow">{model}</Text>
|
||||
<Text dimColor> · prefix </Text>
|
||||
<Text dimColor>{prefixHash}</Text>
|
||||
{harvestOn ? <Text color="magenta"> · harvest</Text> : null}
|
||||
{branchOn ? <Text color="blue"> · branch{branchBudget}</Text> : null}
|
||||
</Text>
|
||||
<Text dimColor>turns {summary.turns} · type /help</Text>
|
||||
</Box>
|
||||
<Box marginTop={1} gap={3}>
|
||||
<Text>
|
||||
<Text dimColor>cache hit </Text>
|
||||
<Text color={hitColor} bold>
|
||||
{hitPct}%
|
||||
</Text>
|
||||
</Text>
|
||||
<Text>
|
||||
<Text dimColor>cost </Text>
|
||||
<Text color="green">${summary.totalCostUsd.toFixed(6)}</Text>
|
||||
</Text>
|
||||
<Text>
|
||||
<Text dimColor>vs Claude </Text>
|
||||
<Text>${summary.claudeEquivalentUsd.toFixed(6)}</Text>
|
||||
</Text>
|
||||
<Text>
|
||||
<Text dimColor>saving </Text>
|
||||
<Text color="green" bold>
|
||||
{summary.savingsVsClaudePct.toFixed(1)}%
|
||||
</Text>
|
||||
</Text>
|
||||
</Box>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
/**
|
||||
* Minimal Markdown → Ink renderer for chat output.
|
||||
*
|
||||
* Handles the subset that actually shows up in LLM answers:
|
||||
* - ATX headers (# ##)
|
||||
* - Unordered / ordered lists
|
||||
* - Fenced code blocks (```lang)
|
||||
* - Inline **bold**, *italic*, `code`
|
||||
* - Paragraphs separated by blank lines
|
||||
* - LaTeX delimiters are stripped (\( \), \[ \], \boxed{X})
|
||||
*
|
||||
* The goal is not TeX-perfect math — it's "stop showing raw backslashes to
|
||||
* the user." When the model insists on LaTeX, we strip the scaffolding and
|
||||
* show the expression verbatim; terminals don't do math fonts anyway.
|
||||
*/
|
||||
|
||||
import { Box, Text } from "ink";
|
||||
import React from "react";
|
||||
|
||||
const SUPERSCRIPT: Record<string, string> = {
|
||||
"0": "⁰",
|
||||
"1": "¹",
|
||||
"2": "²",
|
||||
"3": "³",
|
||||
"4": "⁴",
|
||||
"5": "⁵",
|
||||
"6": "⁶",
|
||||
"7": "⁷",
|
||||
"8": "⁸",
|
||||
"9": "⁹",
|
||||
"+": "⁺",
|
||||
"-": "⁻",
|
||||
n: "ⁿ",
|
||||
};
|
||||
const SUBSCRIPT: Record<string, string> = {
|
||||
"0": "₀",
|
||||
"1": "₁",
|
||||
"2": "₂",
|
||||
"3": "₃",
|
||||
"4": "₄",
|
||||
"5": "₅",
|
||||
"6": "₆",
|
||||
"7": "₇",
|
||||
"8": "₈",
|
||||
"9": "₉",
|
||||
"+": "₊",
|
||||
"-": "₋",
|
||||
};
|
||||
|
||||
function toSuperscript(s: string): string {
|
||||
let out = "";
|
||||
for (const c of s) out += SUPERSCRIPT[c] ?? c;
|
||||
return out;
|
||||
}
|
||||
function toSubscript(s: string): string {
|
||||
let out = "";
|
||||
for (const c of s) out += SUBSCRIPT[c] ?? c;
|
||||
return out;
|
||||
}
|
||||
|
||||
export function stripMath(s: string): string {
|
||||
return (
|
||||
s
|
||||
// Delimiters
|
||||
.replace(/\\\(\s*/g, "")
|
||||
.replace(/\s*\\\)/g, "")
|
||||
.replace(/\\\[\s*/g, "\n")
|
||||
.replace(/\s*\\\]/g, "\n")
|
||||
// Fractions — \frac, \dfrac, \tfrac. Allow whitespace and one nesting
|
||||
// level inside braces (e.g. \frac{\sqrt{2}}{3}). Trim captured groups
|
||||
// so '\frac{ a }{ b }' renders as '(a)/(b)'.
|
||||
.replace(
|
||||
/\\[dt]?frac\s*\{((?:[^{}]|\{[^{}]*\})+)\}\s*\{((?:[^{}]|\{[^{}]*\})+)\}/g,
|
||||
(_m, num: string, den: string) => `(${num.trim()})/(${den.trim()})`,
|
||||
)
|
||||
.replace(
|
||||
/\\binom\s*\{([^{}]+)\}\s*\{([^{}]+)\}/g,
|
||||
(_m, n: string, k: string) => `C(${n.trim()},${k.trim()})`,
|
||||
)
|
||||
.replace(/\\sqrt\s*\{([^{}]+)\}/g, (_m, g: string) => `√(${g.trim()})`)
|
||||
.replace(/\\boxed\s*\{([^{}]+)\}/g, (_m, g: string) => `【${g.trim()}】`)
|
||||
.replace(/\\text\s*\{([^{}]+)\}/g, (_m, g: string) => g.trim())
|
||||
.replace(/\\overline\s*\{([^{}]+)\}/g, (_m, g: string) => `${g.trim()}̄`)
|
||||
.replace(/\\hat\s*\{([^{}]+)\}/g, (_m, g: string) => `${g.trim()}̂`)
|
||||
.replace(/\\vec\s*\{([^{}]+)\}/g, (_m, g: string) => `→${g.trim()}`)
|
||||
// Operators & symbols
|
||||
.replace(/\\cdot/g, "·")
|
||||
.replace(/\\times/g, "×")
|
||||
.replace(/\\div/g, "÷")
|
||||
.replace(/\\pm/g, "±")
|
||||
.replace(/\\mp/g, "∓")
|
||||
.replace(/\\leq/g, "≤")
|
||||
.replace(/\\geq/g, "≥")
|
||||
.replace(/\\neq/g, "≠")
|
||||
.replace(/\\approx/g, "≈")
|
||||
.replace(/\\in\b/g, "∈")
|
||||
.replace(/\\notin\b/g, "∉")
|
||||
.replace(/\\infty/g, "∞")
|
||||
.replace(/\\sum\b/g, "Σ")
|
||||
.replace(/\\prod\b/g, "Π")
|
||||
.replace(/\\int\b/g, "∫")
|
||||
// Greek letters
|
||||
.replace(/\\alpha/g, "α")
|
||||
.replace(/\\beta/g, "β")
|
||||
.replace(/\\gamma/g, "γ")
|
||||
.replace(/\\delta/g, "δ")
|
||||
.replace(/\\theta/g, "θ")
|
||||
.replace(/\\lambda/g, "λ")
|
||||
.replace(/\\mu/g, "μ")
|
||||
.replace(/\\pi/g, "π")
|
||||
.replace(/\\sigma/g, "σ")
|
||||
.replace(/\\phi/g, "φ")
|
||||
.replace(/\\omega/g, "ω")
|
||||
// Arrows / logic
|
||||
.replace(/\\implies\b/g, "⇒")
|
||||
.replace(/\\iff\b/g, "⇔")
|
||||
.replace(/\\to\b/g, "→")
|
||||
.replace(/\\rightarrow/g, "→")
|
||||
.replace(/\\Rightarrow/g, "⇒")
|
||||
.replace(/\\leftarrow/g, "←")
|
||||
.replace(/\\Leftarrow/g, "⇐")
|
||||
.replace(/\\ldots/g, "…")
|
||||
.replace(/\\cdots/g, "⋯")
|
||||
// Spacing commands
|
||||
.replace(/\\quad/g, " ")
|
||||
.replace(/\\qquad/g, " ")
|
||||
.replace(/\\,/g, " ")
|
||||
.replace(/\\;/g, " ")
|
||||
.replace(/\\!/g, "")
|
||||
.replace(/\\\\/g, "\n")
|
||||
// Superscripts / subscripts — single token or {braced group of [\w+-]}
|
||||
.replace(/\^\{([\w+-]+)\}/g, (_m, g: string) => toSuperscript(g))
|
||||
.replace(/\^([0-9+\-n])/g, (_m, g: string) => toSuperscript(g))
|
||||
.replace(/_\{([\w+-]+)\}/g, (_m, g: string) => toSubscript(g))
|
||||
.replace(/_([0-9+\-])/g, (_m, g: string) => toSubscript(g))
|
||||
// Catch-all fallbacks for any LaTeX command we didn't explicitly handle.
|
||||
// Belt-and-braces: even if the model invents a new \weirdcommand{x}{y},
|
||||
// we'd rather show '(x)/(y)' or 'x' than a raw backslash.
|
||||
.replace(/\\[a-zA-Z]+\s*\{([^{}]+)\}\s*\{([^{}]+)\}/g, "($1)/($2)")
|
||||
.replace(/\\[a-zA-Z]+\s*\{([^{}]+)\}/g, "$1")
|
||||
.replace(/\\[a-zA-Z]+/g, "")
|
||||
// Collapse multiple whitespace introduced by the stripping above.
|
||||
.replace(/[ \t]{2,}/g, " ")
|
||||
);
|
||||
}
|
||||
|
||||
/** Split a single line into styled segments for bold / italic / inline code. */
|
||||
const INLINE_RE = /(\*\*([^*\n]+?)\*\*|`([^`\n]+?)`|(?<![*\w])\*([^*\n]+?)\*(?!\w))/g;
|
||||
|
||||
function InlineMd({ text }: { text: string }) {
|
||||
const parts: React.ReactNode[] = [];
|
||||
let last = 0;
|
||||
let idx = 0;
|
||||
for (const m of text.matchAll(INLINE_RE)) {
|
||||
const start = m.index ?? 0;
|
||||
if (start > last) {
|
||||
parts.push(<Text key={`t${idx++}`}>{text.slice(last, start)}</Text>);
|
||||
}
|
||||
if (m[2] !== undefined) {
|
||||
parts.push(
|
||||
<Text key={`b${idx++}`} bold>
|
||||
{m[2]}
|
||||
</Text>,
|
||||
);
|
||||
} else if (m[3] !== undefined) {
|
||||
parts.push(
|
||||
<Text key={`c${idx++}`} color="yellow">
|
||||
{m[3]}
|
||||
</Text>,
|
||||
);
|
||||
} else if (m[4] !== undefined) {
|
||||
parts.push(
|
||||
<Text key={`i${idx++}`} italic>
|
||||
{m[4]}
|
||||
</Text>,
|
||||
);
|
||||
}
|
||||
last = start + m[0].length;
|
||||
}
|
||||
if (last < text.length) {
|
||||
parts.push(<Text key={`t${idx++}`}>{text.slice(last)}</Text>);
|
||||
}
|
||||
return <Text>{parts}</Text>;
|
||||
}
|
||||
|
||||
interface ParagraphBlock {
|
||||
kind: "paragraph";
|
||||
text: string;
|
||||
}
|
||||
interface HeadingBlock {
|
||||
kind: "heading";
|
||||
level: number;
|
||||
text: string;
|
||||
}
|
||||
interface BulletBlock {
|
||||
kind: "bullet";
|
||||
items: string[];
|
||||
ordered: boolean;
|
||||
start: number;
|
||||
}
|
||||
interface CodeBlock {
|
||||
kind: "code";
|
||||
lang: string;
|
||||
text: string;
|
||||
}
|
||||
interface HrBlock {
|
||||
kind: "hr";
|
||||
}
|
||||
|
||||
type Block = ParagraphBlock | HeadingBlock | BulletBlock | CodeBlock | HrBlock;
|
||||
|
||||
function parseBlocks(raw: string): Block[] {
|
||||
const lines = raw.split(/\r?\n/);
|
||||
const out: Block[] = [];
|
||||
let para: string[] = [];
|
||||
let inCode = false;
|
||||
let codeLang = "";
|
||||
let codeBuf: string[] = [];
|
||||
let listBuf: BulletBlock | null = null;
|
||||
|
||||
const flushPara = () => {
|
||||
if (para.length) {
|
||||
out.push({ kind: "paragraph", text: para.join(" ") });
|
||||
para = [];
|
||||
}
|
||||
};
|
||||
const flushList = () => {
|
||||
if (listBuf) {
|
||||
out.push(listBuf);
|
||||
listBuf = null;
|
||||
}
|
||||
};
|
||||
|
||||
for (const rawLine of lines) {
|
||||
const line = rawLine.replace(/\s+$/g, "");
|
||||
|
||||
const fence = line.match(/^```(\w*)/);
|
||||
if (fence) {
|
||||
if (inCode) {
|
||||
out.push({ kind: "code", lang: codeLang, text: codeBuf.join("\n") });
|
||||
codeBuf = [];
|
||||
codeLang = "";
|
||||
inCode = false;
|
||||
} else {
|
||||
flushPara();
|
||||
flushList();
|
||||
inCode = true;
|
||||
codeLang = fence[1] ?? "";
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (inCode) {
|
||||
codeBuf.push(rawLine);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line.trim() === "") {
|
||||
flushPara();
|
||||
flushList();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (/^[-*_]{3,}\s*$/.test(line)) {
|
||||
flushPara();
|
||||
flushList();
|
||||
out.push({ kind: "hr" });
|
||||
continue;
|
||||
}
|
||||
|
||||
const hm = line.match(/^(#{1,6})\s+(.+)$/);
|
||||
if (hm) {
|
||||
flushPara();
|
||||
flushList();
|
||||
out.push({ kind: "heading", level: hm[1]!.length, text: hm[2]!.trim() });
|
||||
continue;
|
||||
}
|
||||
|
||||
const bm = line.match(/^\s*[-*+]\s+(.+)$/);
|
||||
if (bm) {
|
||||
flushPara();
|
||||
if (!listBuf || listBuf.ordered) {
|
||||
flushList();
|
||||
listBuf = { kind: "bullet", items: [], ordered: false, start: 1 };
|
||||
}
|
||||
listBuf.items.push(bm[1]!);
|
||||
continue;
|
||||
}
|
||||
|
||||
const om = line.match(/^\s*(\d+)\.\s+(.+)$/);
|
||||
if (om) {
|
||||
flushPara();
|
||||
if (!listBuf || !listBuf.ordered) {
|
||||
flushList();
|
||||
listBuf = { kind: "bullet", items: [], ordered: true, start: Number(om[1]) };
|
||||
}
|
||||
listBuf.items.push(om[2]!);
|
||||
continue;
|
||||
}
|
||||
|
||||
flushList();
|
||||
para.push(line);
|
||||
}
|
||||
|
||||
if (inCode && codeBuf.length) {
|
||||
out.push({ kind: "code", lang: codeLang, text: codeBuf.join("\n") });
|
||||
}
|
||||
flushPara();
|
||||
flushList();
|
||||
return out;
|
||||
}
|
||||
|
||||
function BlockView({ block }: { block: Block }) {
|
||||
switch (block.kind) {
|
||||
case "heading":
|
||||
return (
|
||||
<Text bold color="cyan">
|
||||
<InlineMd text={block.text} />
|
||||
</Text>
|
||||
);
|
||||
case "paragraph":
|
||||
return <InlineMd text={block.text} />;
|
||||
case "bullet":
|
||||
return (
|
||||
<Box flexDirection="column">
|
||||
{block.items.map((item, i) => (
|
||||
<Box key={`${i}-${item.slice(0, 24)}`}>
|
||||
<Text color="cyan">{block.ordered ? ` ${block.start + i}. ` : " • "}</Text>
|
||||
<InlineMd text={item} />
|
||||
</Box>
|
||||
))}
|
||||
</Box>
|
||||
);
|
||||
case "code":
|
||||
return (
|
||||
<Box borderStyle="single" borderColor="gray" paddingX={1}>
|
||||
<Text color="yellow">{block.text}</Text>
|
||||
</Box>
|
||||
);
|
||||
case "hr":
|
||||
return <Text dimColor>{"────────────────────────"}</Text>;
|
||||
}
|
||||
}
|
||||
|
||||
export function Markdown({ text }: { text: string }) {
|
||||
const cleaned = stripMath(text);
|
||||
const blocks = React.useMemo(() => parseBlocks(cleaned), [cleaned]);
|
||||
return (
|
||||
<Box flexDirection="column" gap={1}>
|
||||
{blocks.map((b, i) => (
|
||||
<BlockView key={`${i}-${b.kind}`} block={b} />
|
||||
))}
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
import type { CacheFirstLoop } from "../../loop.js";
|
||||
import { deleteSession, listSessions } from "../../session.js";
|
||||
|
||||
export interface SlashResult {
|
||||
/** Text to display back to the user as a system/info line. */
|
||||
info?: string;
|
||||
/** Exit the app. */
|
||||
exit?: boolean;
|
||||
/** Clear the visible history. */
|
||||
clear?: boolean;
|
||||
/** Unknown command — display usage hint. */
|
||||
unknown?: boolean;
|
||||
}
|
||||
|
||||
export function parseSlash(text: string): { cmd: string; args: string[] } | null {
|
||||
if (!text.startsWith("/")) return null;
|
||||
const parts = text.slice(1).trim().split(/\s+/);
|
||||
const cmd = parts[0]?.toLowerCase() ?? "";
|
||||
if (!cmd) return null;
|
||||
return { cmd, args: parts.slice(1) };
|
||||
}
|
||||
|
||||
export function handleSlash(cmd: string, args: string[], loop: CacheFirstLoop): SlashResult {
|
||||
switch (cmd) {
|
||||
case "exit":
|
||||
case "quit":
|
||||
return { exit: true };
|
||||
|
||||
case "clear":
|
||||
return { clear: true };
|
||||
|
||||
case "help":
|
||||
case "?":
|
||||
return {
|
||||
info: [
|
||||
"Commands:",
|
||||
" /help this message",
|
||||
" /status show current settings",
|
||||
" /preset <fast|smart|max> one-tap presets — see below",
|
||||
" /model <id> deepseek-chat or deepseek-reasoner",
|
||||
" /harvest [on|off] Pillar 2: structured plan-state extraction",
|
||||
" /branch <N|off> run N parallel samples (N>=2), pick most confident",
|
||||
" /sessions list saved sessions (current is marked with ▸)",
|
||||
" /forget delete the current session from disk",
|
||||
" /clear clear displayed history (log + session kept)",
|
||||
" /exit quit",
|
||||
"",
|
||||
"Presets:",
|
||||
" fast deepseek-chat no harvest no branch ~1¢/100turns ← default",
|
||||
" smart reasoner harvest ~10x cost, slower",
|
||||
" max reasoner harvest branch 3 ~30x cost, slowest",
|
||||
"",
|
||||
"Sessions (auto-enabled by default, named 'default'):",
|
||||
" reasonix chat --session <name> use a different named session",
|
||||
" reasonix chat --no-session disable persistence for this run",
|
||||
].join("\n"),
|
||||
};
|
||||
|
||||
case "sessions": {
|
||||
const items = listSessions();
|
||||
if (items.length === 0) {
|
||||
return {
|
||||
info: "no saved sessions yet — chat normally and your messages will be saved automatically",
|
||||
};
|
||||
}
|
||||
const lines = ["Saved sessions:"];
|
||||
for (const s of items) {
|
||||
const sizeKb = (s.size / 1024).toFixed(1);
|
||||
const when = s.mtime.toISOString().replace("T", " ").slice(0, 16);
|
||||
const marker = s.name === loop.sessionName ? "▸" : " ";
|
||||
lines.push(
|
||||
` ${marker} ${s.name.padEnd(22)} ${String(s.messageCount).padStart(5)} msgs ${sizeKb.padStart(7)} KB ${when}`,
|
||||
);
|
||||
}
|
||||
lines.push("");
|
||||
lines.push("Resume with: reasonix chat --session <name>");
|
||||
return { info: lines.join("\n") };
|
||||
}
|
||||
|
||||
case "forget": {
|
||||
if (!loop.sessionName) {
|
||||
return { info: "not in a session — nothing to forget" };
|
||||
}
|
||||
const name = loop.sessionName;
|
||||
const ok = deleteSession(name);
|
||||
return {
|
||||
info: ok
|
||||
? `▸ deleted session "${name}" — current screen still shows the conversation, but next launch starts fresh`
|
||||
: `could not delete session "${name}" (already gone?)`,
|
||||
};
|
||||
}
|
||||
|
||||
case "status": {
|
||||
const branchBudget = loop.branchOptions.budget ?? 1;
|
||||
return {
|
||||
info:
|
||||
`model=${loop.model} ` +
|
||||
`harvest=${loop.harvestEnabled ? "on" : "off"} ` +
|
||||
`branch=${branchBudget > 1 ? branchBudget : "off"} ` +
|
||||
`stream=${loop.stream ? "on" : "off"}`,
|
||||
};
|
||||
}
|
||||
|
||||
case "model": {
|
||||
const id = args[0];
|
||||
if (!id) return { info: "usage: /model <id> (try deepseek-chat or deepseek-reasoner)" };
|
||||
loop.configure({ model: id });
|
||||
return { info: `model → ${id}` };
|
||||
}
|
||||
|
||||
case "harvest": {
|
||||
const arg = (args[0] ?? "").toLowerCase();
|
||||
const on = arg === "" ? !loop.harvestEnabled : arg === "on" || arg === "true" || arg === "1";
|
||||
loop.configure({ harvest: on });
|
||||
return { info: `harvest → ${loop.harvestEnabled ? "on" : "off"}` };
|
||||
}
|
||||
|
||||
case "preset": {
|
||||
const name = (args[0] ?? "").toLowerCase();
|
||||
if (name === "fast" || name === "default") {
|
||||
loop.configure({ model: "deepseek-chat", harvest: false, branch: 1 });
|
||||
return { info: "preset → fast (deepseek-chat, no harvest, no branch)" };
|
||||
}
|
||||
if (name === "smart") {
|
||||
loop.configure({ model: "deepseek-reasoner", harvest: true, branch: 1 });
|
||||
return { info: "preset → smart (reasoner + harvest, ~10x cost vs fast)" };
|
||||
}
|
||||
if (name === "max" || name === "best") {
|
||||
loop.configure({ model: "deepseek-reasoner", harvest: true, branch: 3 });
|
||||
return {
|
||||
info: "preset → max (reasoner + harvest + branch3, ~30x cost vs fast, slowest)",
|
||||
};
|
||||
}
|
||||
return { info: "usage: /preset <fast|smart|max>" };
|
||||
}
|
||||
|
||||
case "branch": {
|
||||
const raw = (args[0] ?? "").toLowerCase();
|
||||
if (raw === "" || raw === "off" || raw === "0" || raw === "1") {
|
||||
loop.configure({ branch: 1 });
|
||||
return { info: "branch → off" };
|
||||
}
|
||||
const n = Number.parseInt(raw, 10);
|
||||
if (!Number.isFinite(n) || n < 2) {
|
||||
return { info: "usage: /branch <N> (N>=2, or 'off')" };
|
||||
}
|
||||
if (n > 8) {
|
||||
return { info: "branch budget capped at 8 to prevent runaway cost" };
|
||||
}
|
||||
loop.configure({ branch: n });
|
||||
return { info: `branch → ${n} (harvest auto-enabled; streaming disabled)` };
|
||||
}
|
||||
|
||||
default:
|
||||
return { unknown: true, info: `unknown command: /${cmd} (try /help)` };
|
||||
}
|
||||
}
|
||||
+225
@@ -0,0 +1,225 @@
|
||||
import { type EventSourceMessage, createParser } from "eventsource-parser";
|
||||
import { type RetryOptions, fetchWithRetry } from "./retry.js";
|
||||
import type { ChatMessage, ChatRequestOptions, RawUsage, ToolCall, ToolSpec } from "./types.js";
|
||||
|
||||
export class Usage {
|
||||
constructor(
|
||||
public promptTokens = 0,
|
||||
public completionTokens = 0,
|
||||
public totalTokens = 0,
|
||||
public promptCacheHitTokens = 0,
|
||||
public promptCacheMissTokens = 0,
|
||||
) {}
|
||||
|
||||
get cacheHitRatio(): number {
|
||||
const denom = this.promptCacheHitTokens + this.promptCacheMissTokens;
|
||||
return denom > 0 ? this.promptCacheHitTokens / denom : 0;
|
||||
}
|
||||
|
||||
static fromApi(raw: RawUsage | undefined | null): Usage {
|
||||
const u = raw ?? {};
|
||||
return new Usage(
|
||||
u.prompt_tokens ?? 0,
|
||||
u.completion_tokens ?? 0,
|
||||
u.total_tokens ?? 0,
|
||||
u.prompt_cache_hit_tokens ?? 0,
|
||||
u.prompt_cache_miss_tokens ?? 0,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export interface ChatResponse {
|
||||
content: string;
|
||||
reasoningContent: string | null;
|
||||
toolCalls: ToolCall[];
|
||||
usage: Usage;
|
||||
raw: unknown;
|
||||
}
|
||||
|
||||
export interface StreamChunk {
|
||||
contentDelta?: string;
|
||||
reasoningDelta?: string;
|
||||
toolCallDelta?: { index: number; id?: string; name?: string; argumentsDelta?: string };
|
||||
usage?: Usage;
|
||||
finishReason?: string;
|
||||
raw: any;
|
||||
}
|
||||
|
||||
export interface DeepSeekClientOptions {
|
||||
apiKey?: string;
|
||||
baseUrl?: string;
|
||||
timeoutMs?: number;
|
||||
fetch?: typeof fetch;
|
||||
/** Retry configuration. Pass `{ maxAttempts: 1 }` to disable retries. */
|
||||
retry?: RetryOptions;
|
||||
}
|
||||
|
||||
export class DeepSeekClient {
|
||||
readonly apiKey: string;
|
||||
readonly baseUrl: string;
|
||||
readonly timeoutMs: number;
|
||||
readonly retry: RetryOptions;
|
||||
private readonly _fetch: typeof fetch;
|
||||
|
||||
constructor(opts: DeepSeekClientOptions = {}) {
|
||||
const apiKey = opts.apiKey ?? process.env.DEEPSEEK_API_KEY;
|
||||
if (!apiKey) {
|
||||
throw new Error(
|
||||
"DEEPSEEK_API_KEY is not set. Put it in .env or pass apiKey to DeepSeekClient.",
|
||||
);
|
||||
}
|
||||
this.apiKey = apiKey;
|
||||
this.baseUrl = (
|
||||
opts.baseUrl ??
|
||||
process.env.DEEPSEEK_BASE_URL ??
|
||||
"https://api.deepseek.com"
|
||||
).replace(/\/+$/, "");
|
||||
this.timeoutMs = opts.timeoutMs ?? 120_000;
|
||||
this._fetch = opts.fetch ?? globalThis.fetch.bind(globalThis);
|
||||
this.retry = opts.retry ?? {};
|
||||
}
|
||||
|
||||
private buildPayload(opts: ChatRequestOptions, stream: boolean) {
|
||||
const payload: Record<string, unknown> = {
|
||||
model: opts.model,
|
||||
messages: opts.messages,
|
||||
stream,
|
||||
};
|
||||
if (opts.tools?.length) payload.tools = opts.tools;
|
||||
if (opts.temperature !== undefined) payload.temperature = opts.temperature;
|
||||
if (opts.maxTokens !== undefined) payload.max_tokens = opts.maxTokens;
|
||||
if (opts.responseFormat) payload.response_format = opts.responseFormat;
|
||||
return payload;
|
||||
}
|
||||
|
||||
async chat(opts: ChatRequestOptions): Promise<ChatResponse> {
|
||||
const ctrl = new AbortController();
|
||||
const timer = setTimeout(() => ctrl.abort(), this.timeoutMs);
|
||||
const signal = opts.signal ?? ctrl.signal;
|
||||
|
||||
try {
|
||||
const resp = await fetchWithRetry(
|
||||
this._fetch,
|
||||
`${this.baseUrl}/chat/completions`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(this.buildPayload(opts, false)),
|
||||
signal,
|
||||
},
|
||||
{ ...this.retry, signal },
|
||||
);
|
||||
if (!resp.ok) {
|
||||
throw new Error(`DeepSeek ${resp.status}: ${await resp.text()}`);
|
||||
}
|
||||
const data: any = await resp.json();
|
||||
const choice = data.choices?.[0]?.message ?? {};
|
||||
return {
|
||||
content: choice.content ?? "",
|
||||
reasoningContent: choice.reasoning_content ?? null,
|
||||
toolCalls: choice.tool_calls ?? [],
|
||||
usage: Usage.fromApi(data.usage),
|
||||
raw: data,
|
||||
};
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
async *stream(opts: ChatRequestOptions): AsyncGenerator<StreamChunk> {
|
||||
const ctrl = new AbortController();
|
||||
const timer = setTimeout(() => ctrl.abort(), this.timeoutMs);
|
||||
const signal = opts.signal ?? ctrl.signal;
|
||||
|
||||
let resp: Response;
|
||||
try {
|
||||
// Only the initial fetch is retried. Once the server has started sending
|
||||
// the stream body we do NOT retry — a mid-stream retry would re-bill and
|
||||
// desync the session context.
|
||||
resp = await fetchWithRetry(
|
||||
this._fetch,
|
||||
`${this.baseUrl}/chat/completions`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
Accept: "text/event-stream",
|
||||
},
|
||||
body: JSON.stringify(this.buildPayload(opts, true)),
|
||||
signal,
|
||||
},
|
||||
{ ...this.retry, signal },
|
||||
);
|
||||
} catch (err) {
|
||||
clearTimeout(timer);
|
||||
throw err;
|
||||
}
|
||||
if (!resp.ok || !resp.body) {
|
||||
clearTimeout(timer);
|
||||
throw new Error(`DeepSeek ${resp.status}: ${await resp.text().catch(() => "")}`);
|
||||
}
|
||||
|
||||
const queue: StreamChunk[] = [];
|
||||
let done = false;
|
||||
const parser = createParser({
|
||||
onEvent: (ev: EventSourceMessage) => {
|
||||
if (!ev.data || ev.data === "[DONE]") {
|
||||
done = true;
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const json = JSON.parse(ev.data);
|
||||
const delta = json.choices?.[0]?.delta ?? {};
|
||||
const finishReason = json.choices?.[0]?.finish_reason ?? undefined;
|
||||
const chunk: StreamChunk = { raw: json, finishReason };
|
||||
if (typeof delta.content === "string" && delta.content.length > 0) {
|
||||
chunk.contentDelta = delta.content;
|
||||
}
|
||||
if (typeof delta.reasoning_content === "string" && delta.reasoning_content.length > 0) {
|
||||
chunk.reasoningDelta = delta.reasoning_content;
|
||||
}
|
||||
if (Array.isArray(delta.tool_calls) && delta.tool_calls.length > 0) {
|
||||
const tc = delta.tool_calls[0];
|
||||
chunk.toolCallDelta = {
|
||||
index: tc.index ?? 0,
|
||||
id: tc.id,
|
||||
name: tc.function?.name,
|
||||
argumentsDelta: tc.function?.arguments,
|
||||
};
|
||||
}
|
||||
if (json.usage) {
|
||||
chunk.usage = Usage.fromApi(json.usage);
|
||||
}
|
||||
queue.push(chunk);
|
||||
} catch {
|
||||
/* skip malformed sse frame */
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const reader = resp.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
try {
|
||||
while (true) {
|
||||
if (queue.length > 0) {
|
||||
yield queue.shift()!;
|
||||
continue;
|
||||
}
|
||||
if (done) break;
|
||||
const { value, done: streamDone } = await reader.read();
|
||||
if (streamDone) break;
|
||||
parser.feed(decoder.decode(value, { stream: true }));
|
||||
}
|
||||
while (queue.length > 0) yield queue.shift()!;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export type { ChatMessage, ToolCall, ToolSpec };
|
||||
@@ -0,0 +1,70 @@
|
||||
/**
|
||||
* User-level config storage for the Reasonix CLI.
|
||||
*
|
||||
* Lookup order for the API key:
|
||||
* 1. `DEEPSEEK_API_KEY` env var (highest priority — for CI / power users)
|
||||
* 2. `~/.reasonix/config.json` (set by the first-run setup flow)
|
||||
*
|
||||
* The library itself never touches the config file — it only reads
|
||||
* `DEEPSEEK_API_KEY` from the environment. The CLI is responsible for
|
||||
* pulling from the config file and exposing it via env var to the loop.
|
||||
*/
|
||||
|
||||
import { chmodSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
|
||||
export interface ReasonixConfig {
|
||||
apiKey?: string;
|
||||
baseUrl?: string;
|
||||
}
|
||||
|
||||
export function defaultConfigPath(): string {
|
||||
return join(homedir(), ".reasonix", "config.json");
|
||||
}
|
||||
|
||||
export function readConfig(path: string = defaultConfigPath()): ReasonixConfig {
|
||||
try {
|
||||
const raw = readFileSync(path, "utf8");
|
||||
const parsed = JSON.parse(raw);
|
||||
if (parsed && typeof parsed === "object") return parsed as ReasonixConfig;
|
||||
} catch {
|
||||
/* missing or malformed → empty config */
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
export function writeConfig(cfg: ReasonixConfig, path: string = defaultConfigPath()): void {
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
writeFileSync(path, JSON.stringify(cfg, null, 2), "utf8");
|
||||
// Restrict permissions on Unix; chmod is a no-op on Windows but won't throw.
|
||||
try {
|
||||
chmodSync(path, 0o600);
|
||||
} catch {
|
||||
/* ignore on platforms without chmod */
|
||||
}
|
||||
}
|
||||
|
||||
/** Resolve the API key from env var first, then the config file. */
|
||||
export function loadApiKey(path: string = defaultConfigPath()): string | undefined {
|
||||
if (process.env.DEEPSEEK_API_KEY) return process.env.DEEPSEEK_API_KEY;
|
||||
return readConfig(path).apiKey;
|
||||
}
|
||||
|
||||
export function saveApiKey(key: string, path: string = defaultConfigPath()): void {
|
||||
const cfg = readConfig(path);
|
||||
cfg.apiKey = key.trim();
|
||||
writeConfig(cfg, path);
|
||||
}
|
||||
|
||||
export function isPlausibleKey(key: string): boolean {
|
||||
const trimmed = key.trim();
|
||||
return /^sk-[A-Za-z0-9_-]{16,}$/.test(trimmed);
|
||||
}
|
||||
|
||||
/** Mask a key for display: `sk-abcd...wxyz`. */
|
||||
export function redactKey(key: string): string {
|
||||
if (!key) return "";
|
||||
if (key.length <= 12) return "****";
|
||||
return `${key.slice(0, 6)}…${key.slice(-4)}`;
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
/**
|
||||
* Self-consistency branching.
|
||||
*
|
||||
* When enabled, the loop fans out into N parallel samples per turn (varied
|
||||
* temperatures), runs Pillar 2 harvest on each, and selects the sample with
|
||||
* the fewest flagged uncertainties (ties broken by answer length — a crude
|
||||
* Occam prior).
|
||||
*
|
||||
* The unique opportunity here: because DeepSeek is ~20× cheaper than Claude,
|
||||
* running N=3–5 samples per turn is still cheaper than a single Claude call,
|
||||
* while the majority-confidence selection tends to dominate single-sample
|
||||
* answers on fuzzy multi-step reasoning tasks.
|
||||
*/
|
||||
|
||||
import type { ChatResponse, DeepSeekClient } from "./client.js";
|
||||
import { type HarvestOptions, type TypedPlanState, harvest } from "./harvest.js";
|
||||
import type { ChatRequestOptions } from "./types.js";
|
||||
|
||||
export interface BranchSample {
|
||||
index: number;
|
||||
temperature: number;
|
||||
response: ChatResponse;
|
||||
planState: TypedPlanState;
|
||||
}
|
||||
|
||||
export type BranchSelector = (samples: BranchSample[]) => BranchSample;
|
||||
|
||||
export interface BranchOptions {
|
||||
/** Number of parallel samples. 1 disables branching. Default 1. */
|
||||
budget?: number;
|
||||
/** Temperatures for each branch. Default spreads across [0, 1]. */
|
||||
temperatures?: readonly number[];
|
||||
/** Harvest options; the selector needs harvest to score samples. */
|
||||
harvestOptions?: HarvestOptions;
|
||||
/** Custom selector. Default: min uncertainties, tie-break shortest answer. */
|
||||
selector?: BranchSelector;
|
||||
/**
|
||||
* Fires as each sample finishes (main call + harvest both complete).
|
||||
* Useful for progress UI. Not awaited; exceptions are swallowed.
|
||||
*/
|
||||
onSampleDone?: (sample: BranchSample) => void;
|
||||
}
|
||||
|
||||
export interface BranchResult {
|
||||
chosen: BranchSample;
|
||||
samples: BranchSample[];
|
||||
}
|
||||
|
||||
/** Default: fewest uncertainties wins, ties broken by shorter answer content. */
|
||||
export const defaultSelector: BranchSelector = (samples) => {
|
||||
if (samples.length === 0) throw new Error("defaultSelector: samples is empty");
|
||||
return samples.slice().sort((a, b) => {
|
||||
const uDiff = a.planState.uncertainties.length - b.planState.uncertainties.length;
|
||||
if (uDiff !== 0) return uDiff;
|
||||
const aLen = a.response.content?.length ?? 0;
|
||||
const bLen = b.response.content?.length ?? 0;
|
||||
return aLen - bLen;
|
||||
})[0]!;
|
||||
};
|
||||
|
||||
export async function runBranches(
|
||||
client: DeepSeekClient,
|
||||
request: ChatRequestOptions,
|
||||
opts: BranchOptions = {},
|
||||
): Promise<BranchResult> {
|
||||
const budget = Math.max(1, opts.budget ?? 1);
|
||||
const temperatures = resolveTemperatures(budget, opts.temperatures);
|
||||
const selector = opts.selector ?? defaultSelector;
|
||||
|
||||
const samples = await Promise.all(
|
||||
temperatures.map(async (temperature, index): Promise<BranchSample> => {
|
||||
const response = await client.chat({ ...request, temperature });
|
||||
const planState = await harvest(response.reasoningContent, client, opts.harvestOptions);
|
||||
const sample: BranchSample = { index, temperature, response, planState };
|
||||
try {
|
||||
opts.onSampleDone?.(sample);
|
||||
} catch {
|
||||
/* callback errors must not poison the await */
|
||||
}
|
||||
return sample;
|
||||
}),
|
||||
);
|
||||
|
||||
return { chosen: selector(samples), samples };
|
||||
}
|
||||
|
||||
/** Sum usage across branch samples for telemetry purposes. */
|
||||
export function aggregateBranchUsage(samples: readonly BranchSample[]) {
|
||||
let promptTokens = 0;
|
||||
let completionTokens = 0;
|
||||
let totalTokens = 0;
|
||||
let promptCacheHitTokens = 0;
|
||||
let promptCacheMissTokens = 0;
|
||||
for (const s of samples) {
|
||||
promptTokens += s.response.usage.promptTokens;
|
||||
completionTokens += s.response.usage.completionTokens;
|
||||
totalTokens += s.response.usage.totalTokens;
|
||||
promptCacheHitTokens += s.response.usage.promptCacheHitTokens;
|
||||
promptCacheMissTokens += s.response.usage.promptCacheMissTokens;
|
||||
}
|
||||
return {
|
||||
promptTokens,
|
||||
completionTokens,
|
||||
totalTokens,
|
||||
promptCacheHitTokens,
|
||||
promptCacheMissTokens,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveTemperatures(budget: number, custom?: readonly number[]): number[] {
|
||||
if (custom && custom.length >= budget) return [...custom.slice(0, budget)];
|
||||
// Spread evenly across [0, 1] to encourage reasoning-path diversity.
|
||||
if (budget === 1) return [0];
|
||||
const out: number[] = [];
|
||||
for (let i = 0; i < budget; i++) {
|
||||
out.push(Number((i / (budget - 1)).toFixed(2)));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
+436
@@ -0,0 +1,436 @@
|
||||
/**
|
||||
* Diff — compare two transcripts and produce a summary + divergence report.
|
||||
*
|
||||
* Two transcripts are "comparable" when they stem from the same task (or
|
||||
* the same user prompt). Alignment is by turn number: assistant_final #N
|
||||
* in A pairs with assistant_final #N in B. If one side ran more turns, the
|
||||
* extras are labeled "only in A" / "only in B".
|
||||
*
|
||||
* What we compute:
|
||||
* - Aggregate deltas: turns, tool calls, cache hit, cost, token counts
|
||||
* - First divergence: the lowest turn where A and B's tool calls or
|
||||
* assistant text differ meaningfully
|
||||
* - Prefix-stability story: how many unique prefix hashes each side used
|
||||
*
|
||||
* Non-goals (deliberately):
|
||||
* - LLM-judge quality comparison
|
||||
* - Per-token delta rendering — not useful at the fidelity we're at
|
||||
* - Embedding similarity — Levenshtein ratio is cheap and good enough
|
||||
*/
|
||||
|
||||
import { type ReplayStats, computeReplayStats } from "./replay.js";
|
||||
import type { ReadTranscriptResult, TranscriptRecord } from "./transcript.js";
|
||||
|
||||
export interface DiffSide {
|
||||
label: string;
|
||||
meta: ReadTranscriptResult["meta"];
|
||||
records: TranscriptRecord[];
|
||||
stats: ReplayStats;
|
||||
}
|
||||
|
||||
export interface TurnPair {
|
||||
turn: number;
|
||||
aAssistant?: TranscriptRecord;
|
||||
bAssistant?: TranscriptRecord;
|
||||
aTools: TranscriptRecord[];
|
||||
bTools: TranscriptRecord[];
|
||||
/**
|
||||
* Classification of the pair:
|
||||
* "match" — both sides present, text & tool calls within threshold
|
||||
* "diverge" — both sides present, but text or tool calls differ
|
||||
* "only_in_a" — assistant_final in A but not B
|
||||
* "only_in_b" — assistant_final in B but not A
|
||||
*/
|
||||
kind: "match" | "diverge" | "only_in_a" | "only_in_b";
|
||||
/** When kind === "diverge", a short one-liner pointing at what differs. */
|
||||
divergenceNote?: string;
|
||||
}
|
||||
|
||||
export interface DiffReport {
|
||||
a: DiffSide;
|
||||
b: DiffSide;
|
||||
pairs: TurnPair[];
|
||||
firstDivergenceTurn: number | null;
|
||||
}
|
||||
|
||||
// ---------- navigation helpers (used by the Ink diff TUI) ----------
|
||||
|
||||
/**
|
||||
* Find the next pair (strictly after `fromIdx`) whose kind is not "match".
|
||||
* Returns -1 when no later divergence exists. Used by DiffApp's `n` key.
|
||||
*/
|
||||
export function findNextDivergence(pairs: TurnPair[], fromIdx: number): number {
|
||||
for (let i = fromIdx + 1; i < pairs.length; i++) {
|
||||
if (pairs[i]!.kind !== "match") return i;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the previous pair (strictly before `fromIdx`) whose kind is not
|
||||
* "match". Returns -1 when no earlier divergence exists. Used by
|
||||
* DiffApp's `N` / `p` key.
|
||||
*/
|
||||
export function findPrevDivergence(pairs: TurnPair[], fromIdx: number): number {
|
||||
const start = Math.min(fromIdx - 1, pairs.length - 1);
|
||||
for (let i = start; i >= 0; i--) {
|
||||
if (pairs[i]!.kind !== "match") return i;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
export function diffTranscripts(
|
||||
a: { label: string; parsed: ReadTranscriptResult },
|
||||
b: { label: string; parsed: ReadTranscriptResult },
|
||||
): DiffReport {
|
||||
const aSide: DiffSide = {
|
||||
label: a.label,
|
||||
meta: a.parsed.meta,
|
||||
records: a.parsed.records,
|
||||
stats: computeReplayStats(a.parsed.records),
|
||||
};
|
||||
const bSide: DiffSide = {
|
||||
label: b.label,
|
||||
meta: b.parsed.meta,
|
||||
records: b.parsed.records,
|
||||
stats: computeReplayStats(b.parsed.records),
|
||||
};
|
||||
|
||||
const aByTurn = groupByTurn(a.parsed.records);
|
||||
const bByTurn = groupByTurn(b.parsed.records);
|
||||
const turns = [...new Set([...aByTurn.keys(), ...bByTurn.keys()])].sort((x, y) => x - y);
|
||||
|
||||
const pairs: TurnPair[] = [];
|
||||
let firstDivergenceTurn: number | null = null;
|
||||
for (const turn of turns) {
|
||||
const aGroup = aByTurn.get(turn) ?? { assistant: undefined, tools: [] };
|
||||
const bGroup = bByTurn.get(turn) ?? { assistant: undefined, tools: [] };
|
||||
const aAssistant = aGroup.assistant;
|
||||
const bAssistant = bGroup.assistant;
|
||||
const aTools = aGroup.tools;
|
||||
const bTools = bGroup.tools;
|
||||
|
||||
let kind: TurnPair["kind"];
|
||||
let divergenceNote: string | undefined;
|
||||
if (!aAssistant && bAssistant) kind = "only_in_b";
|
||||
else if (aAssistant && !bAssistant) kind = "only_in_a";
|
||||
else if (!aAssistant && !bAssistant)
|
||||
kind = "diverge"; // tool-only turn (rare)
|
||||
else {
|
||||
divergenceNote = classifyDivergence(aAssistant!, bAssistant!, aTools, bTools);
|
||||
kind = divergenceNote ? "diverge" : "match";
|
||||
}
|
||||
|
||||
if (kind !== "match" && firstDivergenceTurn === null) firstDivergenceTurn = turn;
|
||||
pairs.push({ turn, aAssistant, bAssistant, aTools, bTools, kind, divergenceNote });
|
||||
}
|
||||
|
||||
return { a: aSide, b: bSide, pairs, firstDivergenceTurn };
|
||||
}
|
||||
|
||||
// ---------- divergence classification ----------
|
||||
|
||||
/**
|
||||
* Return a short reason string if two sides meaningfully disagree, or
|
||||
* undefined if they're close enough to call a match.
|
||||
*
|
||||
* Ranking of divergence signals (cheapest first):
|
||||
* 1. Different set of tool names → clearest diff
|
||||
* 2. Different tool args for the same tool → second-clearest
|
||||
* 3. Text similarity below threshold → fuzziest
|
||||
*/
|
||||
function classifyDivergence(
|
||||
a: TranscriptRecord,
|
||||
b: TranscriptRecord,
|
||||
aTools: TranscriptRecord[],
|
||||
bTools: TranscriptRecord[],
|
||||
): string | undefined {
|
||||
const aNames = aTools.map((t) => t.tool ?? "").sort();
|
||||
const bNames = bTools.map((t) => t.tool ?? "").sort();
|
||||
if (aNames.join(",") !== bNames.join(",")) {
|
||||
return `tool calls differ: A=[${aNames.join(",") || "—"}] B=[${bNames.join(",") || "—"}]`;
|
||||
}
|
||||
// Same tool names — did they pass different args?
|
||||
for (let i = 0; i < aTools.length; i++) {
|
||||
const at = aTools[i]!;
|
||||
const bt = bTools[i]!;
|
||||
if (at.tool !== bt.tool) continue;
|
||||
if ((at.args ?? "") !== (bt.args ?? "")) {
|
||||
return `"${at.tool}" args differ`;
|
||||
}
|
||||
}
|
||||
const simRatio = similarity(a.content, b.content);
|
||||
if (simRatio < 0.75) return `text similarity ${(simRatio * 100).toFixed(0)}%`;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalized Levenshtein similarity ratio in [0, 1]. 1 = identical.
|
||||
* Early-exits for long strings (> 2000 chars) with a cheap token-overlap
|
||||
* estimate to keep diff fast on chatty transcripts.
|
||||
*/
|
||||
export function similarity(a: string, b: string): number {
|
||||
if (a === b) return 1;
|
||||
if (!a && !b) return 1;
|
||||
if (!a || !b) return 0;
|
||||
const maxLen = Math.max(a.length, b.length);
|
||||
if (maxLen > 2000) return tokenOverlap(a, b);
|
||||
const dist = levenshtein(a, b);
|
||||
return 1 - dist / maxLen;
|
||||
}
|
||||
|
||||
function tokenOverlap(a: string, b: string): number {
|
||||
const ta = new Set(a.toLowerCase().split(/\s+/).filter(Boolean));
|
||||
const tb = new Set(b.toLowerCase().split(/\s+/).filter(Boolean));
|
||||
if (ta.size === 0 && tb.size === 0) return 1;
|
||||
let shared = 0;
|
||||
for (const t of ta) if (tb.has(t)) shared++;
|
||||
return (2 * shared) / (ta.size + tb.size);
|
||||
}
|
||||
|
||||
function levenshtein(a: string, b: string): number {
|
||||
const m = a.length;
|
||||
const n = b.length;
|
||||
if (m === 0) return n;
|
||||
if (n === 0) return m;
|
||||
let prev = new Array(n + 1);
|
||||
let curr = new Array(n + 1);
|
||||
for (let j = 0; j <= n; j++) prev[j] = j;
|
||||
for (let i = 1; i <= m; i++) {
|
||||
curr[0] = i;
|
||||
for (let j = 1; j <= n; j++) {
|
||||
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
|
||||
curr[j] = Math.min(curr[j - 1] + 1, prev[j] + 1, prev[j - 1] + cost);
|
||||
}
|
||||
[prev, curr] = [curr, prev];
|
||||
}
|
||||
return prev[n];
|
||||
}
|
||||
|
||||
// ---------- grouping ----------
|
||||
|
||||
interface TurnGroup {
|
||||
assistant?: TranscriptRecord;
|
||||
tools: TranscriptRecord[];
|
||||
}
|
||||
|
||||
function groupByTurn(records: TranscriptRecord[]): Map<number, TurnGroup> {
|
||||
const out = new Map<number, TurnGroup>();
|
||||
for (const rec of records) {
|
||||
if (rec.role === "user") continue; // user msg is input to the turn, not its output
|
||||
const g = out.get(rec.turn) ?? { tools: [] };
|
||||
if (rec.role === "assistant_final") g.assistant = rec;
|
||||
else if (rec.role === "tool") g.tools.push(rec);
|
||||
out.set(rec.turn, g);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------- rendering ----------
|
||||
|
||||
export interface RenderOptions {
|
||||
/** Monochrome output (for file redirection or piping). Defaults to true. */
|
||||
monochrome?: boolean;
|
||||
}
|
||||
|
||||
export function renderSummaryTable(report: DiffReport, _opts: RenderOptions = {}): string {
|
||||
const a = report.a;
|
||||
const b = report.b;
|
||||
const lines: string[] = [];
|
||||
lines.push("Comparing:");
|
||||
lines.push(` A ${a.label}`);
|
||||
lines.push(` B ${b.label}`);
|
||||
lines.push("");
|
||||
lines.push(row(["", "A", "B", "Δ"], [20, 14, 14, 14]));
|
||||
lines.push(
|
||||
row(["─".repeat(20), "─".repeat(14), "─".repeat(14), "─".repeat(14)], [20, 14, 14, 14]),
|
||||
);
|
||||
lines.push(statRow("model calls", a.stats.turns, b.stats.turns));
|
||||
lines.push(statRow("user turns", a.stats.userTurns, b.stats.userTurns));
|
||||
lines.push(statRow("tool calls", a.stats.toolCalls, b.stats.toolCalls));
|
||||
lines.push(
|
||||
row(
|
||||
[
|
||||
"cache hit",
|
||||
`${pct(a.stats.cacheHitRatio)}`,
|
||||
`${pct(b.stats.cacheHitRatio)}`,
|
||||
signPct(b.stats.cacheHitRatio - a.stats.cacheHitRatio),
|
||||
],
|
||||
[20, 14, 14, 14],
|
||||
),
|
||||
);
|
||||
lines.push(
|
||||
row(
|
||||
[
|
||||
"cost (USD)",
|
||||
`$${a.stats.totalCostUsd.toFixed(6)}`,
|
||||
`$${b.stats.totalCostUsd.toFixed(6)}`,
|
||||
costDelta(a.stats.totalCostUsd, b.stats.totalCostUsd),
|
||||
],
|
||||
[20, 14, 14, 14],
|
||||
),
|
||||
);
|
||||
lines.push(statRow("prefix hashes", a.stats.prefixHashes.length, b.stats.prefixHashes.length));
|
||||
lines.push("");
|
||||
|
||||
// Prefix stability story — the headline finding when comparing bench modes.
|
||||
const aPrefixStable = a.stats.prefixHashes.length <= 1;
|
||||
const bPrefixStable = b.stats.prefixHashes.length <= 1;
|
||||
if (aPrefixStable !== bPrefixStable) {
|
||||
const stable = aPrefixStable ? "A" : "B";
|
||||
const churn = aPrefixStable ? "B" : "A";
|
||||
const churnCount = aPrefixStable ? b.stats.prefixHashes.length : a.stats.prefixHashes.length;
|
||||
lines.push(
|
||||
`prefix stability: ${stable} stayed byte-stable across ${Math.max(
|
||||
a.stats.turns,
|
||||
b.stats.turns,
|
||||
)} turns; ${churn} churned ${churnCount} distinct prefixes.`,
|
||||
);
|
||||
lines.push("");
|
||||
} else if (a.stats.prefixHashes[0] && a.stats.prefixHashes[0] === b.stats.prefixHashes[0]) {
|
||||
lines.push(
|
||||
`prefix: A and B share the same prefix hash (${a.stats.prefixHashes[0].slice(0, 12)}…) — cache delta is attributable to log stability, not prompt change.`,
|
||||
);
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
if (report.firstDivergenceTurn !== null) {
|
||||
const p = report.pairs.find((p) => p.turn === report.firstDivergenceTurn);
|
||||
lines.push(
|
||||
`first divergence: turn ${report.firstDivergenceTurn} — ${p?.divergenceNote ?? "?"}`,
|
||||
);
|
||||
if (p?.aAssistant) lines.push(` A → ${truncate(p.aAssistant.content, 100)}`);
|
||||
if (p?.bAssistant) lines.push(` B → ${truncate(p.bAssistant.content, 100)}`);
|
||||
} else {
|
||||
lines.push("no material divergence detected (texts within similarity threshold).");
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
export function renderMarkdown(report: DiffReport): string {
|
||||
const a = report.a;
|
||||
const b = report.b;
|
||||
const out: string[] = [];
|
||||
out.push(`# Transcript diff: ${a.label} vs ${b.label}`);
|
||||
out.push("");
|
||||
if (a.meta || b.meta) {
|
||||
out.push("## Meta");
|
||||
out.push("");
|
||||
out.push(`| | ${a.label} | ${b.label} |`);
|
||||
out.push("|---|---|---|");
|
||||
out.push(`| source | ${a.meta?.source ?? "—"} | ${b.meta?.source ?? "—"} |`);
|
||||
out.push(`| model | ${a.meta?.model ?? "—"} | ${b.meta?.model ?? "—"} |`);
|
||||
out.push(`| task | ${a.meta?.task ?? "—"} | ${b.meta?.task ?? "—"} |`);
|
||||
out.push(`| startedAt | ${a.meta?.startedAt ?? "—"} | ${b.meta?.startedAt ?? "—"} |`);
|
||||
out.push("");
|
||||
}
|
||||
|
||||
out.push("## Summary");
|
||||
out.push("");
|
||||
out.push(`| metric | ${a.label} | ${b.label} | delta |`);
|
||||
out.push("|---|---:|---:|---:|");
|
||||
out.push(
|
||||
`| model calls | ${a.stats.turns} | ${b.stats.turns} | ${signed(b.stats.turns - a.stats.turns)} |`,
|
||||
);
|
||||
out.push(
|
||||
`| user turns | ${a.stats.userTurns} | ${b.stats.userTurns} | ${signed(b.stats.userTurns - a.stats.userTurns)} |`,
|
||||
);
|
||||
out.push(
|
||||
`| tool calls | ${a.stats.toolCalls} | ${b.stats.toolCalls} | ${signed(b.stats.toolCalls - a.stats.toolCalls)} |`,
|
||||
);
|
||||
out.push(
|
||||
`| cache hit | ${pct(a.stats.cacheHitRatio)} | ${pct(b.stats.cacheHitRatio)} | **${signPct(b.stats.cacheHitRatio - a.stats.cacheHitRatio)}** |`,
|
||||
);
|
||||
out.push(
|
||||
`| cost (USD) | $${a.stats.totalCostUsd.toFixed(6)} | $${b.stats.totalCostUsd.toFixed(6)} | ${costDelta(a.stats.totalCostUsd, b.stats.totalCostUsd)} |`,
|
||||
);
|
||||
out.push(
|
||||
`| prefix hashes | ${a.stats.prefixHashes.length} | ${b.stats.prefixHashes.length} | — |`,
|
||||
);
|
||||
out.push("");
|
||||
|
||||
out.push("## Turn-by-turn");
|
||||
out.push("");
|
||||
out.push(`| turn | kind | ${a.label} tool calls | ${b.label} tool calls | note |`);
|
||||
out.push("|---:|:---:|---|---|---|");
|
||||
for (const p of report.pairs) {
|
||||
const aTools =
|
||||
p.aTools
|
||||
.map((t) => t.tool)
|
||||
.filter(Boolean)
|
||||
.join(", ") || "—";
|
||||
const bTools =
|
||||
p.bTools
|
||||
.map((t) => t.tool)
|
||||
.filter(Boolean)
|
||||
.join(", ") || "—";
|
||||
out.push(`| ${p.turn} | ${p.kind} | ${aTools} | ${bTools} | ${p.divergenceNote ?? ""} |`);
|
||||
}
|
||||
out.push("");
|
||||
|
||||
if (report.firstDivergenceTurn !== null) {
|
||||
const p = report.pairs.find((x) => x.turn === report.firstDivergenceTurn);
|
||||
out.push(`## First divergence (turn ${report.firstDivergenceTurn})`);
|
||||
out.push("");
|
||||
out.push(p?.divergenceNote ?? "");
|
||||
out.push("");
|
||||
if (p?.aAssistant) {
|
||||
out.push(`**${a.label}:**`);
|
||||
out.push("");
|
||||
out.push("```");
|
||||
out.push(p.aAssistant.content);
|
||||
out.push("```");
|
||||
out.push("");
|
||||
}
|
||||
if (p?.bAssistant) {
|
||||
out.push(`**${b.label}:**`);
|
||||
out.push("");
|
||||
out.push("```");
|
||||
out.push(p.bAssistant.content);
|
||||
out.push("```");
|
||||
out.push("");
|
||||
}
|
||||
}
|
||||
return out.join("\n");
|
||||
}
|
||||
|
||||
// ---------- formatting helpers ----------
|
||||
|
||||
function row(cols: string[], widths: number[]): string {
|
||||
return cols.map((c, i) => padRight(c, widths[i] ?? c.length)).join(" ");
|
||||
}
|
||||
|
||||
function statRow(label: string, av: number, bv: number): string {
|
||||
return row([label, `${av}`, `${bv}`, signed(bv - av)], [20, 14, 14, 14]);
|
||||
}
|
||||
|
||||
function padRight(s: string, w: number): string {
|
||||
return s.length >= w ? s : s + " ".repeat(w - s.length);
|
||||
}
|
||||
|
||||
function signed(n: number): string {
|
||||
if (n === 0) return "0";
|
||||
return `${n > 0 ? "+" : ""}${n}`;
|
||||
}
|
||||
|
||||
function signPct(diff: number): string {
|
||||
if (diff === 0) return "0pp";
|
||||
const s = (diff * 100).toFixed(1);
|
||||
return `${diff > 0 ? "+" : ""}${s}pp`;
|
||||
}
|
||||
|
||||
function pct(x: number): string {
|
||||
return `${(x * 100).toFixed(1)}%`;
|
||||
}
|
||||
|
||||
function costDelta(a: number, b: number): string {
|
||||
if (a === 0 && b === 0) return "—";
|
||||
if (a === 0) return "new";
|
||||
const pctChange = ((b - a) / a) * 100;
|
||||
return `${pctChange > 0 ? "+" : ""}${pctChange.toFixed(1)}%`;
|
||||
}
|
||||
|
||||
function truncate(s: string, n: number): string {
|
||||
return s.length > n ? `${s.slice(0, n)}…` : s;
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
/**
|
||||
* Minimal `.env` loader; no dependency on dotenv.
|
||||
*
|
||||
* Reads KEY=VALUE lines and populates `process.env` for keys not already set.
|
||||
* Silently no-ops if the file is missing. Safe to call from library entry
|
||||
* points, CLI commands, examples, and benchmark runners.
|
||||
*/
|
||||
export function loadDotenv(path = ".env"): void {
|
||||
let raw: string;
|
||||
try {
|
||||
raw = readFileSync(resolve(process.cwd(), path), "utf8");
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
for (const line of raw.split(/\r?\n/)) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith("#")) continue;
|
||||
const eq = trimmed.indexOf("=");
|
||||
if (eq === -1) continue;
|
||||
const key = trimmed.slice(0, eq).trim();
|
||||
let value = trimmed.slice(eq + 1).trim();
|
||||
if (
|
||||
(value.startsWith('"') && value.endsWith('"')) ||
|
||||
(value.startsWith("'") && value.endsWith("'"))
|
||||
) {
|
||||
value = value.slice(1, -1);
|
||||
}
|
||||
if (process.env[key] === undefined) process.env[key] = value;
|
||||
}
|
||||
}
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* Pillar 2 — R1 Thought Harvesting.
|
||||
*
|
||||
* Takes the `reasoning_content` emitted by a thinking model (deepseek-reasoner
|
||||
* / R1) and extracts a structured plan state by making a cheap secondary call
|
||||
* to V3 in JSON mode. The typed state is intended for the orchestrator to
|
||||
* branch on — e.g. trigger self-consistency sampling when `uncertainties.length
|
||||
* > 2`, or surface the subgoals to the user.
|
||||
*
|
||||
* Opt-in: loops disable harvesting by default. Failures (bad JSON, API error,
|
||||
* empty reasoning) return an empty TypedPlanState — the main turn is never
|
||||
* aborted because of a harvest hiccup.
|
||||
*/
|
||||
|
||||
import type { DeepSeekClient } from "./client.js";
|
||||
|
||||
export interface TypedPlanState {
|
||||
subgoals: string[];
|
||||
hypotheses: string[];
|
||||
uncertainties: string[];
|
||||
rejectedPaths: string[];
|
||||
}
|
||||
|
||||
export interface HarvestOptions {
|
||||
/** Model used for the extraction call. Defaults to the cheap chat model. */
|
||||
model?: string;
|
||||
/** Cap on how many items land in each array. Default 5. */
|
||||
maxItems?: number;
|
||||
/** Per-item character cap. Default 80. */
|
||||
maxItemLen?: number;
|
||||
/** Abort the extraction if R1 reasoning is shorter than this. Default 40. */
|
||||
minReasoningLen?: number;
|
||||
}
|
||||
|
||||
export function emptyPlanState(): TypedPlanState {
|
||||
return { subgoals: [], hypotheses: [], uncertainties: [], rejectedPaths: [] };
|
||||
}
|
||||
|
||||
export function isPlanStateEmpty(s: TypedPlanState | null | undefined): boolean {
|
||||
if (!s) return true;
|
||||
return (
|
||||
s.subgoals.length === 0 &&
|
||||
s.hypotheses.length === 0 &&
|
||||
s.uncertainties.length === 0 &&
|
||||
s.rejectedPaths.length === 0
|
||||
);
|
||||
}
|
||||
|
||||
const SYSTEM_PROMPT = `You extract a typed plan state from a reasoning trace produced by another LLM.
|
||||
Output ONLY a JSON object. No markdown, no prose, no backticks.
|
||||
|
||||
Schema:
|
||||
{
|
||||
"subgoals": string[], // concrete intermediate objectives the trace identifies
|
||||
"hypotheses": string[], // candidate approaches or assumptions being weighed
|
||||
"uncertainties": string[], // facts the trace flags as unclear / to verify
|
||||
"rejectedPaths": string[] // approaches the trace considered and then abandoned
|
||||
}
|
||||
|
||||
Constraints:
|
||||
- Every field must be present. Use [] if not applicable.
|
||||
- Each array has at most {maxItems} items.
|
||||
- Each item is plain text, at most {maxItemLen} characters, no markdown.
|
||||
- Write in the same language as the trace (Chinese in → Chinese out, etc.).
|
||||
- Do not quote back the trace; write short, specific phrases.`;
|
||||
|
||||
export async function harvest(
|
||||
reasoningContent: string | null | undefined,
|
||||
client?: DeepSeekClient,
|
||||
options: HarvestOptions = {},
|
||||
): Promise<TypedPlanState> {
|
||||
if (!client || !reasoningContent) return emptyPlanState();
|
||||
const minLen = options.minReasoningLen ?? 40;
|
||||
const trimmed = reasoningContent.trim();
|
||||
if (trimmed.length < minLen) return emptyPlanState();
|
||||
|
||||
const model = options.model ?? "deepseek-chat";
|
||||
const maxItems = options.maxItems ?? 5;
|
||||
const maxItemLen = options.maxItemLen ?? 80;
|
||||
const system = SYSTEM_PROMPT.replace("{maxItems}", String(maxItems)).replace(
|
||||
"{maxItemLen}",
|
||||
String(maxItemLen),
|
||||
);
|
||||
|
||||
try {
|
||||
const resp = await client.chat({
|
||||
model,
|
||||
messages: [
|
||||
{ role: "system", content: system },
|
||||
{ role: "user", content: trimmed },
|
||||
],
|
||||
responseFormat: { type: "json_object" },
|
||||
temperature: 0,
|
||||
maxTokens: 600,
|
||||
});
|
||||
return parsePlanState(resp.content, maxItems, maxItemLen);
|
||||
} catch {
|
||||
return emptyPlanState();
|
||||
}
|
||||
}
|
||||
|
||||
function parsePlanState(raw: string, maxItems: number, maxItemLen: number): TypedPlanState {
|
||||
const text = (raw ?? "").trim();
|
||||
if (!text) return emptyPlanState();
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(text);
|
||||
} catch {
|
||||
// Occasionally a model wraps JSON in fences despite instructions.
|
||||
const match = text.match(/\{[\s\S]*\}/);
|
||||
if (!match) return emptyPlanState();
|
||||
try {
|
||||
parsed = JSON.parse(match[0]);
|
||||
} catch {
|
||||
return emptyPlanState();
|
||||
}
|
||||
}
|
||||
if (!parsed || typeof parsed !== "object") return emptyPlanState();
|
||||
const obj = parsed as Record<string, unknown>;
|
||||
return {
|
||||
subgoals: sanitizeArray(obj.subgoals, maxItems, maxItemLen),
|
||||
hypotheses: sanitizeArray(obj.hypotheses, maxItems, maxItemLen),
|
||||
uncertainties: sanitizeArray(obj.uncertainties, maxItems, maxItemLen),
|
||||
rejectedPaths: sanitizeArray(obj.rejectedPaths ?? obj.rejected_paths, maxItems, maxItemLen),
|
||||
};
|
||||
}
|
||||
|
||||
function sanitizeArray(raw: unknown, maxItems: number, maxItemLen: number): string[] {
|
||||
if (!Array.isArray(raw)) return [];
|
||||
const out: string[] = [];
|
||||
for (const item of raw) {
|
||||
if (out.length >= maxItems) break;
|
||||
if (typeof item !== "string") continue;
|
||||
const cleaned = item.trim().replace(/\s+/g, " ");
|
||||
if (!cleaned) continue;
|
||||
out.push(cleaned.length <= maxItemLen ? cleaned : `${cleaned.slice(0, maxItemLen - 1)}…`);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
/** Reasonix — DeepSeek-native agent framework. Library entry point. */
|
||||
|
||||
export { DeepSeekClient, Usage } from "./client.js";
|
||||
export type { ChatResponse, StreamChunk, DeepSeekClientOptions } from "./client.js";
|
||||
|
||||
export { CacheFirstLoop } from "./loop.js";
|
||||
export type {
|
||||
CacheFirstLoopOptions,
|
||||
LoopEvent,
|
||||
EventRole,
|
||||
BranchSummary,
|
||||
BranchProgress,
|
||||
ReconfigurableOptions,
|
||||
} from "./loop.js";
|
||||
|
||||
export { runBranches, defaultSelector, aggregateBranchUsage } from "./consistency.js";
|
||||
export type {
|
||||
BranchOptions,
|
||||
BranchSample,
|
||||
BranchResult,
|
||||
BranchSelector,
|
||||
} from "./consistency.js";
|
||||
|
||||
export { ImmutablePrefix, AppendOnlyLog, VolatileScratch } from "./memory.js";
|
||||
export type { ImmutablePrefixOptions } from "./memory.js";
|
||||
|
||||
export { ToolRegistry } from "./tools.js";
|
||||
export type { ToolDefinition } from "./tools.js";
|
||||
|
||||
export { SessionStats, costUsd, claudeEquivalentCost } from "./telemetry.js";
|
||||
export type { TurnStats, SessionSummary } from "./telemetry.js";
|
||||
|
||||
export {
|
||||
ToolCallRepair,
|
||||
scavengeToolCalls,
|
||||
repairTruncatedJson,
|
||||
StormBreaker,
|
||||
analyzeSchema,
|
||||
flattenSchema,
|
||||
nestArguments,
|
||||
} from "./repair/index.js";
|
||||
export type {
|
||||
RepairReport,
|
||||
ToolCallRepairOptions,
|
||||
ScavengeOptions,
|
||||
ScavengeResult,
|
||||
TruncationRepairResult,
|
||||
FlattenDecision,
|
||||
} from "./repair/index.js";
|
||||
|
||||
export { harvest, emptyPlanState, isPlanStateEmpty } from "./harvest.js";
|
||||
export type { TypedPlanState, HarvestOptions } from "./harvest.js";
|
||||
|
||||
export {
|
||||
appendSessionMessage,
|
||||
deleteSession,
|
||||
listSessions,
|
||||
loadSessionMessages,
|
||||
sanitizeName as sanitizeSessionName,
|
||||
sessionPath,
|
||||
sessionsDir,
|
||||
} from "./session.js";
|
||||
export type { SessionInfo } from "./session.js";
|
||||
|
||||
export { loadDotenv } from "./env.js";
|
||||
|
||||
export {
|
||||
openTranscriptFile,
|
||||
parseTranscript,
|
||||
readTranscript,
|
||||
recordFromLoopEvent,
|
||||
writeMeta,
|
||||
writeRecord,
|
||||
} from "./transcript.js";
|
||||
export type { TranscriptRecord, TranscriptMeta, ReadTranscriptResult } from "./transcript.js";
|
||||
|
||||
export { computeReplayStats, replayFromFile } from "./replay.js";
|
||||
export type { ReplayStats } from "./replay.js";
|
||||
|
||||
export {
|
||||
diffTranscripts,
|
||||
renderMarkdown as renderDiffMarkdown,
|
||||
renderSummaryTable as renderDiffSummary,
|
||||
similarity,
|
||||
} from "./diff.js";
|
||||
export type { DiffReport, DiffSide, TurnPair, RenderOptions as DiffRenderOptions } from "./diff.js";
|
||||
|
||||
export { fetchWithRetry } from "./retry.js";
|
||||
export type { RetryOptions, RetryInfo } from "./retry.js";
|
||||
|
||||
export {
|
||||
defaultConfigPath,
|
||||
isPlausibleKey,
|
||||
loadApiKey,
|
||||
readConfig,
|
||||
redactKey,
|
||||
saveApiKey,
|
||||
writeConfig,
|
||||
} from "./config.js";
|
||||
export type { ReasonixConfig } from "./config.js";
|
||||
|
||||
export type {
|
||||
ChatMessage,
|
||||
ToolCall,
|
||||
ToolSpec,
|
||||
ToolFunctionSpec,
|
||||
Role,
|
||||
JSONSchema,
|
||||
} from "./types.js";
|
||||
|
||||
export const VERSION = "0.2.2";
|
||||
+483
@@ -0,0 +1,483 @@
|
||||
import { type DeepSeekClient, Usage } from "./client.js";
|
||||
import {
|
||||
type BranchOptions,
|
||||
type BranchSample,
|
||||
aggregateBranchUsage,
|
||||
runBranches,
|
||||
} from "./consistency.js";
|
||||
import { type HarvestOptions, type TypedPlanState, emptyPlanState, harvest } from "./harvest.js";
|
||||
import { AppendOnlyLog, type ImmutablePrefix, VolatileScratch } from "./memory.js";
|
||||
import { type RepairReport, ToolCallRepair } from "./repair/index.js";
|
||||
import { appendSessionMessage, loadSessionMessages } from "./session.js";
|
||||
import { SessionStats, type TurnStats } from "./telemetry.js";
|
||||
import { ToolRegistry } from "./tools.js";
|
||||
import type { ChatMessage, ToolCall } from "./types.js";
|
||||
|
||||
export type EventRole =
|
||||
| "assistant_delta"
|
||||
| "assistant_final"
|
||||
| "tool"
|
||||
| "done"
|
||||
| "error"
|
||||
| "branch_start"
|
||||
| "branch_progress"
|
||||
| "branch_done";
|
||||
|
||||
export interface BranchSummary {
|
||||
budget: number;
|
||||
chosenIndex: number;
|
||||
uncertainties: number[]; // per-sample uncertainty counts
|
||||
temperatures: number[];
|
||||
}
|
||||
|
||||
export interface BranchProgress {
|
||||
completed: number;
|
||||
total: number;
|
||||
latestIndex: number;
|
||||
latestTemperature: number;
|
||||
latestUncertainties: number;
|
||||
}
|
||||
|
||||
export interface LoopEvent {
|
||||
turn: number;
|
||||
role: EventRole;
|
||||
content: string;
|
||||
reasoningDelta?: string;
|
||||
toolName?: string;
|
||||
/**
|
||||
* Raw JSON-string arguments the model sent for a tool call (role === "tool").
|
||||
* Populated so transcripts can persist *why* a tool was called, not just
|
||||
* what it returned. Needed by `reasonix diff` to explain divergences.
|
||||
*/
|
||||
toolArgs?: string;
|
||||
stats?: TurnStats;
|
||||
planState?: TypedPlanState;
|
||||
repair?: RepairReport;
|
||||
branch?: BranchSummary;
|
||||
branchProgress?: BranchProgress;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface CacheFirstLoopOptions {
|
||||
client: DeepSeekClient;
|
||||
prefix: ImmutablePrefix;
|
||||
tools?: ToolRegistry;
|
||||
model?: string;
|
||||
maxToolIters?: number;
|
||||
stream?: boolean;
|
||||
/**
|
||||
* Pillar 2 — structured harvesting of R1 reasoning into a typed plan state.
|
||||
* Pass `true` for defaults or an options object. Off by default (adds a
|
||||
* cheap but non-zero V3 call per turn).
|
||||
*/
|
||||
harvest?: boolean | HarvestOptions;
|
||||
/**
|
||||
* Self-consistency branching. Pass a number for just a budget (e.g. 3) or
|
||||
* a full `BranchOptions` object. Disables streaming for the branched turn
|
||||
* because all samples must complete before selection. Auto-enables harvest
|
||||
* since the default selector scores samples by plan-state uncertainty.
|
||||
*/
|
||||
branch?: number | BranchOptions;
|
||||
/**
|
||||
* Session name. When set, the loop pre-loads the session's prior messages
|
||||
* into its log on construction, and appends every new log entry to
|
||||
* `~/.reasonix/sessions/<name>.jsonl` so the next run can resume.
|
||||
*/
|
||||
session?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pillar 1 — Cache-First Loop.
|
||||
*
|
||||
* - prefix is immutable (cache target)
|
||||
* - log is append-only (preserves prior-turn prefix)
|
||||
* - scratch is per-turn volatile (never sent upstream)
|
||||
*
|
||||
* Yields a stream of events so a TUI can render progressively.
|
||||
*/
|
||||
export interface ReconfigurableOptions {
|
||||
model?: string;
|
||||
harvest?: boolean | HarvestOptions;
|
||||
branch?: number | BranchOptions;
|
||||
stream?: boolean;
|
||||
}
|
||||
|
||||
export class CacheFirstLoop {
|
||||
readonly client: DeepSeekClient;
|
||||
readonly prefix: ImmutablePrefix;
|
||||
readonly tools: ToolRegistry;
|
||||
readonly maxToolIters: number;
|
||||
readonly log = new AppendOnlyLog();
|
||||
readonly scratch = new VolatileScratch();
|
||||
readonly stats = new SessionStats();
|
||||
readonly repair: ToolCallRepair;
|
||||
|
||||
// Mutable via configure() — slash commands in the TUI / library callers tweak
|
||||
// these mid-session so users don't have to restart to try harvest or branch.
|
||||
model: string;
|
||||
stream: boolean;
|
||||
harvestEnabled: boolean;
|
||||
harvestOptions: HarvestOptions;
|
||||
branchEnabled: boolean;
|
||||
branchOptions: BranchOptions;
|
||||
sessionName: string | null;
|
||||
|
||||
/** Number of messages that were pre-loaded from the session file. */
|
||||
readonly resumedMessageCount: number;
|
||||
|
||||
private _turn = 0;
|
||||
private _streamPreference: boolean;
|
||||
|
||||
constructor(opts: CacheFirstLoopOptions) {
|
||||
this.client = opts.client;
|
||||
this.prefix = opts.prefix;
|
||||
this.tools = opts.tools ?? new ToolRegistry();
|
||||
this.model = opts.model ?? "deepseek-chat";
|
||||
this.maxToolIters = opts.maxToolIters ?? 8;
|
||||
|
||||
// Resolve branch config first (since it forces harvest on).
|
||||
if (typeof opts.branch === "number") {
|
||||
this.branchOptions = { budget: opts.branch };
|
||||
} else if (opts.branch && typeof opts.branch === "object") {
|
||||
this.branchOptions = opts.branch;
|
||||
} else {
|
||||
this.branchOptions = {};
|
||||
}
|
||||
this.branchEnabled = (this.branchOptions.budget ?? 1) > 1;
|
||||
|
||||
// Branching requires harvest for its default selector to work.
|
||||
const harvestForced = this.branchEnabled;
|
||||
this.harvestEnabled =
|
||||
harvestForced ||
|
||||
opts.harvest === true ||
|
||||
(typeof opts.harvest === "object" && opts.harvest !== null);
|
||||
this.harvestOptions =
|
||||
typeof opts.harvest === "object" && opts.harvest !== null
|
||||
? opts.harvest
|
||||
: (this.branchOptions.harvestOptions ?? {});
|
||||
|
||||
// Streaming is incompatible with branching (need all samples to select).
|
||||
this._streamPreference = opts.stream ?? true;
|
||||
this.stream = this.branchEnabled ? false : this._streamPreference;
|
||||
|
||||
const allowedNames = new Set([...this.prefix.toolSpecs.map((s) => s.function.name)]);
|
||||
this.repair = new ToolCallRepair({ allowedToolNames: allowedNames });
|
||||
|
||||
// Session resume: pre-load prior messages into the log if a session name
|
||||
// is provided. New messages appended to the log are also persisted.
|
||||
this.sessionName = opts.session ?? null;
|
||||
if (this.sessionName) {
|
||||
const prior = loadSessionMessages(this.sessionName);
|
||||
for (const msg of prior) this.log.append(msg);
|
||||
this.resumedMessageCount = prior.length;
|
||||
} else {
|
||||
this.resumedMessageCount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
private appendAndPersist(message: ChatMessage): void {
|
||||
this.log.append(message);
|
||||
if (this.sessionName) {
|
||||
try {
|
||||
appendSessionMessage(this.sessionName, message);
|
||||
} catch {
|
||||
/* disk full or permission denied shouldn't kill the chat */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconfigure model/harvest/branch/stream mid-session. The loop's log,
|
||||
* scratch, and stats are preserved — only the per-turn behavior changes.
|
||||
* Used by the TUI's slash commands and by library callers who want to
|
||||
* flip a knob between turns.
|
||||
*/
|
||||
configure(opts: ReconfigurableOptions): void {
|
||||
if (opts.model !== undefined) this.model = opts.model;
|
||||
if (opts.stream !== undefined) this._streamPreference = opts.stream;
|
||||
|
||||
if (opts.branch !== undefined) {
|
||||
if (typeof opts.branch === "number") {
|
||||
this.branchOptions = { budget: opts.branch };
|
||||
} else if (opts.branch && typeof opts.branch === "object") {
|
||||
this.branchOptions = opts.branch;
|
||||
} else {
|
||||
this.branchOptions = {};
|
||||
}
|
||||
this.branchEnabled = (this.branchOptions.budget ?? 1) > 1;
|
||||
}
|
||||
|
||||
if (opts.harvest !== undefined) {
|
||||
const want =
|
||||
opts.harvest === true || (typeof opts.harvest === "object" && opts.harvest !== null);
|
||||
this.harvestEnabled = want || this.branchEnabled;
|
||||
if (typeof opts.harvest === "object" && opts.harvest !== null) {
|
||||
this.harvestOptions = opts.harvest;
|
||||
}
|
||||
} else if (this.branchEnabled) {
|
||||
// branch turned on without explicit harvest → force it on
|
||||
this.harvestEnabled = true;
|
||||
}
|
||||
|
||||
// Branching always forces non-streaming; otherwise honor preference.
|
||||
this.stream = this.branchEnabled ? false : this._streamPreference;
|
||||
}
|
||||
|
||||
private buildMessages(pendingUser: string | null): ChatMessage[] {
|
||||
const msgs: ChatMessage[] = [...this.prefix.toMessages(), ...this.log.toMessages()];
|
||||
if (pendingUser !== null) msgs.push({ role: "user", content: pendingUser });
|
||||
return msgs;
|
||||
}
|
||||
|
||||
async *step(userInput: string): AsyncGenerator<LoopEvent> {
|
||||
this._turn++;
|
||||
this.scratch.reset();
|
||||
let pendingUser: string | null = userInput;
|
||||
const toolSpecs = this.prefix.tools();
|
||||
|
||||
for (let iter = 0; iter < this.maxToolIters; iter++) {
|
||||
const messages = this.buildMessages(pendingUser);
|
||||
|
||||
let assistantContent = "";
|
||||
let reasoningContent = "";
|
||||
let toolCalls: ToolCall[] = [];
|
||||
let usage: TurnStats["usage"] | null = null;
|
||||
|
||||
let branchSummary: BranchSummary | undefined;
|
||||
let preHarvestedPlanState: TypedPlanState | undefined;
|
||||
|
||||
try {
|
||||
if (this.branchEnabled) {
|
||||
const budget = this.branchOptions.budget ?? 1;
|
||||
yield {
|
||||
turn: this._turn,
|
||||
role: "branch_start",
|
||||
content: "",
|
||||
branchProgress: {
|
||||
completed: 0,
|
||||
total: budget,
|
||||
latestIndex: -1,
|
||||
latestTemperature: -1,
|
||||
latestUncertainties: -1,
|
||||
},
|
||||
};
|
||||
|
||||
// Queue samples as they complete so we can yield progress events
|
||||
// in resolution order (not launch order).
|
||||
const queue: BranchSample[] = [];
|
||||
let waiter: ((s: BranchSample) => void) | null = null;
|
||||
|
||||
const onSampleDone = (sample: BranchSample) => {
|
||||
if (waiter) {
|
||||
const w = waiter;
|
||||
waiter = null;
|
||||
w(sample);
|
||||
} else {
|
||||
queue.push(sample);
|
||||
}
|
||||
};
|
||||
|
||||
const branchPromise = runBranches(
|
||||
this.client,
|
||||
{
|
||||
model: this.model,
|
||||
messages,
|
||||
tools: toolSpecs.length ? toolSpecs : undefined,
|
||||
},
|
||||
{
|
||||
...this.branchOptions,
|
||||
harvestOptions: this.harvestOptions,
|
||||
onSampleDone,
|
||||
},
|
||||
);
|
||||
|
||||
for (let k = 0; k < budget; k++) {
|
||||
const sample: BranchSample =
|
||||
queue.shift() ??
|
||||
(await new Promise<BranchSample>((resolve) => {
|
||||
waiter = resolve;
|
||||
}));
|
||||
yield {
|
||||
turn: this._turn,
|
||||
role: "branch_progress",
|
||||
content: "",
|
||||
branchProgress: {
|
||||
completed: k + 1,
|
||||
total: budget,
|
||||
latestIndex: sample.index,
|
||||
latestTemperature: sample.temperature,
|
||||
latestUncertainties: sample.planState.uncertainties.length,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const result = await branchPromise;
|
||||
assistantContent = result.chosen.response.content;
|
||||
reasoningContent = result.chosen.response.reasoningContent ?? "";
|
||||
toolCalls = result.chosen.response.toolCalls;
|
||||
|
||||
// Cost accounting: sum usage across ALL samples, not just the winner.
|
||||
// (We paid for all three.) Harvest-call tokens are not tracked; they
|
||||
// amount to rounding error compared to the main R1 calls.
|
||||
const agg = aggregateBranchUsage(result.samples);
|
||||
usage = new Usage(
|
||||
agg.promptTokens,
|
||||
agg.completionTokens,
|
||||
agg.totalTokens,
|
||||
agg.promptCacheHitTokens,
|
||||
agg.promptCacheMissTokens,
|
||||
);
|
||||
preHarvestedPlanState = result.chosen.planState;
|
||||
branchSummary = summarizeBranch(result.chosen, result.samples);
|
||||
yield {
|
||||
turn: this._turn,
|
||||
role: "branch_done",
|
||||
content: "",
|
||||
branch: branchSummary,
|
||||
};
|
||||
} else if (this.stream) {
|
||||
const callBuf: Map<number, ToolCall> = new Map();
|
||||
for await (const chunk of this.client.stream({
|
||||
model: this.model,
|
||||
messages,
|
||||
tools: toolSpecs.length ? toolSpecs : undefined,
|
||||
})) {
|
||||
if (chunk.contentDelta) {
|
||||
assistantContent += chunk.contentDelta;
|
||||
yield {
|
||||
turn: this._turn,
|
||||
role: "assistant_delta",
|
||||
content: chunk.contentDelta,
|
||||
};
|
||||
}
|
||||
if (chunk.reasoningDelta) {
|
||||
reasoningContent += chunk.reasoningDelta;
|
||||
yield {
|
||||
turn: this._turn,
|
||||
role: "assistant_delta",
|
||||
content: "",
|
||||
reasoningDelta: chunk.reasoningDelta,
|
||||
};
|
||||
}
|
||||
if (chunk.toolCallDelta) {
|
||||
const d = chunk.toolCallDelta;
|
||||
const cur = callBuf.get(d.index) ?? {
|
||||
id: d.id,
|
||||
type: "function" as const,
|
||||
function: { name: "", arguments: "" },
|
||||
};
|
||||
if (d.id) cur.id = d.id;
|
||||
if (d.name) cur.function.name = (cur.function.name ?? "") + d.name;
|
||||
if (d.argumentsDelta)
|
||||
cur.function.arguments = (cur.function.arguments ?? "") + d.argumentsDelta;
|
||||
callBuf.set(d.index, cur);
|
||||
}
|
||||
if (chunk.usage) usage = chunk.usage;
|
||||
}
|
||||
toolCalls = [...callBuf.values()];
|
||||
} else {
|
||||
const resp = await this.client.chat({
|
||||
model: this.model,
|
||||
messages,
|
||||
tools: toolSpecs.length ? toolSpecs : undefined,
|
||||
});
|
||||
assistantContent = resp.content;
|
||||
reasoningContent = resp.reasoningContent ?? "";
|
||||
toolCalls = resp.toolCalls;
|
||||
usage = resp.usage;
|
||||
}
|
||||
} catch (err) {
|
||||
yield {
|
||||
turn: this._turn,
|
||||
role: "error",
|
||||
content: "",
|
||||
error: (err as Error).message,
|
||||
};
|
||||
return;
|
||||
}
|
||||
|
||||
const turnStats = this.stats.record(this._turn, this.model, usage ?? new Usage());
|
||||
|
||||
// Commit the user turn to the log only on success of the first round-trip.
|
||||
if (pendingUser !== null) {
|
||||
this.appendAndPersist({ role: "user", content: pendingUser });
|
||||
pendingUser = null;
|
||||
}
|
||||
|
||||
this.scratch.reasoning = reasoningContent || null;
|
||||
const planState = preHarvestedPlanState
|
||||
? preHarvestedPlanState
|
||||
: this.harvestEnabled
|
||||
? await harvest(reasoningContent || null, this.client, this.harvestOptions)
|
||||
: emptyPlanState();
|
||||
|
||||
const { calls: repairedCalls, report } = this.repair.process(
|
||||
toolCalls,
|
||||
reasoningContent || null,
|
||||
);
|
||||
|
||||
this.appendAndPersist(this.assistantMessage(assistantContent, repairedCalls));
|
||||
|
||||
yield {
|
||||
turn: this._turn,
|
||||
role: "assistant_final",
|
||||
content: assistantContent,
|
||||
stats: turnStats,
|
||||
planState,
|
||||
repair: report,
|
||||
branch: branchSummary,
|
||||
};
|
||||
|
||||
if (repairedCalls.length === 0) {
|
||||
yield { turn: this._turn, role: "done", content: assistantContent };
|
||||
return;
|
||||
}
|
||||
|
||||
for (const call of repairedCalls) {
|
||||
const name = call.function?.name ?? "";
|
||||
const args = call.function?.arguments ?? "{}";
|
||||
const result = await this.tools.dispatch(name, args);
|
||||
this.appendAndPersist({
|
||||
role: "tool",
|
||||
tool_call_id: call.id ?? "",
|
||||
name,
|
||||
content: result,
|
||||
});
|
||||
yield {
|
||||
turn: this._turn,
|
||||
role: "tool",
|
||||
content: result,
|
||||
toolName: name,
|
||||
toolArgs: args,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
yield { turn: this._turn, role: "done", content: "[max_tool_iters reached]" };
|
||||
}
|
||||
|
||||
async run(userInput: string, onEvent?: (ev: LoopEvent) => void): Promise<string> {
|
||||
let final = "";
|
||||
for await (const ev of this.step(userInput)) {
|
||||
onEvent?.(ev);
|
||||
if (ev.role === "assistant_final") final = ev.content;
|
||||
if (ev.role === "done") break;
|
||||
}
|
||||
return final;
|
||||
}
|
||||
|
||||
private assistantMessage(content: string, toolCalls: ToolCall[]): ChatMessage {
|
||||
const msg: ChatMessage = { role: "assistant", content };
|
||||
if (toolCalls.length > 0) msg.tool_calls = toolCalls;
|
||||
return msg;
|
||||
}
|
||||
}
|
||||
|
||||
function summarizeBranch(chosen: BranchSample, samples: BranchSample[]): BranchSummary {
|
||||
return {
|
||||
budget: samples.length,
|
||||
chosenIndex: chosen.index,
|
||||
uncertainties: samples.map((s) => s.planState.uncertainties.length),
|
||||
temperatures: samples.map((s) => s.temperature),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import type { ChatMessage, ToolSpec } from "./types.js";
|
||||
|
||||
export interface ImmutablePrefixOptions {
|
||||
system: string;
|
||||
toolSpecs?: readonly ToolSpec[];
|
||||
fewShots?: readonly ChatMessage[];
|
||||
}
|
||||
|
||||
export class ImmutablePrefix {
|
||||
readonly system: string;
|
||||
readonly toolSpecs: readonly ToolSpec[];
|
||||
readonly fewShots: readonly ChatMessage[];
|
||||
|
||||
constructor(opts: ImmutablePrefixOptions) {
|
||||
this.system = opts.system;
|
||||
this.toolSpecs = Object.freeze([...(opts.toolSpecs ?? [])]);
|
||||
this.fewShots = Object.freeze([...(opts.fewShots ?? [])]);
|
||||
}
|
||||
|
||||
toMessages(): ChatMessage[] {
|
||||
return [{ role: "system", content: this.system }, ...this.fewShots.map((m) => ({ ...m }))];
|
||||
}
|
||||
|
||||
tools(): ToolSpec[] {
|
||||
return this.toolSpecs.map((t) => structuredClone(t) as ToolSpec);
|
||||
}
|
||||
|
||||
get fingerprint(): string {
|
||||
const blob = JSON.stringify({
|
||||
system: this.system,
|
||||
tools: this.toolSpecs,
|
||||
shots: this.fewShots,
|
||||
});
|
||||
return createHash("sha256").update(blob).digest("hex").slice(0, 16);
|
||||
}
|
||||
}
|
||||
|
||||
export class AppendOnlyLog {
|
||||
private _entries: ChatMessage[] = [];
|
||||
|
||||
append(message: ChatMessage): void {
|
||||
if (!message || typeof message !== "object" || !("role" in message)) {
|
||||
throw new Error(`invalid log entry: ${JSON.stringify(message)}`);
|
||||
}
|
||||
this._entries.push(message);
|
||||
}
|
||||
|
||||
extend(messages: ChatMessage[]): void {
|
||||
for (const m of messages) this.append(m);
|
||||
}
|
||||
|
||||
get entries(): readonly ChatMessage[] {
|
||||
return this._entries;
|
||||
}
|
||||
|
||||
toMessages(): ChatMessage[] {
|
||||
return this._entries.map((e) => ({ ...e }));
|
||||
}
|
||||
|
||||
get length(): number {
|
||||
return this._entries.length;
|
||||
}
|
||||
}
|
||||
|
||||
export class VolatileScratch {
|
||||
reasoning: string | null = null;
|
||||
planState: Record<string, unknown> | null = null;
|
||||
notes: string[] = [];
|
||||
|
||||
reset(): void {
|
||||
this.reasoning = null;
|
||||
this.planState = null;
|
||||
this.notes = [];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
/**
|
||||
* Schema flattening for DeepSeek tool calls.
|
||||
*
|
||||
* DeepSeek loses arguments on schemas that are deep (>2 levels of nesting) or
|
||||
* wide (>10 leaf parameters). This module transforms such schemas into a
|
||||
* dot-notation flat schema and re-nests the model's arguments before dispatch.
|
||||
*
|
||||
* Example:
|
||||
* { user: { profile: { name, age } } } ⇄ "user.profile.name", "user.profile.age"
|
||||
*/
|
||||
|
||||
import type { JSONSchema } from "../types.js";
|
||||
|
||||
export interface FlattenDecision {
|
||||
shouldFlatten: boolean;
|
||||
leafCount: number;
|
||||
maxDepth: number;
|
||||
}
|
||||
|
||||
export function analyzeSchema(schema: JSONSchema | undefined): FlattenDecision {
|
||||
if (!schema) return { shouldFlatten: false, leafCount: 0, maxDepth: 0 };
|
||||
let leafCount = 0;
|
||||
let maxDepth = 0;
|
||||
walk(schema, 0, (depth, isLeaf) => {
|
||||
if (isLeaf) leafCount++;
|
||||
if (depth > maxDepth) maxDepth = depth;
|
||||
});
|
||||
return {
|
||||
shouldFlatten: leafCount > 10 || maxDepth > 2,
|
||||
leafCount,
|
||||
maxDepth,
|
||||
};
|
||||
}
|
||||
|
||||
export function flattenSchema(schema: JSONSchema): JSONSchema {
|
||||
const flatProps: Record<string, JSONSchema> = {};
|
||||
const required: string[] = [];
|
||||
collect("", schema, flatProps, required, true);
|
||||
return {
|
||||
type: "object",
|
||||
properties: flatProps,
|
||||
required,
|
||||
};
|
||||
}
|
||||
|
||||
export function nestArguments(flatArgs: Record<string, unknown>): Record<string, unknown> {
|
||||
const out: Record<string, unknown> = {};
|
||||
for (const [key, value] of Object.entries(flatArgs)) {
|
||||
setByPath(out, key.split("."), value);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function walk(
|
||||
schema: JSONSchema,
|
||||
depth: number,
|
||||
visit: (depth: number, isLeaf: boolean) => void,
|
||||
): void {
|
||||
if (schema.type === "object" && schema.properties) {
|
||||
for (const child of Object.values(schema.properties)) {
|
||||
walk(child, depth + 1, visit);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (schema.type === "array" && schema.items) {
|
||||
walk(schema.items, depth + 1, visit);
|
||||
return;
|
||||
}
|
||||
visit(depth, true);
|
||||
}
|
||||
|
||||
function collect(
|
||||
prefix: string,
|
||||
schema: JSONSchema,
|
||||
out: Record<string, JSONSchema>,
|
||||
required: string[],
|
||||
isRootRequired: boolean,
|
||||
): void {
|
||||
if (schema.type === "object" && schema.properties) {
|
||||
const requiredSet = new Set(schema.required ?? []);
|
||||
for (const [key, child] of Object.entries(schema.properties)) {
|
||||
const nextPrefix = prefix ? `${prefix}.${key}` : key;
|
||||
const childRequired = isRootRequired && requiredSet.has(key);
|
||||
collect(nextPrefix, child, out, required, childRequired);
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Treat anything non-object (including arrays) as a leaf for flattening purposes.
|
||||
out[prefix] = schema;
|
||||
if (isRootRequired) required.push(prefix);
|
||||
}
|
||||
|
||||
function setByPath(target: Record<string, unknown>, path: string[], value: unknown): void {
|
||||
let cur: any = target;
|
||||
for (let i = 0; i < path.length - 1; i++) {
|
||||
const key = path[i]!;
|
||||
if (typeof cur[key] !== "object" || cur[key] === null) cur[key] = {};
|
||||
cur = cur[key];
|
||||
}
|
||||
cur[path[path.length - 1]!] = value;
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
/**
|
||||
* Pillar 3 — Tool-Call Repair pipeline.
|
||||
*
|
||||
* Order of passes per turn:
|
||||
* 1. scavenge — recover tool calls leaked into <think>
|
||||
* 2. truncation — close any half-emitted argument JSON
|
||||
* 3. storm breaker — drop call-storm repeats
|
||||
*
|
||||
* Schema flattening is applied during loop construction (it changes what we
|
||||
* advertise to the model), not per-turn.
|
||||
*/
|
||||
|
||||
import type { ToolCall } from "../types.js";
|
||||
import { scavengeToolCalls } from "./scavenge.js";
|
||||
import { StormBreaker } from "./storm.js";
|
||||
import { repairTruncatedJson } from "./truncation.js";
|
||||
|
||||
export { analyzeSchema, flattenSchema, nestArguments } from "./flatten.js";
|
||||
export type { FlattenDecision } from "./flatten.js";
|
||||
export { repairTruncatedJson } from "./truncation.js";
|
||||
export type { TruncationRepairResult } from "./truncation.js";
|
||||
export { scavengeToolCalls } from "./scavenge.js";
|
||||
export type { ScavengeOptions, ScavengeResult } from "./scavenge.js";
|
||||
export { StormBreaker } from "./storm.js";
|
||||
|
||||
export interface RepairReport {
|
||||
scavenged: number;
|
||||
truncationsFixed: number;
|
||||
stormsBroken: number;
|
||||
notes: string[];
|
||||
}
|
||||
|
||||
export interface ToolCallRepairOptions {
|
||||
allowedToolNames: ReadonlySet<string>;
|
||||
stormWindow?: number;
|
||||
stormThreshold?: number;
|
||||
maxScavenge?: number;
|
||||
}
|
||||
|
||||
export class ToolCallRepair {
|
||||
private readonly storm: StormBreaker;
|
||||
private readonly opts: ToolCallRepairOptions;
|
||||
|
||||
constructor(opts: ToolCallRepairOptions) {
|
||||
this.opts = opts;
|
||||
this.storm = new StormBreaker(opts.stormWindow ?? 6, opts.stormThreshold ?? 3);
|
||||
}
|
||||
|
||||
process(
|
||||
declaredCalls: ToolCall[],
|
||||
reasoningContent: string | null,
|
||||
): { calls: ToolCall[]; report: RepairReport } {
|
||||
const report: RepairReport = {
|
||||
scavenged: 0,
|
||||
truncationsFixed: 0,
|
||||
stormsBroken: 0,
|
||||
notes: [],
|
||||
};
|
||||
|
||||
// 1. Scavenge — only add calls whose (name,args) signature is novel.
|
||||
const scavenged = scavengeToolCalls(reasoningContent, {
|
||||
allowedNames: this.opts.allowedToolNames,
|
||||
maxCalls: this.opts.maxScavenge ?? 4,
|
||||
});
|
||||
const seenSignatures = new Set(declaredCalls.map(signature));
|
||||
const merged = [...declaredCalls];
|
||||
for (const sc of scavenged.calls) {
|
||||
if (!seenSignatures.has(signature(sc))) {
|
||||
merged.push(sc);
|
||||
report.scavenged++;
|
||||
seenSignatures.add(signature(sc));
|
||||
}
|
||||
}
|
||||
report.notes.push(...scavenged.notes);
|
||||
|
||||
// 2. Truncation repair on argument JSON.
|
||||
for (const call of merged) {
|
||||
const args = call.function?.arguments ?? "";
|
||||
const r = repairTruncatedJson(args);
|
||||
if (r.changed) {
|
||||
call.function.arguments = r.repaired;
|
||||
report.truncationsFixed++;
|
||||
report.notes.push(...r.notes.map((n) => `[${call.function.name}] ${n}`));
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Storm breaker.
|
||||
const filtered: ToolCall[] = [];
|
||||
for (const call of merged) {
|
||||
const verdict = this.storm.inspect(call);
|
||||
if (verdict.suppress) {
|
||||
report.stormsBroken++;
|
||||
if (verdict.reason) report.notes.push(verdict.reason);
|
||||
continue;
|
||||
}
|
||||
filtered.push(call);
|
||||
}
|
||||
|
||||
return { calls: filtered, report };
|
||||
}
|
||||
}
|
||||
|
||||
function signature(call: ToolCall): string {
|
||||
return `${call.function?.name ?? ""}::${call.function?.arguments ?? ""}`;
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
/**
|
||||
* Scavenge tool calls leaked into reasoning_content.
|
||||
*
|
||||
* R1 sometimes emits tool-call JSON inside <think>…</think> and then forgets
|
||||
* to surface it in `tool_calls`. This pass extracts plausible calls and
|
||||
* proposes them to the loop, which decides whether to merge them with the
|
||||
* declared calls.
|
||||
*/
|
||||
|
||||
import type { ToolCall } from "../types.js";
|
||||
|
||||
export interface ScavengeOptions {
|
||||
/** Names of tools the model may legitimately call. Other names are ignored. */
|
||||
allowedNames: ReadonlySet<string>;
|
||||
/** Maximum number of calls to scavenge per pass (defence against runaway). */
|
||||
maxCalls?: number;
|
||||
}
|
||||
|
||||
export interface ScavengeResult {
|
||||
calls: ToolCall[];
|
||||
notes: string[];
|
||||
}
|
||||
|
||||
export function scavengeToolCalls(
|
||||
reasoningContent: string | null | undefined,
|
||||
opts: ScavengeOptions,
|
||||
): ScavengeResult {
|
||||
if (!reasoningContent) return { calls: [], notes: [] };
|
||||
const max = opts.maxCalls ?? 4;
|
||||
const notes: string[] = [];
|
||||
const out: ToolCall[] = [];
|
||||
|
||||
for (const candidate of iterateJsonObjects(reasoningContent)) {
|
||||
if (out.length >= max) break;
|
||||
const call = coerceToToolCall(candidate, opts.allowedNames);
|
||||
if (call) {
|
||||
out.push(call);
|
||||
notes.push(`scavenged call: ${call.function.name}`);
|
||||
}
|
||||
}
|
||||
return { calls: out, notes };
|
||||
}
|
||||
|
||||
/** Yield every top-level JSON object substring in `text`. */
|
||||
function* iterateJsonObjects(text: string): Generator<string> {
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
if (text[i] !== "{") continue;
|
||||
let depth = 0;
|
||||
let inString = false;
|
||||
let escaped = false;
|
||||
for (let j = i; j < text.length; j++) {
|
||||
const c = text[j]!;
|
||||
if (escaped) {
|
||||
escaped = false;
|
||||
continue;
|
||||
}
|
||||
if (inString) {
|
||||
if (c === "\\") {
|
||||
escaped = true;
|
||||
continue;
|
||||
}
|
||||
if (c === '"') inString = false;
|
||||
continue;
|
||||
}
|
||||
if (c === '"') inString = true;
|
||||
else if (c === "{") depth++;
|
||||
else if (c === "}") {
|
||||
depth--;
|
||||
if (depth === 0) {
|
||||
yield text.slice(i, j + 1);
|
||||
i = j;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function coerceToToolCall(
|
||||
candidateJson: string,
|
||||
allowedNames: ReadonlySet<string>,
|
||||
): ToolCall | null {
|
||||
let parsed: any;
|
||||
try {
|
||||
parsed = JSON.parse(candidateJson);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (!parsed || typeof parsed !== "object") return null;
|
||||
|
||||
// Pattern 1: { name, arguments }
|
||||
if (typeof parsed.name === "string" && allowedNames.has(parsed.name)) {
|
||||
const args = parsed.arguments;
|
||||
return {
|
||||
function: {
|
||||
name: parsed.name,
|
||||
arguments: typeof args === "string" ? args : JSON.stringify(args ?? {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Pattern 2: OpenAI-style { type: "function", function: { name, arguments } }
|
||||
if (
|
||||
parsed.type === "function" &&
|
||||
parsed.function &&
|
||||
typeof parsed.function.name === "string" &&
|
||||
allowedNames.has(parsed.function.name)
|
||||
) {
|
||||
const args = parsed.function.arguments;
|
||||
return {
|
||||
type: "function",
|
||||
function: {
|
||||
name: parsed.function.name,
|
||||
arguments: typeof args === "string" ? args : JSON.stringify(args ?? {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Pattern 3: { tool_name, tool_args } (R1 free-form variant)
|
||||
if (typeof parsed.tool_name === "string" && allowedNames.has(parsed.tool_name)) {
|
||||
return {
|
||||
function: {
|
||||
name: parsed.tool_name,
|
||||
arguments: JSON.stringify(parsed.tool_args ?? {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
import type { ToolCall } from "../types.js";
|
||||
|
||||
/**
|
||||
* Call-storm breaker.
|
||||
*
|
||||
* Detects (tool, args) tuples repeating within a sliding window and suppresses
|
||||
* the offending call. Surfaces a synthetic tool_result advising the model to
|
||||
* change strategy on its next turn.
|
||||
*/
|
||||
export class StormBreaker {
|
||||
private readonly windowSize: number;
|
||||
private readonly threshold: number;
|
||||
private readonly recent: Array<readonly [string, string]> = [];
|
||||
|
||||
constructor(windowSize = 6, threshold = 3) {
|
||||
this.windowSize = windowSize;
|
||||
this.threshold = threshold;
|
||||
}
|
||||
|
||||
inspect(call: ToolCall): { suppress: boolean; reason?: string } {
|
||||
const sig = signature(call);
|
||||
if (!sig) return { suppress: false };
|
||||
const count = this.recent.reduce(
|
||||
(n, [name, args]) => (name === sig[0] && args === sig[1] ? n + 1 : n),
|
||||
0,
|
||||
);
|
||||
if (count >= this.threshold - 1) {
|
||||
return {
|
||||
suppress: true,
|
||||
reason: `call-storm suppressed: ${sig[0]} called with identical args ${count + 1} times within window=${this.windowSize}`,
|
||||
};
|
||||
}
|
||||
this.recent.push(sig);
|
||||
while (this.recent.length > this.windowSize) this.recent.shift();
|
||||
return { suppress: false };
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
this.recent.length = 0;
|
||||
}
|
||||
}
|
||||
|
||||
function signature(call: ToolCall): readonly [string, string] | null {
|
||||
const name = call.function?.name;
|
||||
if (!name) return null;
|
||||
return [name, call.function?.arguments ?? ""] as const;
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
/**
|
||||
* Truncation recovery for tool-call argument JSON cut off mid-structure
|
||||
* (typically when the model hits max_tokens before finishing the JSON object).
|
||||
*
|
||||
* Strategy is purely local: balance braces, close strings, fill missing values
|
||||
* with `null`. We deliberately do NOT make a continuation API call here — that
|
||||
* decision belongs to the loop, which knows about budgets.
|
||||
*/
|
||||
|
||||
export interface TruncationRepairResult {
|
||||
repaired: string;
|
||||
changed: boolean;
|
||||
notes: string[];
|
||||
}
|
||||
|
||||
export function repairTruncatedJson(input: string): TruncationRepairResult {
|
||||
const notes: string[] = [];
|
||||
if (!input || !input.trim()) {
|
||||
return { repaired: "{}", changed: input !== "{}", notes: ["empty input → {}"] };
|
||||
}
|
||||
// Fast path: already parseable.
|
||||
try {
|
||||
JSON.parse(input);
|
||||
return { repaired: input, changed: false, notes: [] };
|
||||
} catch {
|
||||
/* fall through */
|
||||
}
|
||||
|
||||
const stack: ("{" | "[" | '"')[] = [];
|
||||
let escaped = false;
|
||||
let inString = false;
|
||||
let lastSignificant = -1;
|
||||
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
const c = input[i]!;
|
||||
if (!/\s/.test(c)) lastSignificant = i;
|
||||
if (escaped) {
|
||||
escaped = false;
|
||||
continue;
|
||||
}
|
||||
if (inString) {
|
||||
if (c === "\\") {
|
||||
escaped = true;
|
||||
continue;
|
||||
}
|
||||
if (c === '"') {
|
||||
inString = false;
|
||||
stack.pop();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (c === '"') {
|
||||
inString = true;
|
||||
stack.push('"');
|
||||
continue;
|
||||
}
|
||||
if (c === "{" || c === "[") stack.push(c);
|
||||
else if (c === "}" || c === "]") stack.pop();
|
||||
}
|
||||
|
||||
let s = input.slice(0, lastSignificant + 1);
|
||||
|
||||
// Trim a trailing comma which would block re-parse.
|
||||
if (/,$/.test(s)) {
|
||||
s = s.replace(/,$/, "");
|
||||
notes.push("trimmed trailing comma");
|
||||
}
|
||||
|
||||
// If we ended on a key without a value: "foo": → "foo": null
|
||||
if (/"\s*:\s*$/.test(s)) {
|
||||
s += " null";
|
||||
notes.push("filled dangling key with null");
|
||||
}
|
||||
|
||||
// If we ended inside a string, close it.
|
||||
if (inString) {
|
||||
s += '"';
|
||||
stack.pop();
|
||||
notes.push("closed unterminated string");
|
||||
}
|
||||
|
||||
// Pop remaining open structures in reverse order.
|
||||
while (stack.length > 0) {
|
||||
const top = stack.pop();
|
||||
if (top === "{") s += "}";
|
||||
else if (top === "[") s += "]";
|
||||
else if (top === '"') s += '"';
|
||||
}
|
||||
|
||||
try {
|
||||
JSON.parse(s);
|
||||
return { repaired: s, changed: true, notes };
|
||||
} catch (err) {
|
||||
notes.push(`fallback to {}: ${(err as Error).message}`);
|
||||
return { repaired: "{}", changed: true, notes };
|
||||
}
|
||||
}
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
/**
|
||||
* Replay — reconstruct session economics from a transcript file.
|
||||
*
|
||||
* Given a transcript written by App.tsx or the bench runner, rebuild a
|
||||
* SessionSummary-compatible aggregate (turn count, total cost, cache-hit
|
||||
* ratio, vs-Claude estimate) without replaying the LLM calls.
|
||||
*
|
||||
* The whole point is offline auditing: a reader should be able to reproduce
|
||||
* the headline numbers from a transcript alone, without an API key.
|
||||
*/
|
||||
|
||||
import { Usage } from "./client.js";
|
||||
import { type SessionSummary, type TurnStats, claudeEquivalentCost, costUsd } from "./telemetry.js";
|
||||
import { type ReadTranscriptResult, type TranscriptRecord, readTranscript } from "./transcript.js";
|
||||
|
||||
/**
|
||||
* A single turn's worth of records — the unit of navigation in replay TUI.
|
||||
* Records are grouped by their `turn` field, preserving file order within
|
||||
* each group (so tool events interleave with assistant_final events the
|
||||
* way they were actually emitted).
|
||||
*/
|
||||
export interface TurnPage {
|
||||
turn: number;
|
||||
records: TranscriptRecord[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Group transcript records into turn-pages. Pages are returned in ascending
|
||||
* turn order. Records without a numeric turn (meta lines, malformed) are
|
||||
* already filtered by the transcript reader, so this sees clean input.
|
||||
*/
|
||||
export function groupRecordsByTurn(records: TranscriptRecord[]): TurnPage[] {
|
||||
const byTurn = new Map<number, TranscriptRecord[]>();
|
||||
for (const rec of records) {
|
||||
const list = byTurn.get(rec.turn);
|
||||
if (list) list.push(rec);
|
||||
else byTurn.set(rec.turn, [rec]);
|
||||
}
|
||||
return [...byTurn.entries()]
|
||||
.sort(([a], [b]) => a - b)
|
||||
.map(([turn, records]) => ({ turn, records }));
|
||||
}
|
||||
|
||||
/**
|
||||
* Cumulative replay stats up to and including pages[0..upToIdx]. Returns
|
||||
* empty stats if upToIdx < 0. Used by replay TUI's sidebar to show "stats
|
||||
* so far" as the user scrolls through a transcript.
|
||||
*/
|
||||
export function computeCumulativeStats(pages: TurnPage[], upToIdx: number): ReplayStats {
|
||||
if (upToIdx < 0) return computeReplayStats([]);
|
||||
const flat: TranscriptRecord[] = [];
|
||||
for (let i = 0; i <= upToIdx && i < pages.length; i++) {
|
||||
const records = pages[i]?.records;
|
||||
if (records) flat.push(...records);
|
||||
}
|
||||
return computeReplayStats(flat);
|
||||
}
|
||||
|
||||
export interface ReplayStats extends SessionSummary {
|
||||
/** Per-turn stats, in turn order. Only assistant_final records contribute. */
|
||||
perTurn: TurnStats[];
|
||||
/** Unique models that appeared in the transcript's assistant_final records. */
|
||||
models: string[];
|
||||
/** Unique prefix hashes that appeared. Length > 1 means the prefix churned (cache-hostile). */
|
||||
prefixHashes: string[];
|
||||
/** Count of user-role records (user turns issued). */
|
||||
userTurns: number;
|
||||
/** Count of tool-role records (tool calls executed). */
|
||||
toolCalls: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a transcript file and compute replay stats. Throws only on I/O
|
||||
* errors; malformed lines inside the file are skipped silently.
|
||||
*/
|
||||
export function replayFromFile(path: string): { parsed: ReadTranscriptResult; stats: ReplayStats } {
|
||||
const parsed = readTranscript(path);
|
||||
return { parsed, stats: computeReplayStats(parsed.records) };
|
||||
}
|
||||
|
||||
export function computeReplayStats(records: TranscriptRecord[]): ReplayStats {
|
||||
const turns: TurnStats[] = [];
|
||||
const models = new Set<string>();
|
||||
const prefixHashes = new Set<string>();
|
||||
let userTurns = 0;
|
||||
let toolCalls = 0;
|
||||
|
||||
for (const rec of records) {
|
||||
if (rec.role === "user") userTurns++;
|
||||
else if (rec.role === "tool") toolCalls++;
|
||||
else if (rec.role === "assistant_final") {
|
||||
if (rec.model) models.add(rec.model);
|
||||
if (rec.prefixHash) prefixHashes.add(rec.prefixHash);
|
||||
if (rec.usage && rec.model) {
|
||||
const u = new Usage(
|
||||
rec.usage.prompt_tokens ?? 0,
|
||||
rec.usage.completion_tokens ?? 0,
|
||||
rec.usage.total_tokens ?? 0,
|
||||
rec.usage.prompt_cache_hit_tokens ?? 0,
|
||||
rec.usage.prompt_cache_miss_tokens ?? 0,
|
||||
);
|
||||
turns.push({
|
||||
turn: rec.turn,
|
||||
model: rec.model,
|
||||
usage: u,
|
||||
// `rec.cost` wins when present — honors whatever the writer computed
|
||||
// even if pricing tables have since changed. Only recompute when
|
||||
// the transcript didn't record it (old format).
|
||||
cost: rec.cost ?? costUsd(rec.model, u),
|
||||
cacheHitRatio: u.cacheHitRatio,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
perTurn: turns,
|
||||
models: [...models],
|
||||
prefixHashes: [...prefixHashes],
|
||||
userTurns,
|
||||
toolCalls,
|
||||
...summarizeTurns(turns),
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeTurns(turns: TurnStats[]): SessionSummary {
|
||||
const totalCost = turns.reduce((s, t) => s + t.cost, 0);
|
||||
const totalClaude = turns.reduce((s, t) => s + claudeEquivalentCost(t.usage), 0);
|
||||
let hit = 0;
|
||||
let miss = 0;
|
||||
for (const t of turns) {
|
||||
hit += t.usage.promptCacheHitTokens;
|
||||
miss += t.usage.promptCacheMissTokens;
|
||||
}
|
||||
const cacheHitRatio = hit + miss > 0 ? hit / (hit + miss) : 0;
|
||||
const savingsVsClaude = totalClaude > 0 ? 1 - totalCost / totalClaude : 0;
|
||||
return {
|
||||
turns: turns.length,
|
||||
totalCostUsd: round(totalCost, 6),
|
||||
claudeEquivalentUsd: round(totalClaude, 6),
|
||||
savingsVsClaudePct: round(savingsVsClaude * 100, 2),
|
||||
cacheHitRatio: round(cacheHitRatio, 4),
|
||||
};
|
||||
}
|
||||
|
||||
function round(n: number, digits: number): number {
|
||||
const f = 10 ** digits;
|
||||
return Math.round(n * f) / f;
|
||||
}
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
/**
|
||||
* Retry layer for DeepSeek API calls.
|
||||
*
|
||||
* Wraps a `fetch` function so that transient failures (rate limiting, server
|
||||
* overload, network blips) don't kill an agent session. We explicitly DO NOT
|
||||
* retry:
|
||||
* - 4xx client errors other than 408 / 429 (bad key, bad request, ...)
|
||||
* - aborted requests (user cancelled)
|
||||
* - mid-stream body read errors (retrying costs money AND would desync)
|
||||
*
|
||||
* Retrying is controlled by attempt count + exponential backoff with jitter.
|
||||
* If the server sends a `Retry-After` header we honor it (capped by
|
||||
* `maxBackoffMs` so a misconfigured upstream can't park us forever).
|
||||
*/
|
||||
|
||||
export interface RetryOptions {
|
||||
/** Maximum total attempts (including the first). Default 4. */
|
||||
maxAttempts?: number;
|
||||
/** Initial backoff in ms. Doubles each retry, with jitter. Default 500. */
|
||||
initialBackoffMs?: number;
|
||||
/** Upper bound on any single backoff delay. Default 10000 (10s). */
|
||||
maxBackoffMs?: number;
|
||||
/** HTTP statuses to treat as retryable. Default [408, 429, 500, 502, 503, 504]. */
|
||||
retryableStatuses?: readonly number[];
|
||||
/** Abort signal; we do NOT retry once aborted. */
|
||||
signal?: AbortSignal;
|
||||
/** Telemetry hook — called before each wait. */
|
||||
onRetry?: (info: RetryInfo) => void;
|
||||
}
|
||||
|
||||
export interface RetryInfo {
|
||||
attempt: number;
|
||||
reason: string;
|
||||
waitMs: number;
|
||||
}
|
||||
|
||||
const DEFAULT_RETRYABLE_STATUSES = [408, 429, 500, 502, 503, 504] as const;
|
||||
|
||||
export async function fetchWithRetry(
|
||||
fetchFn: typeof fetch,
|
||||
url: string,
|
||||
init: RequestInit,
|
||||
opts: RetryOptions = {},
|
||||
): Promise<Response> {
|
||||
const maxAttempts = opts.maxAttempts ?? 4;
|
||||
const initial = opts.initialBackoffMs ?? 500;
|
||||
const cap = opts.maxBackoffMs ?? 10_000;
|
||||
const retryable = new Set(opts.retryableStatuses ?? DEFAULT_RETRYABLE_STATUSES);
|
||||
|
||||
let lastError: unknown;
|
||||
|
||||
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
||||
if (opts.signal?.aborted) throw new Error("aborted");
|
||||
|
||||
try {
|
||||
const resp = await fetchFn(url, init);
|
||||
|
||||
// Success or non-retryable failure: return as-is.
|
||||
if (resp.ok || !retryable.has(resp.status)) return resp;
|
||||
|
||||
// Retryable but out of attempts: return the last response so the caller
|
||||
// can surface the status to the user.
|
||||
if (attempt === maxAttempts - 1) return resp;
|
||||
|
||||
// Drain the body so the connection can be reused on the next attempt.
|
||||
await resp.text().catch(() => undefined);
|
||||
|
||||
const waitMs = computeWait(attempt, initial, cap, resp.headers.get("Retry-After"));
|
||||
opts.onRetry?.({ attempt: attempt + 1, reason: `http ${resp.status}`, waitMs });
|
||||
await sleep(waitMs, opts.signal);
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
// Respect explicit aborts — do not retry.
|
||||
if (isAbortError(err) || opts.signal?.aborted) throw err;
|
||||
if (attempt === maxAttempts - 1) throw err;
|
||||
|
||||
const waitMs = computeWait(attempt, initial, cap, null);
|
||||
opts.onRetry?.({
|
||||
attempt: attempt + 1,
|
||||
reason: `network: ${messageOf(err)}`,
|
||||
waitMs,
|
||||
});
|
||||
await sleep(waitMs, opts.signal);
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError ?? new Error("fetchWithRetry: loop exited unexpectedly");
|
||||
}
|
||||
|
||||
function computeWait(
|
||||
attempt: number,
|
||||
initial: number,
|
||||
cap: number,
|
||||
retryAfter: string | null,
|
||||
): number {
|
||||
if (retryAfter) {
|
||||
const seconds = Number.parseFloat(retryAfter);
|
||||
if (Number.isFinite(seconds) && seconds > 0) {
|
||||
return Math.min(seconds * 1000, cap);
|
||||
}
|
||||
}
|
||||
const exp = initial * 2 ** attempt;
|
||||
// Jitter range [75%, 125%] to spread retries out when many clients hit 429 together.
|
||||
const jitter = exp * (0.75 + Math.random() * 0.5);
|
||||
return Math.min(Math.max(jitter, 0), cap);
|
||||
}
|
||||
|
||||
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
if (ms <= 0) return Promise.resolve();
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(resolve, ms);
|
||||
if (signal) {
|
||||
const onAbort = () => {
|
||||
clearTimeout(timer);
|
||||
reject(new Error("aborted"));
|
||||
};
|
||||
if (signal.aborted) onAbort();
|
||||
else signal.addEventListener("abort", onAbort, { once: true });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function isAbortError(err: unknown): boolean {
|
||||
if (!err || typeof err !== "object") return false;
|
||||
const name = (err as { name?: unknown }).name;
|
||||
return name === "AbortError";
|
||||
}
|
||||
|
||||
function messageOf(err: unknown): string {
|
||||
if (err instanceof Error) return err.message;
|
||||
try {
|
||||
return String(err);
|
||||
} catch {
|
||||
return "unknown error";
|
||||
}
|
||||
}
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
/**
|
||||
* Session persistence.
|
||||
*
|
||||
* Every turn's log entries (user / assistant / tool messages) are appended to
|
||||
* a JSONL file under `~/.reasonix/sessions/<name>.jsonl`. Next time the user
|
||||
* starts the CLI with the same session name, the loop pre-loads the file
|
||||
* into its AppendOnlyLog so the new turn has full prior context.
|
||||
*
|
||||
* Design notes:
|
||||
* - JSONL rather than JSON so concurrent writes don't corrupt.
|
||||
* - 0600 permissions on Unix (chmod no-ops on Windows).
|
||||
* - Name sanitization keeps paths safe: only [\w-] and CJK letters pass;
|
||||
* anything else is replaced with underscore, max 64 chars.
|
||||
* - The loop's stats/session aren't persisted — only the message log.
|
||||
* Cost accounting resets each run (by design — old costs are sunk).
|
||||
*/
|
||||
|
||||
import {
|
||||
appendFileSync,
|
||||
chmodSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
statSync,
|
||||
unlinkSync,
|
||||
} from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import type { ChatMessage } from "./types.js";
|
||||
|
||||
export interface SessionInfo {
|
||||
name: string;
|
||||
path: string;
|
||||
size: number;
|
||||
messageCount: number;
|
||||
mtime: Date;
|
||||
}
|
||||
|
||||
export function sessionsDir(): string {
|
||||
return join(homedir(), ".reasonix", "sessions");
|
||||
}
|
||||
|
||||
export function sessionPath(name: string): string {
|
||||
return join(sessionsDir(), `${sanitizeName(name)}.jsonl`);
|
||||
}
|
||||
|
||||
export function sanitizeName(name: string): string {
|
||||
const cleaned = name.replace(/[^\w\-\u4e00-\u9fa5]/g, "_").slice(0, 64);
|
||||
return cleaned || "default";
|
||||
}
|
||||
|
||||
export function loadSessionMessages(name: string): ChatMessage[] {
|
||||
const path = sessionPath(name);
|
||||
if (!existsSync(path)) return [];
|
||||
try {
|
||||
const raw = readFileSync(path, "utf8");
|
||||
const out: ChatMessage[] = [];
|
||||
for (const line of raw.split(/\r?\n/)) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
try {
|
||||
const msg = JSON.parse(trimmed) as ChatMessage;
|
||||
if (msg && typeof msg === "object" && "role" in msg) out.push(msg);
|
||||
} catch {
|
||||
/* skip malformed line */
|
||||
}
|
||||
}
|
||||
return out;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export function appendSessionMessage(name: string, message: ChatMessage): void {
|
||||
const path = sessionPath(name);
|
||||
mkdirSync(dirname(path), { recursive: true });
|
||||
appendFileSync(path, `${JSON.stringify(message)}\n`, "utf8");
|
||||
try {
|
||||
chmodSync(path, 0o600);
|
||||
} catch {
|
||||
/* chmod not supported on this platform */
|
||||
}
|
||||
}
|
||||
|
||||
export function listSessions(): SessionInfo[] {
|
||||
const dir = sessionsDir();
|
||||
if (!existsSync(dir)) return [];
|
||||
try {
|
||||
const files = readdirSync(dir).filter((f) => f.endsWith(".jsonl"));
|
||||
return files
|
||||
.map((file) => {
|
||||
const path = join(dir, file);
|
||||
const stat = statSync(path);
|
||||
const name = file.replace(/\.jsonl$/, "");
|
||||
const messageCount = countLines(path);
|
||||
return { name, path, size: stat.size, messageCount, mtime: stat.mtime };
|
||||
})
|
||||
.sort((a, b) => b.mtime.getTime() - a.mtime.getTime());
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export function deleteSession(name: string): boolean {
|
||||
const path = sessionPath(name);
|
||||
try {
|
||||
unlinkSync(path);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function countLines(path: string): number {
|
||||
try {
|
||||
const raw = readFileSync(path, "utf8");
|
||||
return raw.split(/\r?\n/).filter((l) => l.trim()).length;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
import type { Usage } from "./client.js";
|
||||
|
||||
/** USD per 1M tokens. Update as DeepSeek pricing changes. */
|
||||
export const DEEPSEEK_PRICING: Record<
|
||||
string,
|
||||
{ inputCacheHit: number; inputCacheMiss: number; output: number }
|
||||
> = {
|
||||
"deepseek-chat": { inputCacheHit: 0.07, inputCacheMiss: 0.27, output: 1.1 },
|
||||
"deepseek-reasoner": { inputCacheHit: 0.14, inputCacheMiss: 0.55, output: 2.19 },
|
||||
};
|
||||
|
||||
/** Reference Claude Sonnet 4.6 pricing (USD per 1M tokens). */
|
||||
export const CLAUDE_SONNET_PRICING = { input: 3.0, output: 15.0 };
|
||||
|
||||
export function costUsd(model: string, usage: Usage): number {
|
||||
const p = DEEPSEEK_PRICING[model];
|
||||
if (!p) return 0;
|
||||
return (
|
||||
(usage.promptCacheHitTokens * p.inputCacheHit +
|
||||
usage.promptCacheMissTokens * p.inputCacheMiss +
|
||||
usage.completionTokens * p.output) /
|
||||
1_000_000
|
||||
);
|
||||
}
|
||||
|
||||
export function claudeEquivalentCost(usage: Usage): number {
|
||||
return (
|
||||
(usage.promptTokens * CLAUDE_SONNET_PRICING.input +
|
||||
usage.completionTokens * CLAUDE_SONNET_PRICING.output) /
|
||||
1_000_000
|
||||
);
|
||||
}
|
||||
|
||||
export interface TurnStats {
|
||||
turn: number;
|
||||
model: string;
|
||||
usage: Usage;
|
||||
cost: number;
|
||||
cacheHitRatio: number;
|
||||
}
|
||||
|
||||
export interface SessionSummary {
|
||||
turns: number;
|
||||
totalCostUsd: number;
|
||||
claudeEquivalentUsd: number;
|
||||
savingsVsClaudePct: number;
|
||||
cacheHitRatio: number;
|
||||
}
|
||||
|
||||
export class SessionStats {
|
||||
readonly turns: TurnStats[] = [];
|
||||
|
||||
record(turn: number, model: string, usage: Usage): TurnStats {
|
||||
const cost = costUsd(model, usage);
|
||||
const stats: TurnStats = {
|
||||
turn,
|
||||
model,
|
||||
usage,
|
||||
cost,
|
||||
cacheHitRatio: usage.cacheHitRatio,
|
||||
};
|
||||
this.turns.push(stats);
|
||||
return stats;
|
||||
}
|
||||
|
||||
get totalCost(): number {
|
||||
return this.turns.reduce((sum, t) => sum + t.cost, 0);
|
||||
}
|
||||
|
||||
get totalClaudeEquivalent(): number {
|
||||
return this.turns.reduce((sum, t) => sum + claudeEquivalentCost(t.usage), 0);
|
||||
}
|
||||
|
||||
get savingsVsClaude(): number {
|
||||
const c = this.totalClaudeEquivalent;
|
||||
return c > 0 ? 1 - this.totalCost / c : 0;
|
||||
}
|
||||
|
||||
get aggregateCacheHitRatio(): number {
|
||||
let hit = 0;
|
||||
let miss = 0;
|
||||
for (const t of this.turns) {
|
||||
hit += t.usage.promptCacheHitTokens;
|
||||
miss += t.usage.promptCacheMissTokens;
|
||||
}
|
||||
const denom = hit + miss;
|
||||
return denom > 0 ? hit / denom : 0;
|
||||
}
|
||||
|
||||
summary(): SessionSummary {
|
||||
return {
|
||||
turns: this.turns.length,
|
||||
totalCostUsd: round(this.totalCost, 6),
|
||||
claudeEquivalentUsd: round(this.totalClaudeEquivalent, 6),
|
||||
savingsVsClaudePct: round(this.savingsVsClaude * 100, 2),
|
||||
cacheHitRatio: round(this.aggregateCacheHitRatio, 4),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function round(n: number, digits: number): number {
|
||||
const f = 10 ** digits;
|
||||
return Math.round(n * f) / f;
|
||||
}
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
import { analyzeSchema, flattenSchema, nestArguments } from "./repair/flatten.js";
|
||||
import type { JSONSchema, ToolSpec } from "./types.js";
|
||||
|
||||
export interface ToolDefinition<A = any, R = any> {
|
||||
name: string;
|
||||
description?: string;
|
||||
parameters?: JSONSchema;
|
||||
fn: (args: A) => R | Promise<R>;
|
||||
}
|
||||
|
||||
interface InternalTool extends ToolDefinition {
|
||||
/**
|
||||
* Pillar 3 — flatten metadata. Set when the registered schema is deep
|
||||
* (>2 levels) or wide (>10 leaf params), conditions on which DeepSeek
|
||||
* V3/R1 are known to drop arguments. We advertise the flattened schema
|
||||
* to the model, then re-nest the model's args before calling fn.
|
||||
*/
|
||||
flatSchema?: JSONSchema;
|
||||
}
|
||||
|
||||
export interface ToolRegistryOptions {
|
||||
/**
|
||||
* Auto-flatten schemas that exceed depth/width thresholds before sending
|
||||
* them to the model. Re-nests arguments transparently on dispatch.
|
||||
* Default: true. Pass false to opt out.
|
||||
*/
|
||||
autoFlatten?: boolean;
|
||||
}
|
||||
|
||||
export class ToolRegistry {
|
||||
private readonly _tools = new Map<string, InternalTool>();
|
||||
private readonly _autoFlatten: boolean;
|
||||
|
||||
constructor(opts: ToolRegistryOptions = {}) {
|
||||
this._autoFlatten = opts.autoFlatten !== false;
|
||||
}
|
||||
|
||||
register<A, R>(def: ToolDefinition<A, R>): this {
|
||||
if (!def.name) throw new Error("tool requires a name");
|
||||
const internal: InternalTool = { ...(def as ToolDefinition) };
|
||||
if (this._autoFlatten && def.parameters) {
|
||||
const decision = analyzeSchema(def.parameters);
|
||||
if (decision.shouldFlatten) {
|
||||
internal.flatSchema = flattenSchema(def.parameters);
|
||||
}
|
||||
}
|
||||
this._tools.set(def.name, internal);
|
||||
return this;
|
||||
}
|
||||
|
||||
has(name: string): boolean {
|
||||
return this._tools.has(name);
|
||||
}
|
||||
|
||||
get(name: string): ToolDefinition | undefined {
|
||||
return this._tools.get(name);
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this._tools.size;
|
||||
}
|
||||
|
||||
/** True if a registered tool's schema was flattened for the model. */
|
||||
wasFlattened(name: string): boolean {
|
||||
return Boolean(this._tools.get(name)?.flatSchema);
|
||||
}
|
||||
|
||||
specs(): ToolSpec[] {
|
||||
return [...this._tools.values()].map((t) => ({
|
||||
type: "function",
|
||||
function: {
|
||||
name: t.name,
|
||||
description: t.description ?? "",
|
||||
parameters: t.flatSchema ?? t.parameters ?? { type: "object", properties: {} },
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
async dispatch(name: string, argumentsRaw: string | Record<string, unknown>): Promise<string> {
|
||||
const tool = this._tools.get(name);
|
||||
if (!tool) {
|
||||
return JSON.stringify({ error: `unknown tool: ${name}` });
|
||||
}
|
||||
let args: Record<string, unknown>;
|
||||
try {
|
||||
args =
|
||||
typeof argumentsRaw === "string"
|
||||
? argumentsRaw.trim()
|
||||
? (JSON.parse(argumentsRaw) ?? {})
|
||||
: {}
|
||||
: (argumentsRaw ?? {});
|
||||
} catch (err) {
|
||||
return JSON.stringify({
|
||||
error: `invalid tool arguments JSON: ${(err as Error).message}`,
|
||||
});
|
||||
}
|
||||
|
||||
// Re-nest dot-notation args back to the original shape, but only when
|
||||
// (a) we flattened this tool's schema, AND
|
||||
// (b) the incoming args actually use dot keys.
|
||||
// The second condition handles the case where a model ignores the flat
|
||||
// spec and emits nested args anyway — we shouldn't double-process them.
|
||||
if (tool.flatSchema && args && typeof args === "object" && hasDotKey(args)) {
|
||||
args = nestArguments(args);
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await tool.fn(args);
|
||||
return typeof result === "string" ? result : JSON.stringify(result);
|
||||
} catch (err) {
|
||||
return JSON.stringify({
|
||||
error: `${(err as Error).name}: ${(err as Error).message}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function hasDotKey(obj: Record<string, unknown>): boolean {
|
||||
for (const k of Object.keys(obj)) {
|
||||
if (k.includes(".")) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
/**
|
||||
* Transcript format — the canonical "audit log" of a Reasonix session.
|
||||
*
|
||||
* Design split:
|
||||
* - Session file (`~/.reasonix/sessions/<name>.jsonl`) stores only the
|
||||
* `ChatMessage`s the model needs to resume. See session.ts.
|
||||
* - Transcript file (this module) stores every LoopEvent with usage, cost,
|
||||
* model, and prefix fingerprint attached where available — enough for
|
||||
* replay and diff to reconstruct economics.
|
||||
*
|
||||
* The two are different contracts: sessions are the user's *memory*;
|
||||
* transcripts are the *receipts*. Don't conflate them.
|
||||
*
|
||||
* Backward compatibility: all fields beyond {ts, turn, role, content} are
|
||||
* optional on read. A v0.1 transcript (pre-usage) still parses and renders
|
||||
* — it just shows cost/cache as n/a.
|
||||
*/
|
||||
|
||||
import { type WriteStream, createWriteStream, readFileSync } from "node:fs";
|
||||
import type { LoopEvent } from "./loop.js";
|
||||
import type { RawUsage } from "./types.js";
|
||||
|
||||
export interface TranscriptRecord {
|
||||
/** ISO-8601 timestamp at emit time. */
|
||||
ts: string;
|
||||
/** 1-based turn number within the session. */
|
||||
turn: number;
|
||||
/** LoopEvent role — "assistant_delta" | "assistant_final" | "tool" | "done" | ... */
|
||||
role: string;
|
||||
/** For assistant events, the final (or delta) text; for tool events, the tool result. */
|
||||
content: string;
|
||||
/** Tool name (role === "tool"). */
|
||||
tool?: string;
|
||||
/** JSON-string args the model sent for a tool call (role === "tool"). Persisted so diff can explain *why* two runs made different calls. */
|
||||
args?: string;
|
||||
/** DeepSeek token-usage snapshot (role === "assistant_final"). */
|
||||
usage?: RawUsage;
|
||||
/** USD cost of this turn (role === "assistant_final"). */
|
||||
cost?: number;
|
||||
/** Model id that produced this turn. */
|
||||
model?: string;
|
||||
/**
|
||||
* The ImmutablePrefix fingerprint at this turn. Lets diff prove two runs
|
||||
* share a prefix — i.e. any cache-hit delta is attributable to log
|
||||
* stability, not to a different system prompt.
|
||||
*/
|
||||
prefixHash?: string;
|
||||
/** Optional error message (role === "error"). */
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface TranscriptMeta {
|
||||
/**
|
||||
* Optional metadata written as the first line of a transcript. Lets
|
||||
* downstream tooling know what it's reading without guessing.
|
||||
* Recognized by a special role "_meta".
|
||||
*/
|
||||
version: 1;
|
||||
source: string; // e.g. "reasonix chat", "bench/baseline", "bench/reasonix"
|
||||
model?: string;
|
||||
task?: string;
|
||||
mode?: string;
|
||||
repeat?: number;
|
||||
startedAt: string;
|
||||
}
|
||||
|
||||
interface MetaLine {
|
||||
role: "_meta";
|
||||
meta: TranscriptMeta;
|
||||
}
|
||||
|
||||
export interface ReadTranscriptResult {
|
||||
meta: TranscriptMeta | null;
|
||||
records: TranscriptRecord[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a TranscriptRecord from a LoopEvent. Extra fields (model,
|
||||
* prefixHash) that the LoopEvent doesn't carry are passed in separately
|
||||
* because they're session-level, not event-level.
|
||||
*/
|
||||
export function recordFromLoopEvent(
|
||||
ev: LoopEvent,
|
||||
extra: { model: string; prefixHash: string },
|
||||
): TranscriptRecord {
|
||||
const rec: TranscriptRecord = {
|
||||
ts: new Date().toISOString(),
|
||||
turn: ev.turn,
|
||||
role: ev.role,
|
||||
content: ev.content,
|
||||
};
|
||||
if (ev.toolName !== undefined) rec.tool = ev.toolName;
|
||||
if (ev.toolArgs !== undefined) rec.args = ev.toolArgs;
|
||||
if (ev.error !== undefined) rec.error = ev.error;
|
||||
if (ev.stats) {
|
||||
rec.usage = {
|
||||
prompt_tokens: ev.stats.usage.promptTokens,
|
||||
completion_tokens: ev.stats.usage.completionTokens,
|
||||
total_tokens: ev.stats.usage.totalTokens,
|
||||
prompt_cache_hit_tokens: ev.stats.usage.promptCacheHitTokens,
|
||||
prompt_cache_miss_tokens: ev.stats.usage.promptCacheMissTokens,
|
||||
};
|
||||
rec.cost = ev.stats.cost;
|
||||
rec.model = ev.stats.model;
|
||||
rec.prefixHash = extra.prefixHash;
|
||||
} else if (ev.role === "assistant_final") {
|
||||
// assistant_final without stats (shouldn't happen in the live loop but
|
||||
// might in test fixtures) — still persist model + prefix for continuity.
|
||||
rec.model = extra.model;
|
||||
rec.prefixHash = extra.prefixHash;
|
||||
}
|
||||
return rec;
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a record to an open write stream. Caller owns the stream lifecycle.
|
||||
*/
|
||||
export function writeRecord(stream: WriteStream, record: TranscriptRecord): void {
|
||||
stream.write(`${JSON.stringify(record)}\n`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a _meta line to an open write stream. Call exactly once, at the top.
|
||||
*/
|
||||
export function writeMeta(stream: WriteStream, meta: TranscriptMeta): void {
|
||||
const line: MetaLine = { role: "_meta", meta };
|
||||
stream.write(`${JSON.stringify(line)}\n`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience: open a stream, write meta, return stream.
|
||||
*/
|
||||
export function openTranscriptFile(path: string, meta: TranscriptMeta): WriteStream {
|
||||
const stream = createWriteStream(path, { flags: "a" });
|
||||
writeMeta(stream, meta);
|
||||
return stream;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a transcript file. Returns meta (if the first line is a _meta record)
|
||||
* and the full record list.
|
||||
*
|
||||
* Robustness contract:
|
||||
* - Empty lines are skipped.
|
||||
* - Malformed JSON lines are skipped silently (do not crash on partial
|
||||
* files — live chats may be mid-write).
|
||||
* - Records missing optional fields still parse — they're just rendered
|
||||
* with n/a where the optional value would go.
|
||||
*/
|
||||
export function readTranscript(path: string): ReadTranscriptResult {
|
||||
const raw = readFileSync(path, "utf8");
|
||||
return parseTranscript(raw);
|
||||
}
|
||||
|
||||
export function parseTranscript(raw: string): ReadTranscriptResult {
|
||||
const out: ReadTranscriptResult = { meta: null, records: [] };
|
||||
for (const line of raw.split(/\r?\n/)) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
let obj: unknown;
|
||||
try {
|
||||
obj = JSON.parse(trimmed);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!obj || typeof obj !== "object") continue;
|
||||
const rec = obj as Record<string, unknown>;
|
||||
if (rec.role === "_meta" && rec.meta && typeof rec.meta === "object") {
|
||||
out.meta = rec.meta as TranscriptMeta;
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
typeof rec.ts === "string" &&
|
||||
typeof rec.turn === "number" &&
|
||||
typeof rec.role === "string" &&
|
||||
typeof rec.content === "string"
|
||||
) {
|
||||
out.records.push(rec as unknown as TranscriptRecord);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
export interface JSONSchema {
|
||||
type?: string;
|
||||
properties?: Record<string, JSONSchema>;
|
||||
items?: JSONSchema;
|
||||
required?: string[];
|
||||
description?: string;
|
||||
enum?: unknown[];
|
||||
[k: string]: unknown;
|
||||
}
|
||||
|
||||
export interface ToolFunctionSpec {
|
||||
name: string;
|
||||
description: string;
|
||||
parameters: JSONSchema;
|
||||
}
|
||||
|
||||
export interface ToolSpec {
|
||||
type: "function";
|
||||
function: ToolFunctionSpec;
|
||||
}
|
||||
|
||||
export interface ToolCall {
|
||||
id?: string;
|
||||
type?: "function";
|
||||
function: {
|
||||
name: string;
|
||||
arguments: string;
|
||||
};
|
||||
}
|
||||
|
||||
export type Role = "system" | "user" | "assistant" | "tool";
|
||||
|
||||
export interface ChatMessage {
|
||||
role: Role;
|
||||
content?: string | null;
|
||||
name?: string;
|
||||
tool_call_id?: string;
|
||||
tool_calls?: ToolCall[];
|
||||
}
|
||||
|
||||
export interface RawUsage {
|
||||
prompt_tokens?: number;
|
||||
completion_tokens?: number;
|
||||
total_tokens?: number;
|
||||
prompt_cache_hit_tokens?: number;
|
||||
prompt_cache_miss_tokens?: number;
|
||||
}
|
||||
|
||||
export interface ChatRequestOptions {
|
||||
model: string;
|
||||
messages: ChatMessage[];
|
||||
tools?: ToolSpec[];
|
||||
temperature?: number;
|
||||
maxTokens?: number;
|
||||
stream?: boolean;
|
||||
signal?: AbortSignal;
|
||||
/** DeepSeek response_format — use { type: "json_object" } to force valid JSON. */
|
||||
responseFormat?: { type: "json_object" | "text" };
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
/**
|
||||
* Smoke tests for the τ-bench-lite harness.
|
||||
*
|
||||
* The goal is NOT to verify benchmark numbers (those only exist when you
|
||||
* point the harness at a real API key). These tests verify the harness
|
||||
* itself is robust:
|
||||
* - each seed task's tools actually operate on *its* db snapshot, not a
|
||||
* shared one (i.e. cloneDb isolation works)
|
||||
* - each task's check() predicate is satisfiable when the correct mutation
|
||||
* happens, and refusal tasks stay false-on-no-change
|
||||
* - the baseline agent's shuffle is deterministic per turn-number so
|
||||
* reproducibility holds across runs
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { cloneDb } from "../benchmarks/tau-bench/db.js";
|
||||
import { TASKS } from "../benchmarks/tau-bench/tasks.js";
|
||||
import type { TaskDefinition, Turn } from "../benchmarks/tau-bench/types.js";
|
||||
|
||||
function buildToolsFor(task: TaskDefinition) {
|
||||
const db = cloneDb(task.initialDb);
|
||||
const tools = task.tools.map((f) => f(db));
|
||||
const byName = new Map(tools.map((t) => [t.name, t]));
|
||||
return { db, tools, byName };
|
||||
}
|
||||
|
||||
describe("tau-bench-lite task set", () => {
|
||||
it("exposes at least 8 tasks with unique ids", () => {
|
||||
expect(TASKS.length).toBeGreaterThanOrEqual(8);
|
||||
const ids = new Set(TASKS.map((t) => t.id));
|
||||
expect(ids.size).toBe(TASKS.length);
|
||||
});
|
||||
|
||||
it("every task's tool factories close over an isolated db", async () => {
|
||||
// Run the same tool mutation on two independent clones of one task's db
|
||||
// and assert the two dbs diverge.
|
||||
const task = TASKS.find((t) => t.id === "t01_address_happy");
|
||||
if (!task) throw new Error("t01 missing");
|
||||
|
||||
const a = buildToolsFor(task);
|
||||
const b = buildToolsFor(task);
|
||||
|
||||
const updateA = a.byName.get("update_address");
|
||||
if (!updateA) throw new Error("update_address tool missing");
|
||||
await updateA.fn({ orderId: "o_1002", address: "A ADDRESS" });
|
||||
|
||||
expect(a.db.orders.o_1002?.address).toBe("A ADDRESS");
|
||||
expect(b.db.orders.o_1002?.address).toBe("22 Oak Rd, NYC, NY 10001");
|
||||
});
|
||||
});
|
||||
|
||||
describe("task check() predicates", () => {
|
||||
it("t01 passes only after the correct address mutation", async () => {
|
||||
const task = TASKS.find((t) => t.id === "t01_address_happy");
|
||||
if (!task) throw new Error("t01 missing");
|
||||
|
||||
const { db, byName } = buildToolsFor(task);
|
||||
const transcript: Turn[] = [];
|
||||
|
||||
// Before mutation — should fail.
|
||||
expect(task.check({ db, finalAgentMessage: "", transcript })).toBe(false);
|
||||
|
||||
const update = byName.get("update_address");
|
||||
if (!update) throw new Error("update_address tool missing");
|
||||
await update.fn({ orderId: "o_1002", address: "5 Birch Rd, NYC, NY 10001" });
|
||||
|
||||
expect(task.check({ db, finalAgentMessage: "", transcript })).toBe(true);
|
||||
});
|
||||
|
||||
it("t02 (refusal) passes when the DB is left unchanged", () => {
|
||||
const task = TASKS.find((t) => t.id === "t02_address_not_allowed");
|
||||
if (!task) throw new Error("t02 missing");
|
||||
const { db } = buildToolsFor(task);
|
||||
expect(task.check({ db, finalAgentMessage: "", transcript: [] })).toBe(true);
|
||||
});
|
||||
|
||||
it("t02 (refusal) fails if the agent mutates anyway", async () => {
|
||||
const task = TASKS.find((t) => t.id === "t02_address_not_allowed");
|
||||
if (!task) throw new Error("t02 missing");
|
||||
const { db } = buildToolsFor(task);
|
||||
// The update_address tool itself refuses non-processing orders, so
|
||||
// simulate a misbehaving agent by mutating the DB directly.
|
||||
db.orders.o_1001!.address = "99 New St, SF, CA";
|
||||
expect(task.check({ db, finalAgentMessage: "", transcript: [] })).toBe(false);
|
||||
});
|
||||
|
||||
it("t04 refund check requires both status change and refund record", async () => {
|
||||
const task = TASKS.find((t) => t.id === "t04_refund_delivered");
|
||||
if (!task) throw new Error("t04 missing");
|
||||
const { db, byName } = buildToolsFor(task);
|
||||
const refund = byName.get("refund_order");
|
||||
if (!refund) throw new Error("refund_order tool missing");
|
||||
await refund.fn({ orderId: "o_1003", reason: "arrived broken" });
|
||||
expect(task.check({ db, finalAgentMessage: "", transcript: [] })).toBe(true);
|
||||
});
|
||||
|
||||
it("t05 passes when the agent refuses outright (status stays processing, no refund)", () => {
|
||||
const task = TASKS.find((t) => t.id === "t05_refund_not_delivered");
|
||||
if (!task) throw new Error("t05 missing");
|
||||
const { db } = buildToolsFor(task);
|
||||
// Out-of-the-box seed: o_1002 is processing, no refunds row.
|
||||
expect(task.check({ db, finalAgentMessage: "", transcript: [] })).toBe(true);
|
||||
});
|
||||
|
||||
it("t05 ALSO passes when the agent cancels the order instead (acceptable alternative)", async () => {
|
||||
const task = TASKS.find((t) => t.id === "t05_refund_not_delivered");
|
||||
if (!task) throw new Error("t05 missing");
|
||||
const { db, byName } = buildToolsFor(task);
|
||||
const cancel = byName.get("cancel_order");
|
||||
if (!cancel) throw new Error("cancel_order tool missing");
|
||||
await cancel.fn({ orderId: "o_1002" });
|
||||
expect(db.orders.o_1002?.status).toBe("cancelled");
|
||||
expect(task.check({ db, finalAgentMessage: "", transcript: [] })).toBe(true);
|
||||
});
|
||||
|
||||
it("t05 FAILS if the agent writes a refund row (the behavior we actually forbid)", () => {
|
||||
const task = TASKS.find((t) => t.id === "t05_refund_not_delivered");
|
||||
if (!task) throw new Error("t05 missing");
|
||||
const { db } = buildToolsFor(task);
|
||||
// Simulate the forbidden mutation directly — the refund_order tool
|
||||
// itself guards against non-delivered orders, so we have to be the
|
||||
// misbehaving agent here.
|
||||
db.refunds.o_1002 = { orderId: "o_1002", reason: "any", amount: 140 };
|
||||
expect(task.check({ db, finalAgentMessage: "", transcript: [] })).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("tool guards", () => {
|
||||
it("update_address refuses non-processing orders", async () => {
|
||||
const task = TASKS[0]!;
|
||||
const { byName } = buildToolsFor(task);
|
||||
const update = byName.get("update_address");
|
||||
if (!update) throw new Error("update_address tool missing");
|
||||
const out = await update.fn({ orderId: "o_1001", address: "anything" });
|
||||
expect(JSON.stringify(out)).toContain("cannot edit");
|
||||
});
|
||||
|
||||
it("refund_order refuses non-delivered orders", async () => {
|
||||
const task = TASKS[0]!;
|
||||
const { byName } = buildToolsFor(task);
|
||||
const refund = byName.get("refund_order");
|
||||
if (!refund) throw new Error("refund_order tool missing");
|
||||
const out = await refund.fn({ orderId: "o_1002", reason: "any" });
|
||||
expect(JSON.stringify(out)).toContain("cannot refund");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,82 @@
|
||||
import { existsSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
isPlausibleKey,
|
||||
loadApiKey,
|
||||
readConfig,
|
||||
redactKey,
|
||||
saveApiKey,
|
||||
writeConfig,
|
||||
} from "../src/config.js";
|
||||
|
||||
describe("config", () => {
|
||||
let dir: string;
|
||||
let path: string;
|
||||
const originalEnv = process.env.DEEPSEEK_API_KEY;
|
||||
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(join(tmpdir(), "reasonix-test-"));
|
||||
path = join(dir, "config.json");
|
||||
// biome-ignore lint/performance/noDelete: the string "undefined" leaks into process.env otherwise
|
||||
delete process.env.DEEPSEEK_API_KEY;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (existsSync(dir)) rmSync(dir, { recursive: true, force: true });
|
||||
if (originalEnv === undefined) {
|
||||
// biome-ignore lint/performance/noDelete: same reason as beforeEach
|
||||
delete process.env.DEEPSEEK_API_KEY;
|
||||
} else {
|
||||
process.env.DEEPSEEK_API_KEY = originalEnv;
|
||||
}
|
||||
});
|
||||
|
||||
it("readConfig returns {} when file is missing", () => {
|
||||
expect(readConfig(path)).toEqual({});
|
||||
});
|
||||
|
||||
it("writeConfig + readConfig round-trip", () => {
|
||||
writeConfig({ apiKey: "sk-test123abcdefghijkl" }, path);
|
||||
expect(readConfig(path).apiKey).toBe("sk-test123abcdefghijkl");
|
||||
});
|
||||
|
||||
it("saveApiKey trims whitespace", () => {
|
||||
saveApiKey(" sk-test123abcdefghijkl ", path);
|
||||
expect(readConfig(path).apiKey).toBe("sk-test123abcdefghijkl");
|
||||
});
|
||||
|
||||
it("loadApiKey prefers env var over config file", () => {
|
||||
saveApiKey("sk-fromfile1234567890ab", path);
|
||||
process.env.DEEPSEEK_API_KEY = "sk-fromenv1234567890abcd";
|
||||
expect(loadApiKey(path)).toBe("sk-fromenv1234567890abcd");
|
||||
});
|
||||
|
||||
it("loadApiKey falls back to config file when env unset", () => {
|
||||
saveApiKey("sk-fromfile1234567890ab", path);
|
||||
expect(loadApiKey(path)).toBe("sk-fromfile1234567890ab");
|
||||
});
|
||||
|
||||
it("loadApiKey returns undefined when nothing set", () => {
|
||||
expect(loadApiKey(path)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("isPlausibleKey accepts valid sk- keys", () => {
|
||||
expect(isPlausibleKey("sk-1234567890abcdef")).toBe(true);
|
||||
expect(isPlausibleKey("sk-abcDEF_123-456789012")).toBe(true);
|
||||
});
|
||||
|
||||
it("isPlausibleKey rejects bad input", () => {
|
||||
expect(isPlausibleKey("")).toBe(false);
|
||||
expect(isPlausibleKey("hello")).toBe(false);
|
||||
expect(isPlausibleKey("sk-short")).toBe(false);
|
||||
expect(isPlausibleKey("token-1234567890abcdef")).toBe(false);
|
||||
});
|
||||
|
||||
it("redactKey hides the middle", () => {
|
||||
expect(redactKey("sk-1234567890abcdefghij")).toBe("sk-123…ghij");
|
||||
expect(redactKey("short")).toBe("****");
|
||||
expect(redactKey("")).toBe("");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,231 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { DeepSeekClient, Usage } from "../src/client.js";
|
||||
import {
|
||||
type BranchSample,
|
||||
aggregateBranchUsage,
|
||||
defaultSelector,
|
||||
runBranches,
|
||||
} from "../src/consistency.js";
|
||||
import { emptyPlanState } from "../src/harvest.js";
|
||||
|
||||
interface MainCallScript {
|
||||
content: string;
|
||||
reasoning: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fake fetch that returns different main-call responses per branch (keyed by
|
||||
* temperature) and a canned plan-state JSON for every harvest call.
|
||||
*/
|
||||
function makeFakeFetch(
|
||||
mainByTemp: Record<string, MainCallScript>,
|
||||
harvestJson: string | ((reasoning: string) => string),
|
||||
) {
|
||||
return vi.fn(async (_url: string, init: RequestInit) => {
|
||||
const body = JSON.parse((init.body as string) ?? "{}");
|
||||
const isHarvest = body.response_format?.type === "json_object";
|
||||
if (isHarvest) {
|
||||
const reasoningMsg = body.messages?.[1]?.content ?? "";
|
||||
const payload = typeof harvestJson === "function" ? harvestJson(reasoningMsg) : harvestJson;
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
choices: [{ index: 0, message: { role: "assistant", content: payload } }],
|
||||
usage: { prompt_tokens: 50, completion_tokens: 20 },
|
||||
}),
|
||||
{ status: 200, headers: { "Content-Type": "application/json" } },
|
||||
);
|
||||
}
|
||||
const key = String(body.temperature ?? 0);
|
||||
const script = mainByTemp[key] ?? Object.values(mainByTemp)[0]!;
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: script.content,
|
||||
reasoning_content: script.reasoning.length > 0 ? script.reasoning : null,
|
||||
},
|
||||
},
|
||||
],
|
||||
usage: { prompt_tokens: 100, completion_tokens: 30 },
|
||||
}),
|
||||
{ status: 200, headers: { "Content-Type": "application/json" } },
|
||||
);
|
||||
}) as unknown as typeof fetch;
|
||||
}
|
||||
|
||||
describe("aggregateBranchUsage", () => {
|
||||
it("sums token counts across all samples", () => {
|
||||
const mk = (pt: number, ct: number, hit: number, miss: number): BranchSample => ({
|
||||
index: 0,
|
||||
temperature: 0,
|
||||
response: {
|
||||
content: "",
|
||||
reasoningContent: null,
|
||||
toolCalls: [],
|
||||
usage: new Usage(pt, ct, pt + ct, hit, miss),
|
||||
raw: null,
|
||||
},
|
||||
planState: emptyPlanState(),
|
||||
});
|
||||
const agg = aggregateBranchUsage([
|
||||
mk(100, 50, 80, 20),
|
||||
mk(200, 70, 150, 50),
|
||||
mk(120, 40, 100, 20),
|
||||
]);
|
||||
expect(agg.promptTokens).toBe(420);
|
||||
expect(agg.completionTokens).toBe(160);
|
||||
expect(agg.promptCacheHitTokens).toBe(330);
|
||||
expect(agg.promptCacheMissTokens).toBe(90);
|
||||
});
|
||||
});
|
||||
|
||||
describe("defaultSelector", () => {
|
||||
it("picks the sample with fewest uncertainties", () => {
|
||||
const mk = (i: number, u: number, content = "x"): BranchSample => ({
|
||||
index: i,
|
||||
temperature: 0,
|
||||
response: {
|
||||
content,
|
||||
reasoningContent: null,
|
||||
toolCalls: [],
|
||||
usage: {} as any,
|
||||
raw: null,
|
||||
},
|
||||
planState: { ...emptyPlanState(), uncertainties: new Array(u).fill("u") },
|
||||
});
|
||||
const chosen = defaultSelector([mk(0, 3), mk(1, 0), mk(2, 2)]);
|
||||
expect(chosen.index).toBe(1);
|
||||
});
|
||||
|
||||
it("tie-breaks ties on shorter content", () => {
|
||||
const mk = (i: number, len: number): BranchSample => ({
|
||||
index: i,
|
||||
temperature: 0,
|
||||
response: {
|
||||
content: "x".repeat(len),
|
||||
reasoningContent: null,
|
||||
toolCalls: [],
|
||||
usage: {} as any,
|
||||
raw: null,
|
||||
},
|
||||
planState: emptyPlanState(),
|
||||
});
|
||||
const chosen = defaultSelector([mk(0, 500), mk(1, 100), mk(2, 1000)]);
|
||||
expect(chosen.index).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("runBranches", () => {
|
||||
it("runs 1 sample when budget=1 (no branching)", async () => {
|
||||
const fakeFetch = makeFakeFetch(
|
||||
{ "0": { content: "answer", reasoning: "a long enough reasoning trace to pass threshold." } },
|
||||
JSON.stringify({
|
||||
subgoals: [],
|
||||
hypotheses: [],
|
||||
uncertainties: [],
|
||||
rejectedPaths: [],
|
||||
}),
|
||||
);
|
||||
const client = new DeepSeekClient({ apiKey: "sk-test", fetch: fakeFetch });
|
||||
const r = await runBranches(
|
||||
client,
|
||||
{ model: "deepseek-reasoner", messages: [{ role: "user", content: "hi" }] },
|
||||
{ budget: 1 },
|
||||
);
|
||||
expect(r.samples.length).toBe(1);
|
||||
expect(r.chosen.response.content).toBe("answer");
|
||||
});
|
||||
|
||||
it("runs N samples in parallel with varied temperatures", async () => {
|
||||
const fakeFetch = makeFakeFetch(
|
||||
{
|
||||
"0": { content: "ans-0", reasoning: "reasoning trace for sample 0 that is long enough." },
|
||||
"0.5": {
|
||||
content: "ans-05",
|
||||
reasoning: "reasoning trace for sample 1 that is long enough.",
|
||||
},
|
||||
"1": { content: "ans-1", reasoning: "reasoning trace for sample 2 that is long enough." },
|
||||
},
|
||||
JSON.stringify({
|
||||
subgoals: [],
|
||||
hypotheses: [],
|
||||
uncertainties: [],
|
||||
rejectedPaths: [],
|
||||
}),
|
||||
);
|
||||
const client = new DeepSeekClient({ apiKey: "sk-test", fetch: fakeFetch });
|
||||
const r = await runBranches(
|
||||
client,
|
||||
{ model: "deepseek-reasoner", messages: [{ role: "user", content: "hi" }] },
|
||||
{ budget: 3 },
|
||||
);
|
||||
expect(r.samples.length).toBe(3);
|
||||
const temps = r.samples.map((s) => s.temperature);
|
||||
expect(temps).toEqual([0, 0.5, 1]);
|
||||
});
|
||||
|
||||
it("selects the sample with fewest uncertainties", async () => {
|
||||
const fakeFetch = makeFakeFetch(
|
||||
{
|
||||
"0": { content: "ans-0", reasoning: "reasoning trace for sample 0 that is long enough." },
|
||||
"1": { content: "ans-1", reasoning: "reasoning trace for sample 1 that is long enough." },
|
||||
},
|
||||
(reasoning) => {
|
||||
const count = /sample 0/.test(reasoning) ? 3 : 0;
|
||||
return JSON.stringify({
|
||||
subgoals: [],
|
||||
hypotheses: [],
|
||||
uncertainties: new Array(count).fill("unclear thing"),
|
||||
rejectedPaths: [],
|
||||
});
|
||||
},
|
||||
);
|
||||
const client = new DeepSeekClient({ apiKey: "sk-test", fetch: fakeFetch });
|
||||
const r = await runBranches(
|
||||
client,
|
||||
{ model: "deepseek-reasoner", messages: [{ role: "user", content: "hi" }] },
|
||||
{ budget: 2 },
|
||||
);
|
||||
expect(r.chosen.response.content).toBe("ans-1");
|
||||
expect(r.chosen.planState.uncertainties.length).toBe(0);
|
||||
});
|
||||
|
||||
it("fires onSampleDone exactly once per sample", async () => {
|
||||
const fakeFetch = makeFakeFetch(
|
||||
{
|
||||
"0": { content: "ans", reasoning: "reasoning trace for sample 0, long enough." },
|
||||
"0.5": { content: "ans", reasoning: "reasoning trace for sample 1, long enough." },
|
||||
"1": { content: "ans", reasoning: "reasoning trace for sample 2, long enough." },
|
||||
},
|
||||
JSON.stringify({ subgoals: [], hypotheses: [], uncertainties: [], rejectedPaths: [] }),
|
||||
);
|
||||
const client = new DeepSeekClient({ apiKey: "sk-test", fetch: fakeFetch });
|
||||
const onSampleDone = vi.fn();
|
||||
await runBranches(
|
||||
client,
|
||||
{ model: "deepseek-reasoner", messages: [{ role: "user", content: "hi" }] },
|
||||
{ budget: 3, onSampleDone },
|
||||
);
|
||||
expect(onSampleDone).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it("accepts custom temperatures override", async () => {
|
||||
const fakeFetch = makeFakeFetch(
|
||||
{
|
||||
"0.2": { content: "ans", reasoning: "reasoning trace for this sample, long enough." },
|
||||
"0.9": { content: "ans", reasoning: "reasoning trace for this sample, long enough." },
|
||||
},
|
||||
JSON.stringify({ subgoals: [], hypotheses: [], uncertainties: [], rejectedPaths: [] }),
|
||||
);
|
||||
const client = new DeepSeekClient({ apiKey: "sk-test", fetch: fakeFetch });
|
||||
const r = await runBranches(
|
||||
client,
|
||||
{ model: "deepseek-reasoner", messages: [{ role: "user", content: "hi" }] },
|
||||
{ budget: 2, temperatures: [0.2, 0.9] },
|
||||
);
|
||||
expect(r.samples.map((s) => s.temperature)).toEqual([0.2, 0.9]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,263 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
diffTranscripts,
|
||||
findNextDivergence,
|
||||
findPrevDivergence,
|
||||
renderMarkdown,
|
||||
renderSummaryTable,
|
||||
similarity,
|
||||
} from "../src/diff.js";
|
||||
import type { ReadTranscriptResult, TranscriptRecord } from "../src/transcript.js";
|
||||
|
||||
function mkParsed(records: TranscriptRecord[], task = "t01"): ReadTranscriptResult {
|
||||
return {
|
||||
meta: {
|
||||
version: 1,
|
||||
source: "test",
|
||||
model: "deepseek-chat",
|
||||
task,
|
||||
mode: "test",
|
||||
startedAt: "2026-04-21T00:00:00Z",
|
||||
},
|
||||
records,
|
||||
};
|
||||
}
|
||||
|
||||
const mkUserA = (turn: number, content: string): TranscriptRecord => ({
|
||||
ts: "t",
|
||||
turn,
|
||||
role: "user",
|
||||
content,
|
||||
});
|
||||
const mkAssistant = (
|
||||
turn: number,
|
||||
content: string,
|
||||
opts: { hit?: number; miss?: number; cost?: number; prefixHash?: string } = {},
|
||||
): TranscriptRecord => ({
|
||||
ts: "t",
|
||||
turn,
|
||||
role: "assistant_final",
|
||||
content,
|
||||
model: "deepseek-chat",
|
||||
prefixHash: opts.prefixHash ?? "stable",
|
||||
usage: {
|
||||
prompt_tokens: (opts.hit ?? 900) + (opts.miss ?? 100),
|
||||
completion_tokens: 50,
|
||||
total_tokens: (opts.hit ?? 900) + (opts.miss ?? 100) + 50,
|
||||
prompt_cache_hit_tokens: opts.hit ?? 900,
|
||||
prompt_cache_miss_tokens: opts.miss ?? 100,
|
||||
},
|
||||
cost: opts.cost ?? 0.0001,
|
||||
});
|
||||
const mkTool = (turn: number, name: string, args = "{}"): TranscriptRecord => ({
|
||||
ts: "t",
|
||||
turn,
|
||||
role: "tool",
|
||||
content: "{}",
|
||||
tool: name,
|
||||
args,
|
||||
});
|
||||
|
||||
describe("similarity", () => {
|
||||
it("returns 1 for identical strings", () => {
|
||||
expect(similarity("hello world", "hello world")).toBe(1);
|
||||
});
|
||||
it("returns 0 for disjoint short strings", () => {
|
||||
expect(similarity("abc", "xyz")).toBeCloseTo(0, 1);
|
||||
});
|
||||
it("is high for small edits", () => {
|
||||
expect(similarity("the quick brown fox", "the quick brown dog")).toBeGreaterThan(0.8);
|
||||
});
|
||||
});
|
||||
|
||||
describe("diffTranscripts", () => {
|
||||
it("marks an all-same pair as match with no divergence", () => {
|
||||
const shared: TranscriptRecord[] = [
|
||||
mkUserA(1, "hi"),
|
||||
mkTool(1, "lookup"),
|
||||
mkAssistant(1, "hello there friend"),
|
||||
];
|
||||
const report = diffTranscripts(
|
||||
{ label: "A", parsed: mkParsed(shared) },
|
||||
{ label: "B", parsed: mkParsed([...shared]) },
|
||||
);
|
||||
expect(report.pairs).toHaveLength(1);
|
||||
expect(report.pairs[0]!.kind).toBe("match");
|
||||
expect(report.firstDivergenceTurn).toBeNull();
|
||||
});
|
||||
|
||||
it("flags tool-name disagreement as divergence on the correct turn", () => {
|
||||
const a: TranscriptRecord[] = [
|
||||
mkUserA(1, "hi"),
|
||||
mkTool(1, "lookup_order"),
|
||||
mkAssistant(1, "ok"),
|
||||
mkUserA(2, "change it"),
|
||||
mkTool(2, "update_address"),
|
||||
mkAssistant(2, "done"),
|
||||
];
|
||||
const b: TranscriptRecord[] = [
|
||||
mkUserA(1, "hi"),
|
||||
mkTool(1, "lookup_order"),
|
||||
mkAssistant(1, "ok"),
|
||||
mkUserA(2, "change it"),
|
||||
mkTool(2, "cancel_order"), // <-- different tool on turn 2
|
||||
mkAssistant(2, "done"),
|
||||
];
|
||||
const report = diffTranscripts(
|
||||
{ label: "A", parsed: mkParsed(a) },
|
||||
{ label: "B", parsed: mkParsed(b) },
|
||||
);
|
||||
expect(report.firstDivergenceTurn).toBe(2);
|
||||
const p2 = report.pairs.find((p) => p.turn === 2)!;
|
||||
expect(p2.kind).toBe("diverge");
|
||||
expect(p2.divergenceNote).toMatch(/tool calls differ/);
|
||||
});
|
||||
|
||||
it("flags same-tool-different-args as args-divergence", () => {
|
||||
const a: TranscriptRecord[] = [
|
||||
mkTool(1, "update_address", '{"orderId":"o_1","address":"A"}'),
|
||||
mkAssistant(1, "ok"),
|
||||
];
|
||||
const b: TranscriptRecord[] = [
|
||||
mkTool(1, "update_address", '{"orderId":"o_1","address":"DIFFERENT"}'),
|
||||
mkAssistant(1, "ok"),
|
||||
];
|
||||
const report = diffTranscripts(
|
||||
{ label: "A", parsed: mkParsed(a) },
|
||||
{ label: "B", parsed: mkParsed(b) },
|
||||
);
|
||||
const p1 = report.pairs[0]!;
|
||||
expect(p1.kind).toBe("diverge");
|
||||
expect(p1.divergenceNote).toMatch(/args differ/);
|
||||
});
|
||||
|
||||
it("computes correct aggregate deltas (cache / cost / turns)", () => {
|
||||
const baseline = [
|
||||
mkAssistant(1, "one", { hit: 100, miss: 900, cost: 0.0003, prefixHash: "h1" }),
|
||||
mkAssistant(2, "two", { hit: 100, miss: 900, cost: 0.0003, prefixHash: "h2" }),
|
||||
];
|
||||
const reasonix = [
|
||||
mkAssistant(1, "one", { hit: 900, miss: 100, cost: 0.0001, prefixHash: "stable" }),
|
||||
mkAssistant(2, "two", { hit: 900, miss: 100, cost: 0.0001, prefixHash: "stable" }),
|
||||
];
|
||||
const report = diffTranscripts(
|
||||
{ label: "baseline", parsed: mkParsed(baseline) },
|
||||
{ label: "reasonix", parsed: mkParsed(reasonix) },
|
||||
);
|
||||
expect(report.a.stats.cacheHitRatio).toBeCloseTo(0.1, 2);
|
||||
expect(report.b.stats.cacheHitRatio).toBeCloseTo(0.9, 2);
|
||||
expect(report.a.stats.prefixHashes.length).toBe(2);
|
||||
expect(report.b.stats.prefixHashes.length).toBe(1);
|
||||
expect(report.b.stats.totalCostUsd).toBeLessThan(report.a.stats.totalCostUsd);
|
||||
});
|
||||
|
||||
it("labels only_in_b when transcript B has extra turns", () => {
|
||||
const a = [mkAssistant(1, "one")];
|
||||
const b = [mkAssistant(1, "one"), mkAssistant(2, "two")];
|
||||
const report = diffTranscripts(
|
||||
{ label: "A", parsed: mkParsed(a) },
|
||||
{ label: "B", parsed: mkParsed(b) },
|
||||
);
|
||||
expect(report.pairs).toHaveLength(2);
|
||||
expect(report.pairs[1]!.kind).toBe("only_in_b");
|
||||
});
|
||||
});
|
||||
|
||||
describe("renderers", () => {
|
||||
it("renderSummaryTable includes cache delta + prefix stability story when warranted", () => {
|
||||
const baseline = [
|
||||
mkAssistant(1, "one", { hit: 100, miss: 900, prefixHash: "h1" }),
|
||||
mkAssistant(2, "two", { hit: 100, miss: 900, prefixHash: "h2" }),
|
||||
];
|
||||
const reasonix = [
|
||||
mkAssistant(1, "one", { hit: 900, miss: 100, prefixHash: "stable" }),
|
||||
mkAssistant(2, "two", { hit: 900, miss: 100, prefixHash: "stable" }),
|
||||
];
|
||||
const report = diffTranscripts(
|
||||
{ label: "baseline", parsed: mkParsed(baseline) },
|
||||
{ label: "reasonix", parsed: mkParsed(reasonix) },
|
||||
);
|
||||
const table = renderSummaryTable(report);
|
||||
expect(table).toContain("cache hit");
|
||||
expect(table).toContain("prefix stability");
|
||||
expect(table).toMatch(/byte-stable/);
|
||||
});
|
||||
|
||||
it("renderMarkdown includes the per-turn table and summary", () => {
|
||||
const a = [mkAssistant(1, "A answer")];
|
||||
const b = [mkAssistant(1, "B answer text")];
|
||||
const report = diffTranscripts(
|
||||
{ label: "A", parsed: mkParsed(a) },
|
||||
{ label: "B", parsed: mkParsed(b) },
|
||||
);
|
||||
const md = renderMarkdown(report);
|
||||
expect(md).toMatch(/## Summary/);
|
||||
expect(md).toMatch(/## Turn-by-turn/);
|
||||
expect(md).toMatch(/cache hit/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("divergence navigation (TUI)", () => {
|
||||
// Build a report whose pair kinds form a predictable pattern.
|
||||
function reportWithPattern(): ReturnType<typeof diffTranscripts> {
|
||||
// turns 1-5: match, diverge, match, diverge, match
|
||||
const a = [
|
||||
mkAssistant(1, "same"),
|
||||
mkTool(2, "lookup_order"),
|
||||
mkAssistant(2, "a2"),
|
||||
mkAssistant(3, "same3"),
|
||||
mkTool(4, "lookup_user"),
|
||||
mkAssistant(4, "a4"),
|
||||
mkAssistant(5, "same5"),
|
||||
];
|
||||
const b = [
|
||||
mkAssistant(1, "same"),
|
||||
mkTool(2, "cancel_order"), // tool name differs on turn 2
|
||||
mkAssistant(2, "b2"),
|
||||
mkAssistant(3, "same3"),
|
||||
mkTool(4, "lookup_user"),
|
||||
mkAssistant(4, "a4 very different answer content here"), // text differs on turn 4
|
||||
mkAssistant(5, "same5"),
|
||||
];
|
||||
return diffTranscripts(
|
||||
{ label: "A", parsed: mkParsed(a) },
|
||||
{ label: "B", parsed: mkParsed(b) },
|
||||
);
|
||||
}
|
||||
|
||||
it("findNextDivergence returns the next non-match index, or -1", () => {
|
||||
const report = reportWithPattern();
|
||||
// pattern: [match, diverge, match, diverge, match]
|
||||
expect(findNextDivergence(report.pairs, -1)).toBe(1);
|
||||
expect(findNextDivergence(report.pairs, 0)).toBe(1);
|
||||
expect(findNextDivergence(report.pairs, 1)).toBe(3);
|
||||
expect(findNextDivergence(report.pairs, 3)).toBe(-1);
|
||||
expect(findNextDivergence(report.pairs, 4)).toBe(-1);
|
||||
});
|
||||
|
||||
it("findPrevDivergence walks backwards", () => {
|
||||
const report = reportWithPattern();
|
||||
expect(findPrevDivergence(report.pairs, 4)).toBe(3);
|
||||
expect(findPrevDivergence(report.pairs, 3)).toBe(1);
|
||||
expect(findPrevDivergence(report.pairs, 1)).toBe(-1);
|
||||
expect(findPrevDivergence(report.pairs, 0)).toBe(-1);
|
||||
});
|
||||
|
||||
it("returns -1 on an empty report", () => {
|
||||
const report = diffTranscripts(
|
||||
{ label: "A", parsed: mkParsed([]) },
|
||||
{ label: "B", parsed: mkParsed([]) },
|
||||
);
|
||||
expect(findNextDivergence(report.pairs, -1)).toBe(-1);
|
||||
expect(findPrevDivergence(report.pairs, 99)).toBe(-1);
|
||||
});
|
||||
|
||||
it("returns -1 when every pair matches", () => {
|
||||
const recs = [mkAssistant(1, "ok"), mkAssistant(2, "ok")];
|
||||
const report = diffTranscripts(
|
||||
{ label: "A", parsed: mkParsed(recs) },
|
||||
{ label: "B", parsed: mkParsed(recs) },
|
||||
);
|
||||
expect(findNextDivergence(report.pairs, -1)).toBe(-1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,155 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { DeepSeekClient } from "../src/client.js";
|
||||
import { emptyPlanState, harvest, isPlanStateEmpty } from "../src/harvest.js";
|
||||
|
||||
function stubClient(jsonPayload: string): DeepSeekClient {
|
||||
const fakeFetch = vi.fn(async () => {
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
choices: [{ index: 0, message: { role: "assistant", content: jsonPayload } }],
|
||||
usage: { prompt_tokens: 50, completion_tokens: 30 },
|
||||
}),
|
||||
{ status: 200, headers: { "Content-Type": "application/json" } },
|
||||
);
|
||||
}) as unknown as typeof fetch;
|
||||
return new DeepSeekClient({ apiKey: "sk-test", fetch: fakeFetch });
|
||||
}
|
||||
|
||||
function erroringClient(): DeepSeekClient {
|
||||
const fakeFetch = vi.fn(async () => new Response("boom", { status: 500 }));
|
||||
return new DeepSeekClient({
|
||||
apiKey: "sk-test",
|
||||
fetch: fakeFetch as unknown as typeof fetch,
|
||||
retry: { maxAttempts: 1, initialBackoffMs: 1 },
|
||||
});
|
||||
}
|
||||
|
||||
describe("emptyPlanState / isPlanStateEmpty", () => {
|
||||
it("empty state reports empty", () => {
|
||||
expect(isPlanStateEmpty(emptyPlanState())).toBe(true);
|
||||
expect(isPlanStateEmpty(null)).toBe(true);
|
||||
expect(isPlanStateEmpty(undefined)).toBe(true);
|
||||
});
|
||||
it("non-empty state reports non-empty", () => {
|
||||
const s = emptyPlanState();
|
||||
s.subgoals.push("x");
|
||||
expect(isPlanStateEmpty(s)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("harvest", () => {
|
||||
it("returns empty when no client is provided", async () => {
|
||||
const r = await harvest("some reasoning here that is long enough to qualify", undefined);
|
||||
expect(isPlanStateEmpty(r)).toBe(true);
|
||||
});
|
||||
|
||||
it("returns empty when reasoning is absent", async () => {
|
||||
const client = stubClient("{}");
|
||||
expect(isPlanStateEmpty(await harvest(null, client))).toBe(true);
|
||||
expect(isPlanStateEmpty(await harvest("", client))).toBe(true);
|
||||
});
|
||||
|
||||
it("returns empty when reasoning is too short", async () => {
|
||||
const client = stubClient(
|
||||
'{"subgoals":["x"],"hypotheses":[],"uncertainties":[],"rejectedPaths":[]}',
|
||||
);
|
||||
const r = await harvest("short", client);
|
||||
expect(isPlanStateEmpty(r)).toBe(true);
|
||||
});
|
||||
|
||||
it("parses a well-formed JSON response", async () => {
|
||||
const json = JSON.stringify({
|
||||
subgoals: ["解二次方程", "验证根"],
|
||||
hypotheses: ["可因式分解"],
|
||||
uncertainties: [],
|
||||
rejectedPaths: ["配方法"],
|
||||
});
|
||||
const client = stubClient(json);
|
||||
const r = await harvest(
|
||||
"一段足够长的推理文本,描述了如何解一个二次方程的全部过程,包括因式分解、求根公式对比以及验证步骤。",
|
||||
client,
|
||||
);
|
||||
expect(r.subgoals).toEqual(["解二次方程", "验证根"]);
|
||||
expect(r.hypotheses).toEqual(["可因式分解"]);
|
||||
expect(r.rejectedPaths).toEqual(["配方法"]);
|
||||
expect(r.uncertainties).toEqual([]);
|
||||
});
|
||||
|
||||
it("accepts snake_case rejected_paths as fallback", async () => {
|
||||
const json = JSON.stringify({
|
||||
subgoals: [],
|
||||
hypotheses: [],
|
||||
uncertainties: [],
|
||||
rejected_paths: ["bad-approach"],
|
||||
});
|
||||
const r = await harvest(
|
||||
"a long enough reasoning trace that exceeds the minimum length threshold.",
|
||||
stubClient(json),
|
||||
);
|
||||
expect(r.rejectedPaths).toEqual(["bad-approach"]);
|
||||
});
|
||||
|
||||
it("extracts JSON even when wrapped in prose", async () => {
|
||||
const body = JSON.stringify({
|
||||
subgoals: ["a"],
|
||||
hypotheses: [],
|
||||
uncertainties: [],
|
||||
rejectedPaths: [],
|
||||
});
|
||||
const wrapped = `Here is the JSON:\n\`\`\`json\n${body}\n\`\`\`\n`;
|
||||
const r = await harvest(
|
||||
"a long enough reasoning trace that exceeds the minimum length threshold.",
|
||||
stubClient(wrapped),
|
||||
);
|
||||
expect(r.subgoals).toEqual(["a"]);
|
||||
});
|
||||
|
||||
it("returns empty on malformed JSON response", async () => {
|
||||
const r = await harvest(
|
||||
"a long enough reasoning trace that exceeds the minimum length threshold.",
|
||||
stubClient("not json at all, just text"),
|
||||
);
|
||||
expect(isPlanStateEmpty(r)).toBe(true);
|
||||
});
|
||||
|
||||
it("gracefully swallows API errors and returns empty", async () => {
|
||||
const r = await harvest(
|
||||
"a long enough reasoning trace that exceeds the minimum length threshold.",
|
||||
erroringClient(),
|
||||
);
|
||||
expect(isPlanStateEmpty(r)).toBe(true);
|
||||
});
|
||||
|
||||
it("caps array length and trims long items", async () => {
|
||||
const longs = Array.from({ length: 10 }, (_, i) => `${"x".repeat(200)}${i}`);
|
||||
const json = JSON.stringify({
|
||||
subgoals: longs,
|
||||
hypotheses: [],
|
||||
uncertainties: [],
|
||||
rejectedPaths: [],
|
||||
});
|
||||
const r = await harvest(
|
||||
"a long enough reasoning trace that exceeds the minimum length threshold.",
|
||||
stubClient(json),
|
||||
{ maxItems: 3, maxItemLen: 20 },
|
||||
);
|
||||
expect(r.subgoals.length).toBe(3);
|
||||
for (const s of r.subgoals) {
|
||||
expect(s.length).toBeLessThanOrEqual(20);
|
||||
}
|
||||
});
|
||||
|
||||
it("filters non-string items", async () => {
|
||||
const json = JSON.stringify({
|
||||
subgoals: ["ok", 42, null, { nested: true }, "also-ok"],
|
||||
hypotheses: [],
|
||||
uncertainties: [],
|
||||
rejectedPaths: [],
|
||||
});
|
||||
const r = await harvest(
|
||||
"a long enough reasoning trace that exceeds the minimum length threshold.",
|
||||
stubClient(json),
|
||||
);
|
||||
expect(r.subgoals).toEqual(["ok", "also-ok"]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,209 @@
|
||||
/**
|
||||
* Integration tests for CacheFirstLoop.
|
||||
*
|
||||
* We inject a fake fetch into DeepSeekClient so the loop exercises its real
|
||||
* request/response wiring without hitting the network. The non-streaming
|
||||
* path is covered here; streaming is exercised by the TUI in practice.
|
||||
*/
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { DeepSeekClient } from "../src/client.js";
|
||||
import { CacheFirstLoop } from "../src/loop.js";
|
||||
import { ImmutablePrefix } from "../src/memory.js";
|
||||
import { ToolRegistry } from "../src/tools.js";
|
||||
import type { ChatMessage } from "../src/types.js";
|
||||
|
||||
interface FakeResponseShape {
|
||||
content?: string;
|
||||
reasoning_content?: string;
|
||||
tool_calls?: any[];
|
||||
usage?: Record<string, number>;
|
||||
}
|
||||
|
||||
function fakeFetch(responses: FakeResponseShape[]): typeof fetch {
|
||||
let i = 0;
|
||||
return vi.fn(async (_url: any, init: any) => {
|
||||
const body = init?.body ? JSON.parse(init.body) : {};
|
||||
const resp = responses[i++] ?? responses[responses.length - 1]!;
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
_echo_messages: body.messages as ChatMessage[],
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: resp.content ?? "",
|
||||
reasoning_content: resp.reasoning_content ?? null,
|
||||
tool_calls: resp.tool_calls ?? undefined,
|
||||
},
|
||||
finish_reason: resp.tool_calls ? "tool_calls" : "stop",
|
||||
},
|
||||
],
|
||||
usage: resp.usage ?? {
|
||||
prompt_tokens: 100,
|
||||
completion_tokens: 20,
|
||||
total_tokens: 120,
|
||||
prompt_cache_hit_tokens: 0,
|
||||
prompt_cache_miss_tokens: 100,
|
||||
},
|
||||
}),
|
||||
{ status: 200, headers: { "Content-Type": "application/json" } },
|
||||
);
|
||||
}) as unknown as typeof fetch;
|
||||
}
|
||||
|
||||
function makeClient(responses: FakeResponseShape[]) {
|
||||
return new DeepSeekClient({
|
||||
apiKey: "sk-test",
|
||||
fetch: fakeFetch(responses),
|
||||
});
|
||||
}
|
||||
|
||||
describe("CacheFirstLoop (non-streaming)", () => {
|
||||
it("completes a single-turn plain chat", async () => {
|
||||
const client = makeClient([{ content: "hi there" }]);
|
||||
const loop = new CacheFirstLoop({
|
||||
client,
|
||||
prefix: new ImmutablePrefix({ system: "be brief" }),
|
||||
stream: false,
|
||||
});
|
||||
|
||||
const events: string[] = [];
|
||||
for await (const ev of loop.step("hello")) {
|
||||
events.push(ev.role);
|
||||
}
|
||||
|
||||
expect(events).toContain("assistant_final");
|
||||
expect(events[events.length - 1]).toBe("done");
|
||||
expect(loop.stats.turns.length).toBe(1);
|
||||
expect(loop.log.length).toBe(2); // user + assistant
|
||||
});
|
||||
|
||||
it("records cache hit telemetry from API usage", async () => {
|
||||
const client = makeClient([
|
||||
{
|
||||
content: "ok",
|
||||
usage: {
|
||||
prompt_tokens: 1000,
|
||||
completion_tokens: 10,
|
||||
total_tokens: 1010,
|
||||
prompt_cache_hit_tokens: 800,
|
||||
prompt_cache_miss_tokens: 200,
|
||||
},
|
||||
},
|
||||
]);
|
||||
const loop = new CacheFirstLoop({
|
||||
client,
|
||||
prefix: new ImmutablePrefix({ system: "s" }),
|
||||
stream: false,
|
||||
});
|
||||
|
||||
await loop.run("q");
|
||||
expect(loop.stats.aggregateCacheHitRatio).toBeCloseTo(0.8);
|
||||
expect(loop.stats.totalCost).toBeGreaterThan(0);
|
||||
expect(loop.stats.savingsVsClaude).toBeGreaterThan(0.9);
|
||||
});
|
||||
|
||||
it("dispatches a tool call and loops until the model stops", async () => {
|
||||
const client = makeClient([
|
||||
{
|
||||
content: "",
|
||||
tool_calls: [
|
||||
{
|
||||
id: "call_1",
|
||||
type: "function",
|
||||
function: { name: "add", arguments: '{"a":2,"b":3}' },
|
||||
},
|
||||
],
|
||||
},
|
||||
{ content: "The answer is 5." },
|
||||
]);
|
||||
|
||||
const tools = new ToolRegistry();
|
||||
tools.register<{ a: number; b: number }, number>({
|
||||
name: "add",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: { a: { type: "integer" }, b: { type: "integer" } },
|
||||
required: ["a", "b"],
|
||||
},
|
||||
fn: ({ a, b }) => a + b,
|
||||
});
|
||||
|
||||
const loop = new CacheFirstLoop({
|
||||
client,
|
||||
prefix: new ImmutablePrefix({
|
||||
system: "use add tool",
|
||||
toolSpecs: tools.specs(),
|
||||
}),
|
||||
tools,
|
||||
stream: false,
|
||||
});
|
||||
|
||||
const roles: string[] = [];
|
||||
let toolContent = "";
|
||||
let finalContent = "";
|
||||
for await (const ev of loop.step("2 + 3 = ?")) {
|
||||
roles.push(ev.role);
|
||||
if (ev.role === "tool") toolContent = ev.content;
|
||||
if (ev.role === "assistant_final") finalContent = ev.content;
|
||||
}
|
||||
|
||||
expect(roles).toContain("tool");
|
||||
expect(toolContent).toBe("5");
|
||||
expect(finalContent).toBe("The answer is 5.");
|
||||
expect(loop.stats.turns.length).toBe(2); // two model round-trips
|
||||
});
|
||||
|
||||
it("immutable prefix is preserved across turns (cache-stability invariant)", async () => {
|
||||
const sharedFetch = fakeFetch([{ content: "a" }, { content: "b" }]);
|
||||
const client = new DeepSeekClient({ apiKey: "sk-test", fetch: sharedFetch });
|
||||
const loop = new CacheFirstLoop({
|
||||
client,
|
||||
prefix: new ImmutablePrefix({ system: "pinned system" }),
|
||||
stream: false,
|
||||
});
|
||||
|
||||
await loop.run("q1");
|
||||
await loop.run("q2");
|
||||
|
||||
const calls = (sharedFetch as any).mock.calls;
|
||||
expect(calls.length).toBe(2);
|
||||
const msgs1 = JSON.parse(calls[0][1].body).messages as ChatMessage[];
|
||||
const msgs2 = JSON.parse(calls[1][1].body).messages as ChatMessage[];
|
||||
|
||||
// Both requests start with the exact same system prefix (byte-identical).
|
||||
expect(msgs1[0]).toEqual({ role: "system", content: "pinned system" });
|
||||
expect(msgs2[0]).toEqual({ role: "system", content: "pinned system" });
|
||||
|
||||
// Second request should begin with msgs1 as its prefix
|
||||
// (append-only log invariant: history is never rewritten).
|
||||
for (let i = 0; i < msgs1.length; i++) {
|
||||
expect(msgs2[i]).toEqual(msgs1[i]);
|
||||
}
|
||||
// And msgs2 is strictly longer (new user turn + assistant reply from turn 1).
|
||||
expect(msgs2.length).toBeGreaterThan(msgs1.length);
|
||||
});
|
||||
|
||||
it("surfaces an error event when the HTTP call fails with a non-retryable status", async () => {
|
||||
// 401 is non-retryable (bad key). Using this avoids multi-retry waits.
|
||||
const errFetch = vi.fn(async () => new Response("boom", { status: 401 }));
|
||||
const client = new DeepSeekClient({
|
||||
apiKey: "sk-test",
|
||||
fetch: errFetch as unknown as typeof fetch,
|
||||
retry: { initialBackoffMs: 1, maxAttempts: 1 },
|
||||
});
|
||||
const loop = new CacheFirstLoop({
|
||||
client,
|
||||
prefix: new ImmutablePrefix({ system: "s" }),
|
||||
stream: false,
|
||||
});
|
||||
|
||||
const roles: string[] = [];
|
||||
for await (const ev of loop.step("q")) {
|
||||
roles.push(ev.role);
|
||||
}
|
||||
expect(roles).toContain("error");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,69 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { stripMath } from "../src/cli/ui/markdown.js";
|
||||
|
||||
describe("stripMath", () => {
|
||||
it("converts \\frac, \\dfrac, \\tfrac uniformly", () => {
|
||||
expect(stripMath("\\frac{a}{b}")).toBe("(a)/(b)");
|
||||
expect(stripMath("\\dfrac{a}{b}")).toBe("(a)/(b)");
|
||||
expect(stripMath("\\tfrac{a}{b}")).toBe("(a)/(b)");
|
||||
});
|
||||
|
||||
it("converts a \\frac with Chinese content (the original user-reported case)", () => {
|
||||
const out = stripMath("v = \\frac{总路程}{总时间}");
|
||||
expect(out).toContain("(总路程)/(总时间)");
|
||||
expect(out).not.toContain("\\frac");
|
||||
expect(out).not.toContain("\\");
|
||||
});
|
||||
|
||||
it("handles \\frac with a nested \\sqrt", () => {
|
||||
const out = stripMath("\\frac{\\sqrt{2}}{3}");
|
||||
expect(out).not.toContain("\\frac");
|
||||
expect(out).not.toContain("\\sqrt");
|
||||
});
|
||||
|
||||
it("tolerates whitespace inside frac braces", () => {
|
||||
expect(stripMath("\\frac{ a }{ b }")).toContain("(a)/(b)");
|
||||
});
|
||||
|
||||
it("strips \\implies, \\to arrows", () => {
|
||||
expect(stripMath("a \\implies b")).toContain("a ⇒ b");
|
||||
expect(stripMath("x \\to y")).toContain("x → y");
|
||||
});
|
||||
|
||||
it("\\quad becomes spaces", () => {
|
||||
expect(stripMath("a\\quad b")).not.toContain("\\");
|
||||
});
|
||||
|
||||
it("converts single-digit subscripts and superscripts to Unicode", () => {
|
||||
expect(stripMath("t_1 + t_2")).toBe("t₁ + t₂");
|
||||
expect(stripMath("x^2 + y^3")).toBe("x² + y³");
|
||||
});
|
||||
|
||||
it("strips LaTeX math delimiters", () => {
|
||||
const out = stripMath("equation: \\(x^2 + 1\\)");
|
||||
expect(out).toBe("equation: x² + 1");
|
||||
});
|
||||
|
||||
it("unknown commands are stripped too (catch-all fallback)", () => {
|
||||
const out = stripMath("\\weirdmacro{x}{y} + \\unknown{z} + \\alone");
|
||||
expect(out).not.toContain("\\");
|
||||
});
|
||||
|
||||
it("\\boxed wraps in 【…】", () => {
|
||||
expect(stripMath("\\boxed{x = 5}")).toBe("【x = 5】");
|
||||
});
|
||||
|
||||
it("\\sqrt becomes √(...)", () => {
|
||||
expect(stripMath("\\sqrt{49}")).toBe("√(49)");
|
||||
});
|
||||
|
||||
it("the full user-reported line no longer leaks raw LaTeX", () => {
|
||||
const input =
|
||||
"总路程:2d km 总时间:t_1 + t_2 = \\dfrac{d}{30} + \\dfrac{d}{60} = \\dfrac{2d}{60} + \\dfrac{d}{60} = \\dfrac{3d}{60} = \\dfrac{d}{20} 小时 平均速度:v_{avg} = \\frac{总路程}{总时间} = 40 km/h";
|
||||
const out = stripMath(input);
|
||||
expect(out).not.toContain("\\");
|
||||
expect(out).toContain("(总路程)/(总时间)");
|
||||
expect(out).toContain("t₁");
|
||||
expect(out).toContain("(d)/(30)");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,58 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { AppendOnlyLog, ImmutablePrefix, VolatileScratch } from "../src/memory.js";
|
||||
|
||||
describe("ImmutablePrefix", () => {
|
||||
it("fingerprint is stable for identical inputs", () => {
|
||||
const a = new ImmutablePrefix({ system: "hello" });
|
||||
const b = new ImmutablePrefix({ system: "hello" });
|
||||
expect(a.fingerprint).toBe(b.fingerprint);
|
||||
});
|
||||
|
||||
it("fingerprint changes with inputs", () => {
|
||||
const a = new ImmutablePrefix({ system: "hello" });
|
||||
const b = new ImmutablePrefix({ system: "world" });
|
||||
expect(a.fingerprint).not.toBe(b.fingerprint);
|
||||
});
|
||||
|
||||
it("toMessages includes system plus few-shots", () => {
|
||||
const p = new ImmutablePrefix({
|
||||
system: "sys",
|
||||
fewShots: [{ role: "user", content: "hi" }],
|
||||
});
|
||||
expect(p.toMessages()).toEqual([
|
||||
{ role: "system", content: "sys" },
|
||||
{ role: "user", content: "hi" },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("AppendOnlyLog", () => {
|
||||
it("appends in order and rejects malformed entries", () => {
|
||||
const log = new AppendOnlyLog();
|
||||
log.append({ role: "user", content: "hi" });
|
||||
log.append({ role: "assistant", content: "hello" });
|
||||
expect(log.length).toBe(2);
|
||||
expect(() => log.append({ content: "x" } as any)).toThrow();
|
||||
});
|
||||
|
||||
it("toMessages returns a shallow-copy not affecting internal state", () => {
|
||||
const log = new AppendOnlyLog();
|
||||
log.append({ role: "user", content: "hi" });
|
||||
const msgs = log.toMessages();
|
||||
msgs[0]!.content = "tampered";
|
||||
expect(log.entries[0]!.content).toBe("hi");
|
||||
});
|
||||
});
|
||||
|
||||
describe("VolatileScratch", () => {
|
||||
it("resets all fields", () => {
|
||||
const s = new VolatileScratch();
|
||||
s.reasoning = "x";
|
||||
s.planState = { a: 1 };
|
||||
s.notes.push("note");
|
||||
s.reset();
|
||||
expect(s.reasoning).toBeNull();
|
||||
expect(s.planState).toBeNull();
|
||||
expect(s.notes).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,71 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { analyzeSchema, flattenSchema, nestArguments } from "../../src/repair/flatten.js";
|
||||
|
||||
describe("analyzeSchema", () => {
|
||||
it("does not flatten flat shallow schemas", () => {
|
||||
const d = analyzeSchema({
|
||||
type: "object",
|
||||
properties: { a: { type: "string" }, b: { type: "number" } },
|
||||
});
|
||||
expect(d.shouldFlatten).toBe(false);
|
||||
});
|
||||
|
||||
it("flags deep schemas", () => {
|
||||
const d = analyzeSchema({
|
||||
type: "object",
|
||||
properties: {
|
||||
outer: {
|
||||
type: "object",
|
||||
properties: {
|
||||
middle: {
|
||||
type: "object",
|
||||
properties: { leaf: { type: "string" } },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(d.shouldFlatten).toBe(true);
|
||||
expect(d.maxDepth).toBeGreaterThan(2);
|
||||
});
|
||||
|
||||
it("flags wide schemas (>10 leaves)", () => {
|
||||
const props: Record<string, { type: string }> = {};
|
||||
for (let i = 0; i < 12; i++) props[`p${i}`] = { type: "string" };
|
||||
const d = analyzeSchema({ type: "object", properties: props });
|
||||
expect(d.shouldFlatten).toBe(true);
|
||||
expect(d.leafCount).toBe(12);
|
||||
});
|
||||
});
|
||||
|
||||
describe("flattenSchema / nestArguments roundtrip", () => {
|
||||
it("flattens nested schema and re-nests arguments", () => {
|
||||
const schema = {
|
||||
type: "object",
|
||||
required: ["user"],
|
||||
properties: {
|
||||
user: {
|
||||
type: "object",
|
||||
required: ["profile"],
|
||||
properties: {
|
||||
profile: {
|
||||
type: "object",
|
||||
required: ["name"],
|
||||
properties: { name: { type: "string" }, age: { type: "integer" } },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
const flat = flattenSchema(schema);
|
||||
expect(flat.properties).toHaveProperty("user.profile.name");
|
||||
expect(flat.properties).toHaveProperty("user.profile.age");
|
||||
expect(flat.required).toEqual(["user.profile.name"]);
|
||||
|
||||
const nested = nestArguments({
|
||||
"user.profile.name": "alice",
|
||||
"user.profile.age": 30,
|
||||
});
|
||||
expect(nested).toEqual({ user: { profile: { name: "alice", age: 30 } } });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,50 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { ToolCallRepair } from "../../src/repair/index.js";
|
||||
import type { ToolCall } from "../../src/types.js";
|
||||
|
||||
function call(id: string, name: string, args: string): ToolCall {
|
||||
return { id, type: "function", function: { name, arguments: args } };
|
||||
}
|
||||
|
||||
describe("ToolCallRepair pipeline", () => {
|
||||
it("merges scavenged calls with declared calls", () => {
|
||||
const repair = new ToolCallRepair({ allowedToolNames: new Set(["search"]) });
|
||||
const declared = [call("c1", "search", '{"q":"a"}')];
|
||||
const reasoning = `I should also run {"name": "search", "arguments": {"q": "b"}}`;
|
||||
const { calls, report } = repair.process(declared, reasoning);
|
||||
expect(calls.length).toBe(2);
|
||||
expect(report.scavenged).toBe(1);
|
||||
});
|
||||
|
||||
it("repairs truncated arguments JSON", () => {
|
||||
const repair = new ToolCallRepair({ allowedToolNames: new Set(["search"]) });
|
||||
const declared = [call("c1", "search", '{"q":"abc')];
|
||||
const { calls, report } = repair.process(declared, null);
|
||||
expect(calls.length).toBe(1);
|
||||
expect(() => JSON.parse(calls[0]!.function.arguments)).not.toThrow();
|
||||
expect(report.truncationsFixed).toBe(1);
|
||||
});
|
||||
|
||||
it("breaks call storms", () => {
|
||||
const repair = new ToolCallRepair({
|
||||
allowedToolNames: new Set(["x"]),
|
||||
stormWindow: 6,
|
||||
stormThreshold: 3,
|
||||
});
|
||||
for (let i = 0; i < 2; i++) {
|
||||
repair.process([call(`c${i}`, "x", "{}")], null);
|
||||
}
|
||||
const { calls, report } = repair.process([call("c3", "x", "{}")], null);
|
||||
expect(calls.length).toBe(0);
|
||||
expect(report.stormsBroken).toBe(1);
|
||||
});
|
||||
|
||||
it("dedupes scavenge vs declared by signature", () => {
|
||||
const repair = new ToolCallRepair({ allowedToolNames: new Set(["search"]) });
|
||||
const declared = [call("c1", "search", '{"q":"a"}')];
|
||||
const reasoning = `noted: {"name":"search","arguments":{"q":"a"}}`;
|
||||
const { calls, report } = repair.process(declared, reasoning);
|
||||
expect(calls.length).toBe(1);
|
||||
expect(report.scavenged).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,47 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { scavengeToolCalls } from "../../src/repair/scavenge.js";
|
||||
|
||||
const allowed = new Set(["get_weather", "search"]);
|
||||
|
||||
describe("scavengeToolCalls", () => {
|
||||
it("returns nothing for null reasoning", () => {
|
||||
const r = scavengeToolCalls(null, { allowedNames: allowed });
|
||||
expect(r.calls).toEqual([]);
|
||||
});
|
||||
|
||||
it('extracts pattern 1: {"name", "arguments"}', () => {
|
||||
const reasoning = `thinking... I should call {"name": "get_weather", "arguments": {"city": "SF"}}`;
|
||||
const r = scavengeToolCalls(reasoning, { allowedNames: allowed });
|
||||
expect(r.calls.length).toBe(1);
|
||||
expect(r.calls[0]!.function.name).toBe("get_weather");
|
||||
expect(JSON.parse(r.calls[0]!.function.arguments)).toEqual({ city: "SF" });
|
||||
});
|
||||
|
||||
it("extracts OpenAI-style envelope", () => {
|
||||
const reasoning = `plan: {"type":"function","function":{"name":"search","arguments":"{\\"q\\":\\"ts\\"}"}}`;
|
||||
const r = scavengeToolCalls(reasoning, { allowedNames: allowed });
|
||||
expect(r.calls[0]!.function.name).toBe("search");
|
||||
expect(JSON.parse(r.calls[0]!.function.arguments)).toEqual({ q: "ts" });
|
||||
});
|
||||
|
||||
it("extracts tool_name / tool_args variant", () => {
|
||||
const reasoning = `decide: {"tool_name": "search", "tool_args": {"q": "deepseek"}}`;
|
||||
const r = scavengeToolCalls(reasoning, { allowedNames: allowed });
|
||||
expect(r.calls[0]!.function.name).toBe("search");
|
||||
expect(JSON.parse(r.calls[0]!.function.arguments)).toEqual({ q: "deepseek" });
|
||||
});
|
||||
|
||||
it("ignores tools not in the allowed set", () => {
|
||||
const reasoning = `{"name": "rm_rf_slash", "arguments": {}}`;
|
||||
const r = scavengeToolCalls(reasoning, { allowedNames: allowed });
|
||||
expect(r.calls).toEqual([]);
|
||||
});
|
||||
|
||||
it("respects maxCalls", () => {
|
||||
const reasoning = Array.from({ length: 6 })
|
||||
.map(() => `{"name": "search", "arguments": {"q": "x"}}`)
|
||||
.join(" then ");
|
||||
const r = scavengeToolCalls(reasoning, { allowedNames: allowed, maxCalls: 2 });
|
||||
expect(r.calls.length).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,46 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { StormBreaker } from "../../src/repair/storm.js";
|
||||
import type { ToolCall } from "../../src/types.js";
|
||||
|
||||
function call(name: string, args: string): ToolCall {
|
||||
return { function: { name, arguments: args } };
|
||||
}
|
||||
|
||||
describe("StormBreaker", () => {
|
||||
it("passes through below threshold", () => {
|
||||
const sb = new StormBreaker(6, 3);
|
||||
expect(sb.inspect(call("x", "{}")).suppress).toBe(false);
|
||||
expect(sb.inspect(call("x", "{}")).suppress).toBe(false);
|
||||
});
|
||||
|
||||
it("suppresses on threshold reached", () => {
|
||||
const sb = new StormBreaker(6, 3);
|
||||
sb.inspect(call("x", "{}"));
|
||||
sb.inspect(call("x", "{}"));
|
||||
const verdict = sb.inspect(call("x", "{}"));
|
||||
expect(verdict.suppress).toBe(true);
|
||||
expect(verdict.reason).toMatch(/call-storm/);
|
||||
});
|
||||
|
||||
it("distinguishes different args as different calls", () => {
|
||||
const sb = new StormBreaker(6, 3);
|
||||
sb.inspect(call("x", '{"a":1}'));
|
||||
sb.inspect(call("x", '{"a":2}'));
|
||||
sb.inspect(call("x", '{"a":3}'));
|
||||
// different args each time — not a storm
|
||||
const verdict = sb.inspect(call("x", '{"a":4}'));
|
||||
expect(verdict.suppress).toBe(false);
|
||||
});
|
||||
|
||||
it("forgets old calls beyond window", () => {
|
||||
const sb = new StormBreaker(3, 3);
|
||||
sb.inspect(call("x", "{}"));
|
||||
sb.inspect(call("x", "{}"));
|
||||
sb.inspect(call("y", "{}"));
|
||||
sb.inspect(call("z", "{}"));
|
||||
sb.inspect(call("w", "{}"));
|
||||
// only the most recent 3 are in the window now, none of which is "x",
|
||||
// so a single new "x" should not suppress.
|
||||
expect(sb.inspect(call("x", "{}")).suppress).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,44 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { repairTruncatedJson } from "../../src/repair/truncation.js";
|
||||
|
||||
describe("repairTruncatedJson", () => {
|
||||
it("returns parseable JSON unchanged", () => {
|
||||
const r = repairTruncatedJson('{"a":1}');
|
||||
expect(r.changed).toBe(false);
|
||||
expect(r.repaired).toBe('{"a":1}');
|
||||
});
|
||||
|
||||
it("closes unbalanced braces", () => {
|
||||
const r = repairTruncatedJson('{"a":1');
|
||||
expect(r.changed).toBe(true);
|
||||
expect(() => JSON.parse(r.repaired)).not.toThrow();
|
||||
});
|
||||
|
||||
it("closes nested unbalanced structures", () => {
|
||||
const r = repairTruncatedJson('{"a":{"b":[1,2');
|
||||
expect(() => JSON.parse(r.repaired)).not.toThrow();
|
||||
});
|
||||
|
||||
it("closes unterminated string", () => {
|
||||
const r = repairTruncatedJson('{"a":"he');
|
||||
expect(() => JSON.parse(r.repaired)).not.toThrow();
|
||||
expect(JSON.parse(r.repaired).a.startsWith("he")).toBe(true);
|
||||
});
|
||||
|
||||
it("fills dangling key with null", () => {
|
||||
const r = repairTruncatedJson('{"a":');
|
||||
expect(() => JSON.parse(r.repaired)).not.toThrow();
|
||||
expect(JSON.parse(r.repaired)).toEqual({ a: null });
|
||||
});
|
||||
|
||||
it("handles empty input", () => {
|
||||
const r = repairTruncatedJson("");
|
||||
expect(r.repaired).toBe("{}");
|
||||
});
|
||||
|
||||
it("drops trailing comma", () => {
|
||||
const r = repairTruncatedJson('{"a":1,');
|
||||
expect(() => JSON.parse(r.repaired)).not.toThrow();
|
||||
expect(JSON.parse(r.repaired)).toEqual({ a: 1 });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,137 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { computeCumulativeStats, computeReplayStats, groupRecordsByTurn } from "../src/replay.js";
|
||||
import type { TranscriptRecord } from "../src/transcript.js";
|
||||
|
||||
const mkAssistant = (
|
||||
turn: number,
|
||||
hit: number,
|
||||
miss: number,
|
||||
completion: number,
|
||||
cost: number,
|
||||
prefixHash = "stable123",
|
||||
): TranscriptRecord => ({
|
||||
ts: "2026-04-21T00:00:00Z",
|
||||
turn,
|
||||
role: "assistant_final",
|
||||
content: `reply ${turn}`,
|
||||
model: "deepseek-chat",
|
||||
prefixHash,
|
||||
usage: {
|
||||
prompt_tokens: hit + miss,
|
||||
completion_tokens: completion,
|
||||
total_tokens: hit + miss + completion,
|
||||
prompt_cache_hit_tokens: hit,
|
||||
prompt_cache_miss_tokens: miss,
|
||||
},
|
||||
cost,
|
||||
});
|
||||
|
||||
describe("computeReplayStats", () => {
|
||||
it("aggregates cache-hit and cost across assistant_final records", () => {
|
||||
const recs: TranscriptRecord[] = [
|
||||
{ ts: "t", turn: 1, role: "user", content: "q1" },
|
||||
mkAssistant(1, 900, 100, 50, 0.0001),
|
||||
{ ts: "t", turn: 1, role: "tool", content: "{}", tool: "foo", args: "{}" },
|
||||
{ ts: "t", turn: 2, role: "user", content: "q2" },
|
||||
mkAssistant(2, 950, 50, 30, 0.00008),
|
||||
];
|
||||
const stats = computeReplayStats(recs);
|
||||
expect(stats.turns).toBe(2);
|
||||
expect(stats.userTurns).toBe(2);
|
||||
expect(stats.toolCalls).toBe(1);
|
||||
// cache: hit 1850 / (1850+150) = 92.5%
|
||||
expect(stats.cacheHitRatio).toBeCloseTo(0.925, 4);
|
||||
expect(stats.totalCostUsd).toBeCloseTo(0.00018, 6);
|
||||
expect(stats.prefixHashes).toEqual(["stable123"]);
|
||||
expect(stats.models).toEqual(["deepseek-chat"]);
|
||||
});
|
||||
|
||||
it("detects prefix churn when multiple hashes appear (baseline-style transcript)", () => {
|
||||
const recs: TranscriptRecord[] = [
|
||||
mkAssistant(1, 100, 900, 50, 0.0003, "hashA"),
|
||||
mkAssistant(2, 100, 900, 50, 0.0003, "hashB"),
|
||||
mkAssistant(3, 100, 900, 50, 0.0003, "hashC"),
|
||||
];
|
||||
const stats = computeReplayStats(recs);
|
||||
expect(stats.prefixHashes).toHaveLength(3);
|
||||
expect(stats.cacheHitRatio).toBeCloseTo(0.1, 2);
|
||||
});
|
||||
|
||||
it("tolerates old transcripts without usage — produces zero-cost stats gracefully", () => {
|
||||
const recs: TranscriptRecord[] = [
|
||||
{ ts: "t", turn: 1, role: "user", content: "q" },
|
||||
{ ts: "t", turn: 1, role: "assistant_final", content: "a" },
|
||||
];
|
||||
const stats = computeReplayStats(recs);
|
||||
expect(stats.turns).toBe(0); // no usage → no perTurn entries → turns count is 0
|
||||
expect(stats.userTurns).toBe(1);
|
||||
expect(stats.totalCostUsd).toBe(0);
|
||||
expect(stats.cacheHitRatio).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("groupRecordsByTurn (replay nav)", () => {
|
||||
it("groups records by turn and preserves in-turn order", () => {
|
||||
const recs: TranscriptRecord[] = [
|
||||
{ ts: "t", turn: 1, role: "user", content: "q1" },
|
||||
mkAssistant(1, 100, 0, 10, 0),
|
||||
{ ts: "t", turn: 1, role: "tool", content: "r", tool: "foo", args: "{}" },
|
||||
{ ts: "t", turn: 2, role: "user", content: "q2" },
|
||||
mkAssistant(2, 100, 0, 10, 0),
|
||||
];
|
||||
const pages = groupRecordsByTurn(recs);
|
||||
expect(pages).toHaveLength(2);
|
||||
expect(pages[0]!.turn).toBe(1);
|
||||
expect(pages[0]!.records).toHaveLength(3);
|
||||
expect(pages[0]!.records[0]!.role).toBe("user");
|
||||
expect(pages[0]!.records[2]!.role).toBe("tool");
|
||||
expect(pages[1]!.turn).toBe(2);
|
||||
expect(pages[1]!.records).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("returns pages sorted by turn even if records appear out of order", () => {
|
||||
const recs: TranscriptRecord[] = [
|
||||
{ ts: "t", turn: 3, role: "user", content: "c" },
|
||||
{ ts: "t", turn: 1, role: "user", content: "a" },
|
||||
{ ts: "t", turn: 2, role: "user", content: "b" },
|
||||
];
|
||||
const pages = groupRecordsByTurn(recs);
|
||||
expect(pages.map((p) => p.turn)).toEqual([1, 2, 3]);
|
||||
});
|
||||
|
||||
it("handles an empty record list", () => {
|
||||
expect(groupRecordsByTurn([])).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("computeCumulativeStats (replay nav)", () => {
|
||||
it("grows monotonically as the cursor advances", () => {
|
||||
const pages = groupRecordsByTurn([
|
||||
mkAssistant(1, 100, 900, 10, 0.001),
|
||||
mkAssistant(2, 200, 800, 10, 0.002),
|
||||
mkAssistant(3, 300, 700, 10, 0.003),
|
||||
]);
|
||||
const s0 = computeCumulativeStats(pages, 0);
|
||||
const s1 = computeCumulativeStats(pages, 1);
|
||||
const s2 = computeCumulativeStats(pages, 2);
|
||||
expect(s0.turns).toBe(1);
|
||||
expect(s1.turns).toBe(2);
|
||||
expect(s2.turns).toBe(3);
|
||||
expect(s0.totalCostUsd).toBeLessThan(s1.totalCostUsd);
|
||||
expect(s1.totalCostUsd).toBeLessThan(s2.totalCostUsd);
|
||||
});
|
||||
|
||||
it("returns empty stats for upToIdx < 0", () => {
|
||||
const pages = groupRecordsByTurn([mkAssistant(1, 100, 900, 10, 0.001)]);
|
||||
const s = computeCumulativeStats(pages, -1);
|
||||
expect(s.turns).toBe(0);
|
||||
expect(s.totalCostUsd).toBe(0);
|
||||
expect(s.prefixHashes).toEqual([]);
|
||||
});
|
||||
|
||||
it("clamps gracefully when upToIdx exceeds page count", () => {
|
||||
const pages = groupRecordsByTurn([mkAssistant(1, 100, 900, 10, 0.001)]);
|
||||
const s = computeCumulativeStats(pages, 99);
|
||||
expect(s.turns).toBe(1); // only the one real page contributes
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,126 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { fetchWithRetry } from "../src/retry.js";
|
||||
|
||||
function makeFetch(responses: Array<Response | Error | (() => Response | Error)>): {
|
||||
fn: typeof fetch;
|
||||
calls: number;
|
||||
} {
|
||||
let i = 0;
|
||||
const fn = vi.fn(async () => {
|
||||
const r = responses[i++] ?? responses[responses.length - 1]!;
|
||||
const resolved = typeof r === "function" ? r() : r;
|
||||
if (resolved instanceof Error) throw resolved;
|
||||
return resolved;
|
||||
}) as unknown as typeof fetch;
|
||||
return {
|
||||
fn,
|
||||
get calls() {
|
||||
return (fn as any).mock.calls.length as number;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const BASE: Parameters<typeof fetchWithRetry>[3] = { initialBackoffMs: 1, maxBackoffMs: 50 };
|
||||
|
||||
describe("fetchWithRetry", () => {
|
||||
it("returns immediately on success", async () => {
|
||||
const f = makeFetch([new Response("ok", { status: 200 })]);
|
||||
const r = await fetchWithRetry(f.fn, "https://x", {}, BASE);
|
||||
expect(r.status).toBe(200);
|
||||
expect(f.calls).toBe(1);
|
||||
});
|
||||
|
||||
it("retries on 429 and succeeds", async () => {
|
||||
const f = makeFetch([
|
||||
new Response("", { status: 429 }),
|
||||
new Response("", { status: 429 }),
|
||||
new Response("ok", { status: 200 }),
|
||||
]);
|
||||
const r = await fetchWithRetry(f.fn, "https://x", {}, BASE);
|
||||
expect(r.status).toBe(200);
|
||||
expect(f.calls).toBe(3);
|
||||
});
|
||||
|
||||
it("retries on 503 and eventually surfaces the failure", async () => {
|
||||
const f = makeFetch([
|
||||
new Response("", { status: 503 }),
|
||||
new Response("", { status: 503 }),
|
||||
new Response("", { status: 503 }),
|
||||
]);
|
||||
const r = await fetchWithRetry(f.fn, "https://x", {}, { ...BASE, maxAttempts: 3 });
|
||||
expect(r.status).toBe(503);
|
||||
expect(f.calls).toBe(3);
|
||||
});
|
||||
|
||||
it("does NOT retry on 401 Unauthorized", async () => {
|
||||
const f = makeFetch([new Response("bad key", { status: 401 })]);
|
||||
const r = await fetchWithRetry(f.fn, "https://x", {}, BASE);
|
||||
expect(r.status).toBe(401);
|
||||
expect(f.calls).toBe(1);
|
||||
});
|
||||
|
||||
it("does NOT retry on 400 Bad Request", async () => {
|
||||
const f = makeFetch([new Response("oops", { status: 400 })]);
|
||||
const r = await fetchWithRetry(f.fn, "https://x", {}, BASE);
|
||||
expect(r.status).toBe(400);
|
||||
expect(f.calls).toBe(1);
|
||||
});
|
||||
|
||||
it("retries on network error and recovers", async () => {
|
||||
const f = makeFetch([new TypeError("fetch failed"), new Response("ok", { status: 200 })]);
|
||||
const r = await fetchWithRetry(f.fn, "https://x", {}, BASE);
|
||||
expect(r.status).toBe(200);
|
||||
expect(f.calls).toBe(2);
|
||||
});
|
||||
|
||||
it("gives up after maxAttempts and rethrows the last network error", async () => {
|
||||
const err = new TypeError("dns lookup failed");
|
||||
const f = makeFetch([err, err, err]);
|
||||
await expect(
|
||||
fetchWithRetry(f.fn, "https://x", {}, { ...BASE, maxAttempts: 3 }),
|
||||
).rejects.toThrow(/dns lookup failed/);
|
||||
expect(f.calls).toBe(3);
|
||||
});
|
||||
|
||||
it("does NOT retry when request is aborted", async () => {
|
||||
const ctrl = new AbortController();
|
||||
const abortErr = new Error("user aborted");
|
||||
abortErr.name = "AbortError";
|
||||
const f = makeFetch([abortErr]);
|
||||
await expect(
|
||||
fetchWithRetry(f.fn, "https://x", { signal: ctrl.signal }, { ...BASE, signal: ctrl.signal }),
|
||||
).rejects.toThrow();
|
||||
expect(f.calls).toBe(1);
|
||||
});
|
||||
|
||||
it("honours Retry-After header within the cap", async () => {
|
||||
const f = makeFetch([
|
||||
new Response("", {
|
||||
status: 429,
|
||||
headers: { "Retry-After": "0.05" },
|
||||
}),
|
||||
new Response("ok", { status: 200 }),
|
||||
]);
|
||||
const started = Date.now();
|
||||
const r = await fetchWithRetry(f.fn, "https://x", {}, BASE);
|
||||
const elapsed = Date.now() - started;
|
||||
expect(r.status).toBe(200);
|
||||
// Retry-After of 0.05s = 50ms. Allow some scheduler slack.
|
||||
expect(elapsed).toBeGreaterThanOrEqual(40);
|
||||
});
|
||||
|
||||
it("fires onRetry callback between attempts", async () => {
|
||||
const onRetry = vi.fn();
|
||||
const f = makeFetch([new Response("", { status: 429 }), new Response("ok", { status: 200 })]);
|
||||
await fetchWithRetry(f.fn, "https://x", {}, { ...BASE, onRetry });
|
||||
expect(onRetry).toHaveBeenCalledTimes(1);
|
||||
expect(onRetry.mock.calls[0]![0].reason).toMatch(/http 429/);
|
||||
});
|
||||
|
||||
it("maxAttempts=1 effectively disables retry", async () => {
|
||||
const f = makeFetch([new Response("", { status: 503 })]);
|
||||
const r = await fetchWithRetry(f.fn, "https://x", {}, { ...BASE, maxAttempts: 1 });
|
||||
expect(r.status).toBe(503);
|
||||
expect(f.calls).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,106 @@
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { homedir, tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
appendSessionMessage,
|
||||
deleteSession,
|
||||
listSessions,
|
||||
loadSessionMessages,
|
||||
sanitizeName,
|
||||
sessionPath,
|
||||
sessionsDir,
|
||||
} from "../src/session.js";
|
||||
|
||||
describe("sanitizeName", () => {
|
||||
it("keeps alphanumerics, CJK, dashes, underscores", () => {
|
||||
expect(sanitizeName("hello-world_1")).toBe("hello-world_1");
|
||||
expect(sanitizeName("我的对话")).toBe("我的对话");
|
||||
});
|
||||
it("replaces other characters with underscore", () => {
|
||||
expect(sanitizeName("my/path:bad?")).toBe("my_path_bad_");
|
||||
});
|
||||
it("caps at 64 chars and defaults to 'default' when empty", () => {
|
||||
expect(sanitizeName("")).toBe("default");
|
||||
expect(sanitizeName("/:@!").length).toBeLessThanOrEqual(4);
|
||||
expect(sanitizeName("a".repeat(200))).toHaveLength(64);
|
||||
});
|
||||
});
|
||||
|
||||
describe("session persistence", () => {
|
||||
let tmp: string;
|
||||
const realHome = homedir();
|
||||
|
||||
beforeEach(() => {
|
||||
tmp = mkdtempSync(join(tmpdir(), "reasonix-session-"));
|
||||
vi.stubEnv("USERPROFILE", tmp); // Windows
|
||||
vi.stubEnv("HOME", tmp); // Unix
|
||||
// os.homedir() is cached per-process on some platforms — override via spy.
|
||||
vi.spyOn(require("node:os"), "homedir").mockReturnValue(tmp);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllEnvs();
|
||||
if (existsSync(tmp)) rmSync(tmp, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("sessionPath lives under <home>/.reasonix/sessions", () => {
|
||||
const p = sessionPath("demo");
|
||||
expect(p).toContain(".reasonix");
|
||||
expect(p).toContain("sessions");
|
||||
expect(p.endsWith("demo.jsonl")).toBe(true);
|
||||
expect(p.startsWith(tmp)).toBe(true);
|
||||
});
|
||||
|
||||
it("loadSessionMessages returns [] when the file doesn't exist", () => {
|
||||
expect(loadSessionMessages("ghost")).toEqual([]);
|
||||
});
|
||||
|
||||
it("appendSessionMessage + loadSessionMessages round-trip", () => {
|
||||
appendSessionMessage("foo", { role: "user", content: "hi" });
|
||||
appendSessionMessage("foo", { role: "assistant", content: "hello" });
|
||||
const msgs = loadSessionMessages("foo");
|
||||
expect(msgs.length).toBe(2);
|
||||
expect(msgs[0]).toEqual({ role: "user", content: "hi" });
|
||||
expect(msgs[1]).toEqual({ role: "assistant", content: "hello" });
|
||||
});
|
||||
|
||||
it("tolerates malformed lines (skips them)", () => {
|
||||
appendSessionMessage("mix", { role: "user", content: "a" });
|
||||
// inject a garbage line directly
|
||||
const p = sessionPath("mix");
|
||||
writeFileSync(p, `${readFileSync(p, "utf8")}not json\n`);
|
||||
appendSessionMessage("mix", { role: "user", content: "b" });
|
||||
const msgs = loadSessionMessages("mix");
|
||||
expect(msgs.length).toBe(2);
|
||||
});
|
||||
|
||||
it("listSessions returns metadata sorted by mtime desc", () => {
|
||||
appendSessionMessage("alpha", { role: "user", content: "x" });
|
||||
appendSessionMessage("beta", { role: "user", content: "y" });
|
||||
appendSessionMessage("beta", { role: "user", content: "z" });
|
||||
const items = listSessions();
|
||||
expect(items.length).toBe(2);
|
||||
const names = items.map((s) => s.name);
|
||||
expect(names).toContain("alpha");
|
||||
expect(names).toContain("beta");
|
||||
const beta = items.find((s) => s.name === "beta")!;
|
||||
expect(beta.messageCount).toBe(2);
|
||||
expect(beta.size).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("deleteSession removes the file", () => {
|
||||
appendSessionMessage("gone", { role: "user", content: "x" });
|
||||
expect(existsSync(sessionPath("gone"))).toBe(true);
|
||||
expect(deleteSession("gone")).toBe(true);
|
||||
expect(existsSync(sessionPath("gone"))).toBe(false);
|
||||
expect(deleteSession("gone")).toBe(false);
|
||||
});
|
||||
|
||||
it("sessionsDir exists after first append", () => {
|
||||
appendSessionMessage("s", { role: "user", content: "x" });
|
||||
expect(existsSync(sessionsDir())).toBe(true);
|
||||
expect(existsSync(dirname(sessionPath("s")))).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,176 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { handleSlash, parseSlash } from "../src/cli/ui/slash.js";
|
||||
import { DeepSeekClient } from "../src/client.js";
|
||||
import { CacheFirstLoop } from "../src/loop.js";
|
||||
import { ImmutablePrefix } from "../src/memory.js";
|
||||
|
||||
function makeLoop() {
|
||||
const client = new DeepSeekClient({
|
||||
apiKey: "sk-test",
|
||||
fetch: vi.fn() as unknown as typeof fetch,
|
||||
});
|
||||
return new CacheFirstLoop({
|
||||
client,
|
||||
prefix: new ImmutablePrefix({ system: "s" }),
|
||||
});
|
||||
}
|
||||
|
||||
describe("parseSlash", () => {
|
||||
it("returns null on non-slash input", () => {
|
||||
expect(parseSlash("hello")).toBeNull();
|
||||
expect(parseSlash("")).toBeNull();
|
||||
expect(parseSlash("/")).toBeNull();
|
||||
});
|
||||
it("lowercases the command and splits args", () => {
|
||||
expect(parseSlash("/Harvest on")).toEqual({ cmd: "harvest", args: ["on"] });
|
||||
expect(parseSlash("/branch 3")).toEqual({ cmd: "branch", args: ["3"] });
|
||||
expect(parseSlash("/help")).toEqual({ cmd: "help", args: [] });
|
||||
});
|
||||
});
|
||||
|
||||
describe("handleSlash", () => {
|
||||
it("/exit requests exit", () => {
|
||||
const loop = makeLoop();
|
||||
expect(handleSlash("exit", [], loop).exit).toBe(true);
|
||||
expect(handleSlash("quit", [], loop).exit).toBe(true);
|
||||
});
|
||||
|
||||
it("/clear requests history clear", () => {
|
||||
expect(handleSlash("clear", [], makeLoop()).clear).toBe(true);
|
||||
});
|
||||
|
||||
it("/help returns a multi-line message", () => {
|
||||
const r = handleSlash("help", [], makeLoop());
|
||||
expect(r.info).toMatch(/\/status/);
|
||||
expect(r.info).toMatch(/\/harvest/);
|
||||
expect(r.info).toMatch(/\/branch/);
|
||||
});
|
||||
|
||||
it("/status reflects current loop config", () => {
|
||||
const loop = makeLoop();
|
||||
const r = handleSlash("status", [], loop);
|
||||
expect(r.info).toMatch(/model=/);
|
||||
expect(r.info).toMatch(/harvest=off/);
|
||||
expect(r.info).toMatch(/branch=off/);
|
||||
});
|
||||
|
||||
it("/model switches the model", () => {
|
||||
const loop = makeLoop();
|
||||
handleSlash("model", ["deepseek-reasoner"], loop);
|
||||
expect(loop.model).toBe("deepseek-reasoner");
|
||||
});
|
||||
|
||||
it("/harvest on/off toggles", () => {
|
||||
const loop = makeLoop();
|
||||
handleSlash("harvest", ["on"], loop);
|
||||
expect(loop.harvestEnabled).toBe(true);
|
||||
handleSlash("harvest", ["off"], loop);
|
||||
expect(loop.harvestEnabled).toBe(false);
|
||||
});
|
||||
|
||||
it("/harvest with no arg toggles the current state", () => {
|
||||
const loop = makeLoop();
|
||||
expect(loop.harvestEnabled).toBe(false);
|
||||
handleSlash("harvest", [], loop);
|
||||
expect(loop.harvestEnabled).toBe(true);
|
||||
handleSlash("harvest", [], loop);
|
||||
expect(loop.harvestEnabled).toBe(false);
|
||||
});
|
||||
|
||||
it("/branch N enables branching and force-enables harvest + disables stream", () => {
|
||||
const loop = makeLoop();
|
||||
expect(loop.stream).toBe(true);
|
||||
expect(loop.harvestEnabled).toBe(false);
|
||||
handleSlash("branch", ["3"], loop);
|
||||
expect(loop.branchOptions.budget).toBe(3);
|
||||
expect(loop.branchEnabled).toBe(true);
|
||||
expect(loop.harvestEnabled).toBe(true);
|
||||
expect(loop.stream).toBe(false);
|
||||
});
|
||||
|
||||
it("/branch off disables branching and restores stream preference", () => {
|
||||
const loop = makeLoop();
|
||||
handleSlash("branch", ["3"], loop);
|
||||
handleSlash("branch", ["off"], loop);
|
||||
expect(loop.branchEnabled).toBe(false);
|
||||
expect(loop.stream).toBe(true);
|
||||
});
|
||||
|
||||
it("/branch rejects invalid N", () => {
|
||||
const loop = makeLoop();
|
||||
const r = handleSlash("branch", ["abc"], loop);
|
||||
expect(r.info).toMatch(/usage/);
|
||||
expect(loop.branchEnabled).toBe(false);
|
||||
});
|
||||
|
||||
it("/branch caps at 8", () => {
|
||||
const loop = makeLoop();
|
||||
const r = handleSlash("branch", ["99"], loop);
|
||||
expect(r.info).toMatch(/capped/);
|
||||
expect(loop.branchEnabled).toBe(false);
|
||||
});
|
||||
|
||||
it("unknown commands return an unknown flag with hint", () => {
|
||||
const r = handleSlash("nope", [], makeLoop());
|
||||
expect(r.unknown).toBe(true);
|
||||
expect(r.info).toMatch(/unknown command/);
|
||||
});
|
||||
|
||||
it("/preset fast = deepseek-chat, no harvest, no branch", () => {
|
||||
const loop = makeLoop();
|
||||
handleSlash("model", ["deepseek-reasoner"], loop);
|
||||
handleSlash("harvest", ["on"], loop);
|
||||
handleSlash("branch", ["3"], loop);
|
||||
handleSlash("preset", ["fast"], loop);
|
||||
expect(loop.model).toBe("deepseek-chat");
|
||||
expect(loop.harvestEnabled).toBe(false);
|
||||
expect(loop.branchEnabled).toBe(false);
|
||||
});
|
||||
|
||||
it("/preset smart = reasoner + harvest, no branch", () => {
|
||||
const loop = makeLoop();
|
||||
handleSlash("preset", ["smart"], loop);
|
||||
expect(loop.model).toBe("deepseek-reasoner");
|
||||
expect(loop.harvestEnabled).toBe(true);
|
||||
expect(loop.branchEnabled).toBe(false);
|
||||
});
|
||||
|
||||
it("/preset max = reasoner + harvest + branch3", () => {
|
||||
const loop = makeLoop();
|
||||
handleSlash("preset", ["max"], loop);
|
||||
expect(loop.model).toBe("deepseek-reasoner");
|
||||
expect(loop.harvestEnabled).toBe(true);
|
||||
expect(loop.branchOptions.budget).toBe(3);
|
||||
});
|
||||
|
||||
it("/preset with bad name returns usage", () => {
|
||||
const r = handleSlash("preset", ["nonsense"], makeLoop());
|
||||
expect(r.info).toMatch(/usage/);
|
||||
});
|
||||
|
||||
it("/help mentions presets", () => {
|
||||
const r = handleSlash("help", [], makeLoop());
|
||||
expect(r.info).toMatch(/Presets:/);
|
||||
expect(r.info).toMatch(/fast/);
|
||||
expect(r.info).toMatch(/smart/);
|
||||
expect(r.info).toMatch(/max/);
|
||||
});
|
||||
|
||||
it("/help mentions sessions", () => {
|
||||
const r = handleSlash("help", [], makeLoop());
|
||||
expect(r.info).toMatch(/\/sessions/);
|
||||
expect(r.info).toMatch(/\/forget/);
|
||||
});
|
||||
|
||||
it("/sessions returns a hint when none exist", () => {
|
||||
const r = handleSlash("sessions", [], makeLoop());
|
||||
expect(r.info).toMatch(/no saved sessions yet|Saved sessions/);
|
||||
});
|
||||
|
||||
it("/forget on a session-less loop says nothing to forget", () => {
|
||||
const loop = makeLoop();
|
||||
expect(loop.sessionName).toBeNull();
|
||||
const r = handleSlash("forget", [], loop);
|
||||
expect(r.info).toMatch(/nothing to forget/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,45 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { Usage } from "../src/client.js";
|
||||
import { SessionStats, costUsd } from "../src/telemetry.js";
|
||||
|
||||
describe("Usage.cacheHitRatio", () => {
|
||||
it("computes hit ratio", () => {
|
||||
const u = new Usage(0, 0, 0, 80, 20);
|
||||
expect(u.cacheHitRatio).toBe(0.8);
|
||||
});
|
||||
it("is zero on empty", () => {
|
||||
expect(new Usage().cacheHitRatio).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("costUsd", () => {
|
||||
it("applies DeepSeek pricing tiers", () => {
|
||||
const u = new Usage(1000, 100, 0, 800, 200);
|
||||
const c = costUsd("deepseek-chat", u);
|
||||
// (800 * 0.07 + 200 * 0.27 + 100 * 1.10) / 1e6
|
||||
expect(c).toBeCloseTo((800 * 0.07 + 200 * 0.27 + 100 * 1.1) / 1_000_000, 10);
|
||||
});
|
||||
|
||||
it("returns 0 for unknown model", () => {
|
||||
expect(costUsd("unknown-model", new Usage(1000, 100))).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("SessionStats", () => {
|
||||
it("aggregates savings vs Claude", () => {
|
||||
const stats = new SessionStats();
|
||||
stats.record(1, "deepseek-chat", new Usage(1000, 100, 1100, 800, 200));
|
||||
const s = stats.summary();
|
||||
expect(s.turns).toBe(1);
|
||||
expect(s.cacheHitRatio).toBe(0.8);
|
||||
expect(s.savingsVsClaudePct).toBeGreaterThan(90);
|
||||
});
|
||||
|
||||
it("accumulates across turns", () => {
|
||||
const stats = new SessionStats();
|
||||
stats.record(1, "deepseek-chat", new Usage(100, 10, 110, 80, 20));
|
||||
stats.record(2, "deepseek-chat", new Usage(200, 20, 220, 160, 40));
|
||||
expect(stats.turns.length).toBe(2);
|
||||
expect(stats.aggregateCacheHitRatio).toBeCloseTo(240 / 300);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,166 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { ToolRegistry } from "../src/tools.js";
|
||||
|
||||
describe("ToolRegistry", () => {
|
||||
it("registers and dispatches a tool with JSON args", async () => {
|
||||
const reg = new ToolRegistry();
|
||||
reg.register<{ a: number; b: number }, number>({
|
||||
name: "add",
|
||||
description: "add two ints",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: { a: { type: "integer" }, b: { type: "integer" } },
|
||||
required: ["a", "b"],
|
||||
},
|
||||
fn: ({ a, b }) => a + b,
|
||||
});
|
||||
expect(reg.has("add")).toBe(true);
|
||||
const result = await reg.dispatch("add", '{"a":2,"b":3}');
|
||||
expect(result).toBe("5");
|
||||
});
|
||||
|
||||
it("returns structured error for unknown tool", async () => {
|
||||
const reg = new ToolRegistry();
|
||||
const out = await reg.dispatch("nope", "{}");
|
||||
expect(JSON.parse(out)).toEqual({ error: "unknown tool: nope" });
|
||||
});
|
||||
|
||||
it("handles invalid JSON arguments gracefully", async () => {
|
||||
const reg = new ToolRegistry();
|
||||
reg.register({ name: "noop", fn: () => "ok" });
|
||||
const out = await reg.dispatch("noop", "{bad json");
|
||||
expect(JSON.parse(out).error).toMatch(/invalid tool arguments JSON/);
|
||||
});
|
||||
|
||||
it("emits OpenAI-shaped specs", () => {
|
||||
const reg = new ToolRegistry();
|
||||
reg.register({
|
||||
name: "echo",
|
||||
description: "echo input",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: { msg: { type: "string" } },
|
||||
required: ["msg"],
|
||||
},
|
||||
fn: ({ msg }: { msg: string }) => msg,
|
||||
});
|
||||
const spec = reg.specs()[0]!;
|
||||
expect(spec.type).toBe("function");
|
||||
expect(spec.function.name).toBe("echo");
|
||||
expect(spec.function.parameters.required).toEqual(["msg"]);
|
||||
});
|
||||
|
||||
it("does NOT flatten shallow schemas", () => {
|
||||
const reg = new ToolRegistry();
|
||||
reg.register({
|
||||
name: "shallow",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: { a: { type: "string" }, b: { type: "number" } },
|
||||
required: ["a"],
|
||||
},
|
||||
fn: () => "ok",
|
||||
});
|
||||
expect(reg.wasFlattened("shallow")).toBe(false);
|
||||
expect(reg.specs()[0]!.function.parameters.properties).toHaveProperty("a");
|
||||
});
|
||||
|
||||
it("auto-flattens deep schemas and re-nests args on dispatch", async () => {
|
||||
const reg = new ToolRegistry();
|
||||
let received: any = null;
|
||||
reg.register({
|
||||
name: "deep",
|
||||
parameters: {
|
||||
type: "object",
|
||||
required: ["user"],
|
||||
properties: {
|
||||
user: {
|
||||
type: "object",
|
||||
required: ["profile"],
|
||||
properties: {
|
||||
profile: {
|
||||
type: "object",
|
||||
required: ["name"],
|
||||
properties: {
|
||||
name: { type: "string" },
|
||||
age: { type: "integer" },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
fn: (args: any) => {
|
||||
received = args;
|
||||
return "ok";
|
||||
},
|
||||
});
|
||||
|
||||
expect(reg.wasFlattened("deep")).toBe(true);
|
||||
const spec = reg.specs()[0]!;
|
||||
expect(spec.function.parameters.properties).toHaveProperty("user.profile.name");
|
||||
expect(spec.function.parameters.properties).toHaveProperty("user.profile.age");
|
||||
|
||||
// Model emits flat dot-notation args (as it would after seeing the flat spec).
|
||||
await reg.dispatch("deep", '{"user.profile.name":"alice","user.profile.age":30}');
|
||||
expect(received).toEqual({ user: { profile: { name: "alice", age: 30 } } });
|
||||
});
|
||||
|
||||
it("auto-flattens wide schemas (>10 leaf params)", () => {
|
||||
const reg = new ToolRegistry();
|
||||
const props: Record<string, { type: string }> = {};
|
||||
for (let i = 0; i < 15; i++) props[`p${i}`] = { type: "string" };
|
||||
reg.register({
|
||||
name: "wide",
|
||||
parameters: { type: "object", properties: props },
|
||||
fn: () => "ok",
|
||||
});
|
||||
expect(reg.wasFlattened("wide")).toBe(true);
|
||||
});
|
||||
|
||||
it("dispatch passes through nested args even when tool was flattened (defensive)", async () => {
|
||||
const reg = new ToolRegistry();
|
||||
let received: any = null;
|
||||
reg.register({
|
||||
name: "deep",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: {
|
||||
a: {
|
||||
type: "object",
|
||||
properties: {
|
||||
b: { type: "object", properties: { c: { type: "string" } } },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
fn: (args: any) => {
|
||||
received = args;
|
||||
return "ok";
|
||||
},
|
||||
});
|
||||
expect(reg.wasFlattened("deep")).toBe(true);
|
||||
// Some models may ignore the flat spec and emit nested args anyway.
|
||||
await reg.dispatch("deep", '{"a":{"b":{"c":"hi"}}}');
|
||||
expect(received).toEqual({ a: { b: { c: "hi" } } });
|
||||
});
|
||||
|
||||
it("autoFlatten:false opts out", () => {
|
||||
const reg = new ToolRegistry({ autoFlatten: false });
|
||||
reg.register({
|
||||
name: "deep",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: {
|
||||
a: {
|
||||
type: "object",
|
||||
properties: { b: { type: "object", properties: { c: { type: "string" } } } },
|
||||
},
|
||||
},
|
||||
},
|
||||
fn: () => "ok",
|
||||
});
|
||||
expect(reg.wasFlattened("deep")).toBe(false);
|
||||
expect(reg.specs()[0]!.function.parameters.properties).toHaveProperty("a");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,114 @@
|
||||
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { Usage } from "../src/client.js";
|
||||
import type { LoopEvent } from "../src/loop.js";
|
||||
import { SessionStats } from "../src/telemetry.js";
|
||||
import {
|
||||
openTranscriptFile,
|
||||
parseTranscript,
|
||||
recordFromLoopEvent,
|
||||
writeRecord,
|
||||
} from "../src/transcript.js";
|
||||
|
||||
describe("transcript writer / reader round-trip", () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeAll(() => {
|
||||
tmpDir = mkdtempSync(join(tmpdir(), "reasonix-test-"));
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("writes meta + assistant_final + tool records and reads them back", async () => {
|
||||
const path = join(tmpDir, "round-trip.jsonl");
|
||||
const stream = openTranscriptFile(path, {
|
||||
version: 1,
|
||||
source: "test",
|
||||
model: "deepseek-chat",
|
||||
task: "t01",
|
||||
mode: "reasonix",
|
||||
repeat: 1,
|
||||
startedAt: "2026-04-21T00:00:00Z",
|
||||
});
|
||||
|
||||
// Build a realistic assistant_final event using SessionStats.
|
||||
const stats = new SessionStats();
|
||||
const usage = new Usage(1000, 100, 1100, 800, 200);
|
||||
const turnStats = stats.record(1, "deepseek-chat", usage);
|
||||
|
||||
const assistantEv: LoopEvent = {
|
||||
turn: 1,
|
||||
role: "assistant_final",
|
||||
content: "Hello world.",
|
||||
stats: turnStats,
|
||||
};
|
||||
const toolEv: LoopEvent = {
|
||||
turn: 1,
|
||||
role: "tool",
|
||||
content: '{"ok":true}',
|
||||
toolName: "add",
|
||||
toolArgs: '{"a":1,"b":2}',
|
||||
};
|
||||
|
||||
writeRecord(
|
||||
stream,
|
||||
recordFromLoopEvent(assistantEv, { model: "deepseek-chat", prefixHash: "abc123def456" }),
|
||||
);
|
||||
writeRecord(
|
||||
stream,
|
||||
recordFromLoopEvent(toolEv, { model: "deepseek-chat", prefixHash: "abc123def456" }),
|
||||
);
|
||||
await new Promise<void>((resolve) => stream.end(resolve));
|
||||
|
||||
const raw = readFileSync(path, "utf8");
|
||||
const { meta, records } = parseTranscript(raw);
|
||||
|
||||
expect(meta).not.toBeNull();
|
||||
expect(meta?.source).toBe("test");
|
||||
expect(meta?.task).toBe("t01");
|
||||
|
||||
expect(records).toHaveLength(2);
|
||||
const a = records[0]!;
|
||||
expect(a.role).toBe("assistant_final");
|
||||
expect(a.content).toBe("Hello world.");
|
||||
expect(a.usage?.prompt_cache_hit_tokens).toBe(800);
|
||||
expect(a.cost).toBeGreaterThan(0);
|
||||
expect(a.model).toBe("deepseek-chat");
|
||||
expect(a.prefixHash).toBe("abc123def456");
|
||||
|
||||
const t = records[1]!;
|
||||
expect(t.role).toBe("tool");
|
||||
expect(t.tool).toBe("add");
|
||||
expect(t.args).toBe('{"a":1,"b":2}');
|
||||
});
|
||||
|
||||
it("parses a v0.1-shape transcript (no usage / cost / prefixHash) without losing records", () => {
|
||||
// Old format — just ts/turn/role/content/tool. No optional fields.
|
||||
const raw = [
|
||||
`{"ts":"2026-04-21T00:00:00Z","turn":1,"role":"user","content":"hi"}`,
|
||||
`{"ts":"2026-04-21T00:00:01Z","turn":1,"role":"assistant_final","content":"hello"}`,
|
||||
].join("\n");
|
||||
const { meta, records } = parseTranscript(raw);
|
||||
expect(meta).toBeNull();
|
||||
expect(records).toHaveLength(2);
|
||||
expect(records[0]!.role).toBe("user");
|
||||
expect(records[1]!.role).toBe("assistant_final");
|
||||
expect(records[1]!.usage).toBeUndefined();
|
||||
expect(records[1]!.cost).toBeUndefined();
|
||||
});
|
||||
|
||||
it("skips malformed lines silently (partial writes shouldn't crash replay)", () => {
|
||||
const raw = [
|
||||
`{"ts":"2026-04-21T00:00:00Z","turn":1,"role":"user","content":"hi"}`,
|
||||
"{not valid json",
|
||||
`{"wrong":"shape"}`,
|
||||
`{"ts":"2026-04-21T00:00:01Z","turn":1,"role":"assistant_final","content":"yo"}`,
|
||||
].join("\n");
|
||||
const { records } = parseTranscript(raw);
|
||||
expect(records).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"lib": ["ES2023"],
|
||||
"jsx": "react",
|
||||
"esModuleInterop": true,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"strict": true,
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"noImplicitOverride": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist", "tests", "examples", "benchmarks"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import { defineConfig } from "tsup";
|
||||
|
||||
export default defineConfig([
|
||||
{
|
||||
entry: ["src/index.ts"],
|
||||
format: ["esm"],
|
||||
dts: true,
|
||||
clean: true,
|
||||
sourcemap: true,
|
||||
target: "node20",
|
||||
outDir: "dist",
|
||||
},
|
||||
{
|
||||
entry: ["src/cli/index.ts"],
|
||||
format: ["esm"],
|
||||
dts: false,
|
||||
clean: false,
|
||||
sourcemap: true,
|
||||
target: "node20",
|
||||
outDir: "dist/cli",
|
||||
banner: { js: "#!/usr/bin/env node" },
|
||||
},
|
||||
]);
|
||||
@@ -0,0 +1,15 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
include: ["tests/**/*.test.ts"],
|
||||
environment: "node",
|
||||
globals: false,
|
||||
coverage: {
|
||||
provider: "v8",
|
||||
reporter: ["text", "html"],
|
||||
include: ["src/**"],
|
||||
exclude: ["src/cli/ui/**", "src/**/*.test.ts"],
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -1,5 +0,0 @@
|
||||
node_modules/
|
||||
.wrangler/
|
||||
dist/
|
||||
# Local-only secrets for `wrangler dev` (RESEND_API_KEY, SESSION_PEPPER, ...)
|
||||
.dev.vars
|
||||
@@ -1,169 +0,0 @@
|
||||
# reasonix-accounts
|
||||
|
||||
The account service for `reasonix.io`: email/password sign-up, email verification,
|
||||
sessions, password reset, and public profiles. A Cloudflare Worker (Hono) backed
|
||||
by D1. Runs at `id.reasonix.io`, separate from the internal crash dashboard.
|
||||
|
||||
This is the backend API only — there are no HTML pages. The web frontend (and,
|
||||
later, the desktop/CLI) call these JSON endpoints.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
src/
|
||||
index.ts entry — exports the Hono app as the fetch handler
|
||||
app.ts middleware + route wiring
|
||||
env.ts types.ts config.ts
|
||||
auth/ crypto.ts (PBKDF2 + token hashing) cookies.ts (session cookie)
|
||||
db/ users.ts sessions.ts emailTokens.ts deviceGrants.ts index.ts (repos factory)
|
||||
email/ index.ts (Mailer + templates) resend.ts types.ts
|
||||
http/ errors.ts cors.ts auth.ts (cookie + Bearer session) ratelimit.ts
|
||||
lib/ validation.ts (zod) handle.ts
|
||||
routes/ auth.ts device.ts me.ts users.ts health.ts
|
||||
```
|
||||
|
||||
Design notes:
|
||||
|
||||
- **Sessions store `sha256(pepper:token)`** — the raw token only ever lives in the
|
||||
cookie (web) or the client's credential store (CLI/desktop), so a DB read can't
|
||||
resurrect a live session. Protected routes accept the session from the `rxid`
|
||||
cookie or an `Authorization: Bearer <token>` header, so the same table serves
|
||||
both surfaces (`sessions.kind` = `web` | `cli`).
|
||||
- **Device sign-in (RFC 8628-style)** lets the CLI/desktop authenticate without a
|
||||
browser redirect: `/device/start` issues a `device_code` (polled) and a short
|
||||
`user_code` (the human approves it on `APP_ORIGIN/device` while signed in). Only
|
||||
the device code's peppered hash is stored; the `cli` session token is minted on
|
||||
the winning poll (an atomic `DELETE … RETURNING` claim), so it never lands in the
|
||||
DB. Polling isn't IP-limited — a `slow_down` hint plus the 10-minute TTL bound it.
|
||||
- **`password_hash` is nullable** on `users` so OAuth-only identities can be added
|
||||
later without a rebuild.
|
||||
- **Registration is enumeration-safe**: the response never reveals whether an email
|
||||
already exists; login/forgot return generic messages too.
|
||||
- **PBKDF2-HMAC-SHA256, 100k iterations** — the work factor is embedded in each
|
||||
hash. 100k is Cloudflare Workers' hard cap for PBKDF2 (it rejects higher counts).
|
||||
|
||||
## Endpoints
|
||||
|
||||
| Method | Path | Auth | Notes |
|
||||
| ------ | -------------------------- | ---- | --------------------------------------- |
|
||||
| POST | `/auth/register` | — | `{ email, password, displayName? }` |
|
||||
| GET | `/auth/verify?token=` | — | email link → 302 to `APP_ORIGIN/login` |
|
||||
| POST | `/auth/login` | — | sets `rxid` cookie, returns `{ user }` |
|
||||
| POST | `/auth/logout` | — | clears session + cookie |
|
||||
| POST | `/auth/forgot` | — | `{ email }` → reset link |
|
||||
| POST | `/auth/reset` | — | `{ token, password }` |
|
||||
| POST | `/auth/resend-verification`| — | `{ email }` |
|
||||
| POST | `/device/start` | — | CLI begins sign-in → `{ deviceCode, userCode, verificationUri, interval, expiresIn }` |
|
||||
| POST | `/device/poll` | — | `{ deviceCode }` → `authorization_pending` \| `slow_down` \| `{ sessionToken, user }` |
|
||||
| GET | `/device/info?userCode=` | ✓ | approval screen: what a `user_code` will authorize |
|
||||
| POST | `/device/approve` | ✓ | `{ userCode }` — bind the pending grant to the signed-in user |
|
||||
| POST | `/device/deny` | ✓ | `{ userCode }` — reject the pending grant |
|
||||
| GET | `/me` | ✓ | the signed-in account (cookie or Bearer) |
|
||||
| PATCH | `/me` | ✓ | `{ displayName?, bio?, avatarUrl?, handle? }` |
|
||||
| POST | `/me/password` | ✓ | `{ currentPassword, newPassword }` |
|
||||
| GET | `/me/devices` | ✓ | registered remote devices with live `online` state |
|
||||
| POST | `/me/devices` | ✓ | register/rotate a device credential; optional `gatewayId` binds it to one of your gateways |
|
||||
| PATCH | `/me/devices/:id` | ✓ | rename an active device |
|
||||
| GET | `/me/gateways` | ✓ | gateways you run yourself (self-hosted relays) |
|
||||
| POST | `/me/gateways` | ✓ | register a public `https` origin; the scoped gateway `token` (what the gateway presents to this service) and a separate `callbackSecret` (what this service presents to the gateway) are returned once; only the token hash is stored (3 per account) |
|
||||
| DELETE | `/me/gateways/:id` | ✓ | remove a gateway and revoke the devices bound to it |
|
||||
| DELETE | `/me/devices/:id` | ✓ | revoke a device and close its live relay connections |
|
||||
| POST | `/me/remote-grants` | ✓ | issue a 60-second, scoped connection ticket; `403 remote_reauth_required` once the session is 24 h old or is a device-flow session |
|
||||
| GET | `/me/backups` | ✓ | list encrypted config backups + `limits` |
|
||||
| POST | `/me/backups` | ✓ | `{ label, format, appVersion, platform, categories, envelope(base64) }`; ≤ 4 MiB, ≤ 10 per account |
|
||||
| GET | `/me/backups/:id` | ✓ | `{ backup, envelope }` |
|
||||
| DELETE | `/me/backups/:id` | ✓ | delete one backup |
|
||||
| POST | `/remote/devices/authenticate` | gateway | validate a device credential |
|
||||
| POST | `/remote/grants/consume` | gateway | consume a connection ticket once |
|
||||
| POST | `/remote/leases/check` | gateway | `{ leases }` → `active` \| `revoked` \| `reauth_required` per live connection |
|
||||
| DELETE | `/me` | ✓ | soft-delete the account |
|
||||
| GET | `/u/:handle` | — | public profile |
|
||||
| GET | `/health` | — | liveness |
|
||||
|
||||
Errors are `{ "error": { "code": "...", "message": "..." } }` with a matching HTTP status.
|
||||
|
||||
## Configuration
|
||||
|
||||
`wrangler.toml` `[vars]` (non-secret): `APP_ORIGIN` (web frontend),
|
||||
`ACCOUNT_ORIGIN` (this Worker's canonical public origin), `REMOTE_GATEWAY_ORIGIN`, `ALLOWED_ORIGINS`,
|
||||
`COOKIE_DOMAIN`, `EMAIL_PROVIDER` (`stub` | `resend`), `MAIL_FROM`, `ADMIN_EMAILS`.
|
||||
|
||||
Secrets (`wrangler secret put NAME`): `SESSION_PEPPER` (any long random string),
|
||||
`RESEND_API_KEY` (only when `EMAIL_PROVIDER=resend`), and
|
||||
`REMOTE_GATEWAY_TOKEN` (shared only with the remote gateway), and
|
||||
`FEEDBACK_ACCOUNT_SECRET` (shared only with the feedback worker; without it
|
||||
`POST /me/feedback-assertion` answers 503 and deleted accounts queue their erase request).
|
||||
|
||||
`POST /me/feedback-assertion` returns a five minute, feedback-audience assertion for the signed-in
|
||||
account. `DELETE /me` records a `feedback_erasures` row and tells the feedback worker; failures never
|
||||
block the deletion. The row is kept past the six minute window in which an earlier assertion could
|
||||
still create a link, and the daily cron sends once more after it; only that later success removes it.
|
||||
|
||||
When `EMAIL_PROVIDER` isn't `resend` (or no key is set) the worker logs email links
|
||||
to the console — enough to exercise every flow locally without a mail provider.
|
||||
|
||||
## Local development
|
||||
|
||||
```sh
|
||||
pnpm install
|
||||
pnpm db:apply:local # create local D1 tables
|
||||
pnpm dev # wrangler dev on http://localhost:8787
|
||||
```
|
||||
|
||||
Put local overrides and secrets in `.dev.vars` (git-ignored), e.g.:
|
||||
|
||||
```dotenv
|
||||
ACCOUNT_ORIGIN="http://localhost:8787"
|
||||
SESSION_PEPPER="dev-pepper"
|
||||
```
|
||||
|
||||
Register a user, then read the verification link from the `wrangler dev` console.
|
||||
|
||||
## Deploy
|
||||
|
||||
```sh
|
||||
wrangler d1 create reasonix-accounts # paste database_id into wrangler.toml
|
||||
wrangler d1 migrations apply reasonix-accounts --remote
|
||||
wrangler r2 bucket create reasonix-config-backups # config backups (binding BACKUPS)
|
||||
wrangler secret put SESSION_PEPPER
|
||||
wrangler secret put RESEND_API_KEY # if EMAIL_PROVIDER=resend
|
||||
wrangler deploy
|
||||
```
|
||||
|
||||
The `id.reasonix.io` custom domain route is declared in `wrangler.toml`; point the
|
||||
DNS/custom-domain binding at this worker in the Cloudflare dashboard on first deploy.
|
||||
|
||||
The steps above are the one-time bootstrap. Production deploys are manually
|
||||
triggered from the protected `platform` branch through
|
||||
`.github/workflows/deploy-accounts-worker.yml`; that workflow validates the
|
||||
service and applies pending D1 migrations before publishing the Worker.
|
||||
|
||||
`RESEND_API_KEY` is synced to the worker on each deploy from the `RESEND_API_KEY`
|
||||
GitHub Actions repo secret (so the mail key has a single source of truth and needs
|
||||
no local wrangler auth). `SESSION_PEPPER` is not in CI — set it once with
|
||||
`wrangler secret put SESSION_PEPPER`.
|
||||
|
||||
## Controller enrollment (bookkeeping only)
|
||||
|
||||
`remote_controllers` records which browser keys proved possession of a P-256 key
|
||||
for a registered computer. Nothing consumes it yet: grants, the relay and the
|
||||
desktop do not read these rows, so it adds no protection to remote access.
|
||||
|
||||
| Method | Path | Auth | Notes |
|
||||
| ------ | -------------------------------------- | -------------------- | ----- |
|
||||
| POST | `/me/remote-controllers/challenge` | web session | `{ host }` → `{ nonce, expiresAt }`, single use, 60 s |
|
||||
| POST | `/me/remote-controllers/enroll` | web session | `{ host, nonce, publicKey, signature, uaClass, claimsId? }` |
|
||||
| GET | `/me/remote-controllers?host=` | session | the owner's rows for one computer |
|
||||
| POST | `/me/remote-controllers/:id/revoke` | session | terminal |
|
||||
| GET | `/host/controllers?state=` | device credential | pending requests with the key to verify |
|
||||
| POST | `/host/controllers/:id/resolve` | device credential | `{ action: add \| replace \| reject, replaceId? }` |
|
||||
| POST | `/host/controllers/:id/revoke` | device credential | terminal |
|
||||
|
||||
The device credential travels in `x-reasonix-device-id` and `x-reasonix-device-credential`.
|
||||
The signature is ECDSA P-256 over SHA-256 in IEEE P1363 form over the message built by
|
||||
`auth/controllerProof.ts`; its test vector is in `auth/controllerProof.vectors.ts`.
|
||||
Limits are D1 counters (`http/d1RateLimit.ts`) and refuse the request when the counter
|
||||
cannot be written. Error codes: `challenge_invalid`, `challenge_expired`, `invalid_proof`,
|
||||
`invalid_key`, `controller_id_mismatch`, `controller_unconfirmed`, `controller_revoked`,
|
||||
`controller_not_pending`, `pending_expired`, `controller_cap_reached`, `replace_target_invalid`,
|
||||
`pending_limit`, `pending_locked`, `rate_limited`, `rate_limit_unavailable`.
|
||||
Loaded 100 of 438 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user