diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8a2189c..24b1bfa 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -19,10 +19,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - name: Setup Node - uses: actions/setup-node@v4 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: 22 cache: 'npm' @@ -46,7 +46,7 @@ jobs: run: npm run build - name: Upload artifact - uses: actions/upload-pages-artifact@v3 + uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3 with: path: dist @@ -59,4 +59,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 diff --git a/.github/workflows/update-data.yml b/.github/workflows/update-data.yml index 7bca802..046ee69 100644 --- a/.github/workflows/update-data.yml +++ b/.github/workflows/update-data.yml @@ -7,16 +7,23 @@ on: permissions: contents: write + pull-requests: write + +concurrency: + group: 'data-refresh' + cancel-in-progress: false jobs: refresh-data: runs-on: ubuntu-latest steps: + # Actions are pinned by commit SHA (VULN-01): tag refs are mutable and + # a compromised upstream action would inherit `contents: write`. - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - name: Setup Node - uses: actions/setup-node@v4 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: 22 cache: 'npm' @@ -33,14 +40,41 @@ jobs: - name: Validate data invariants run: npm run validate-data - - name: Commit and push changes + # VULN-01: never auto-push third-party-derived data straight to main. + # Refreshes land on a branch as a PR so a human (or required check) + # gates anything that reaches the deployed site. + - name: Create data refresh pull request + env: + GH_TOKEN: ${{ github.token }} run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" + + BRANCH="data-refresh/$(date +%Y-%m-%d)" + git checkout -b "$BRANCH" git add public/data/ + if git diff --cached --quiet; then - echo "No data changes to commit." - else - git commit -m "chore(data): refresh model pricing and benchmarks [skip ci]" - git push + echo "No data changes to propose." + exit 0 fi + + git commit -m "chore(data): refresh model pricing and benchmarks" + git push --force origin "$BRANCH" + + # Reuse an open PR for the same branch if one exists. + EXISTING=$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number' || true) + if [ -n "$EXISTING" ]; then + gh pr comment "$EXISTING" --body "Automated data refresh updated (run ${GITHUB_RUN_ID})." + exit 0 + fi + + gh pr create \ + --title "chore(data): scheduled model & benchmark refresh ($(date +%Y-%m-%d))" \ + --body "Automated daily data refresh from OpenRouter and Artificial Analysis. + + - \`validate-data\` passed (schema + price sanity bounds). + - Review the \`public/data/\` diff before merging; price anomalies are flagged by the validation bounds but cannot be fully ruled out. + - Merging this PR triggers the Pages deploy." \ + --base main \ + --head "$BRANCH" diff --git a/data/estimate-constants.json b/data/estimate-constants.json index 017e343..051e358 100644 --- a/data/estimate-constants.json +++ b/data/estimate-constants.json @@ -20,6 +20,8 @@ }, "defaultCacheRate": 0.75, "cacheWritePremium": 1.25, + "_cacheWriteShare": "Fraction of the cached context rewritten per turn (VULN-14: cold-prefix writes at the cache-write premium were never charged). 6% matches the empirical fresh-context share of the 20K-in agentic request being written to cache each turn.", + "cacheWriteShare": 0.06, "timeOfDayBlend": { "_comment": "Time-of-day volume blend for providers with off-peak discounts (e.g. DeepSeek, Z.ai). Default 80% off-peak, 20% peak.", "offPeakShare": 0.8, @@ -29,5 +31,15 @@ "chat": 0.5, "standard": 0.75, "deepAgent": 0.92 + }, + "_sessionCacheModel": "Multi-turn session cache model (WorkflowCalculator): of the tokens re-read within one session, this residual share re-pays full input price due to cache invalidations (tool outputs, file edits).", + "sessionCacheBustResidual": 0.15, + "_providerCacheReadFallback": "Documented cache-read ratios used ONLY when the upstream API omits cache prices (scripts/fetch-models.mjs). Keys are lowercase provider/id substrings.", + "providerCacheReadFallback": { + "anthropic": 0.10, + "deepseek": 0.10, + "z-ai": 0.10, + "google": 0.25, + "openai": 0.50 } } diff --git a/docs/VERIFICATION.md b/docs/VERIFICATION.md index 6f7bf57..9c75e0f 100644 --- a/docs/VERIFICATION.md +++ b/docs/VERIFICATION.md @@ -456,3 +456,77 @@ and `node scripts/build-data.mjs`. - `npm test` — 109/109 tests pass across 8 test suites. - `npm run build` — Clean production build. + +--- + +## Adversarial Audit Remediation Pass (September 20, 2026) + +Full hostile audit (15 findings, VULN-01 through VULN-15) remediated in one pass. + +### Summary of Fixes + +1. **VULN-01 (Critical) — Data pipeline supply chain**: `update-data.yml` no longer auto-pushes + third-party-derived data to `main`; refreshes open a PR (branch `data-refresh/`) for human + review. All GitHub Actions pinned by commit SHA. `validate-data.mjs` gained price sanity bounds + (non-free models must have positive finite in/out prices; ceilings $1,000/$2,000 per M; cachedInput + must be null or finite ≥ 0). +2. **VULN-02 (High) — Cache scaling on metered quotas**: `computeApplesToApples` now applies + symmetric cache scaling **only to dollar-credit pools** (`isDollarPoolTier`: `limits` containing a + credit key or `perModelTokenBudgets` on a credit/equal-rate basis). Usage-metered quotas deliver + fixed tokens regardless of the user's cache rate and are never scaled. +3. **VULN-03 (High) — payg-overage never throttled**: `simulateSprintThrottle` now emits a distinct + `overage` status for `payg-overage` profiles (Google AI Pro, OpenCode Zen, DeepSeek PAYG) once a + window/monthly ceiling is crossed; `SimulationResult.overageTurns` reports billed excess; overall + status resolves by severity (`smooth < queued < overage < blocked`). BurstSimulator gains an + Overage filter, badge, chart color, and stat column. +4. **VULN-04 (High) — Hardware feasibility & payback basis**: `computeHardwareEconomics` derives + `requiredInferenceHours` from the preset's flagship decode throughput and reports + `infeasibleWorkload` / `feasibleMonthlyVolumeM`. Payback is only quoted when the amortized monthly + alpha is positive, so headline savings and payback cannot contradict. HardwareBreakeven renders + the infeasibility warning with the required-vs-available decode hours. +5. **VULN-05 (High) — Synthetic receipts**: `ParsedAgentSession.measuredTokens` distinguishes + measured from parser-fabricated token counts; `SessionReceiptReport.tokensMeasured` propagates it; + the receipt page and copied receipt both flag estimated sessions; the headline prefix + "ESTIMATED …" prevents synthetic numbers passing as a bill. +6. **VULN-06 (Medium) — Impossible reasoning combos**: `calculateReasoningCost` checks + input+reasoning+output against `contextWindow` and reasoning+output against `maxOutput` (unknown + caps treated as feasible); ReasoningExploder disables infeasible effort pills and renders the + impossibility reason. +7. **VULN-07 (Medium) — Cache-assumption regex hijack**: `parsePlanCacheAssumption` anchors the + percent to cache context (`NN% cache|hit` or `cache … NN%`), so discount percentages cannot be + misparsed as cache rates; results clamp to ≤ 0.95. +8. **VULN-08 (Medium) — Divergent cache heuristics**: the ad-hoc `(1 − assumed×0.8)` formula for + unmatched tiers was replaced by the same model-priced ratio used for matched tiers (via a + lab-representative basis model), gated identically on dollar pools. +9. **VULN-09 (Medium) — Arbitrage spend asymmetry**: the callout now discloses when the API leg + outspends a capped single-seat subscription by >1.25x and points to Dave Mode / Mix & Match. +10. **VULN-10 (Medium) — ToS classifier liability**: `'no guarantee'` removed from TRAINS evidence; + `ipIndemnity: true` renders "Indemnity Offered (warning)" with a carve-outs caveat instead of + "Full Indemnity"; string indemnities get a verify-scope label; `classifyGotcha` only assigns + critical data-privacy severity on strong training/retention signals, weak mentions ("training" + inside a caching sentence) downgrade to warning. +11. **VULN-11 (Medium) — Benchmark starvation**: `build-data.mjs` gained an effort-suffix-stripped + fuzzy matching fallback (recovered 11 of 35 models; match rate 29% → 60%) and a hard floor: + builds fail below a 25% match rate unless `ALLOW_LOW_BENCH_MATCH=1`. +12. **VULN-12 (Medium) — Quota/time realism**: weekly quotas convert at 52/12 weeks (not ×4); the + throttle simulator carries a deferred-turn backlog so slow-queue delay actually reduces + completed turns instead of being cosmetic. +13. **VULN-13 (Medium) — Zero-hostile URL params**: shared `numParam` helper (`src/lib/params.ts`) + replaces `Number(x) || default` in HardwareBreakeven and BurstSimulator; `?salvage=0`, + `?prior=0`, `?idleHours=0` now render as configured. +14. **VULN-14 (Medium) — Cache-write premium + clamp unification**: `cacheWriteShare` (0.06) added to + `data/estimate-constants.json` and wired into `agentBlendedCost` in both `fetch-models.mjs` and + `build-data.mjs` (amortized write cost at the provider write rate); salvage clamp unified at 50 + via `SALVAGE_CLAMP_MAX`; `costPer1kRequests || 1` → `?? 1` (invariant I). +15. **VULN-15 (Low) — Literal extraction**: session cache-bust residual (0.15) and provider + cache-read fallback ratios moved into `data/estimate-constants.json`; teams.ts no longer + fabricates a 10% cache discount for unknown cache prices (conservative full-input fallback). + Residual: `classifyModelTier`'s hardcoded model-name table and teams.ts chat workload shapes + remain flagged for a future data-driven refactor. + +### Verification +- `npm run lint` — 0 errors, 0 warnings. +- `npm run validate-data` — 34 plans and 422 usage-limit entries pass all schema invariants (now + including price sanity bounds). +- `npm test` — 127/127 tests pass across 8 suites (expanded by regression tests for VULN-02/03/04/05/06/07/09 and updated expectations for VULN-10/12). +- `npm run build` — Clean production build. diff --git a/public/data/last-updated.json b/public/data/last-updated.json index 39ca274..eccd012 100644 --- a/public/data/last-updated.json +++ b/public/data/last-updated.json @@ -1,3 +1,3 @@ { - "timestamp": "2026-09-21T02:17:39.710Z" + "timestamp": "2026-09-21T02:50:22.949Z" } \ No newline at end of file diff --git a/public/data/models.json b/public/data/models.json index c3af78b..c94bc93 100644 --- a/public/data/models.json +++ b/public/data/models.json @@ -19,7 +19,7 @@ "webSearch": null }, "blendedCost": 0.5249999999999999, - "agentBlendedCost": 0.1714, + "agentBlendedCost": 0.1849, "costPer1kRequests": 1.8, "benchmarks": { "intelligenceIndex": 29.2, @@ -53,7 +53,7 @@ "webSearch": null }, "blendedCost": 0.5249999999999999, - "agentBlendedCost": 0.1714, + "agentBlendedCost": 0.1849, "costPer1kRequests": 1.8, "benchmarks": { "intelligenceIndex": 29.2, @@ -87,7 +87,7 @@ "webSearch": null }, "blendedCost": 0.28500000000000003, - "agentBlendedCost": 0.1143, + "agentBlendedCost": 0.1214, "costPer1kRequests": 0.96, "benchmarks": { "intelligenceIndex": 25.3, @@ -121,7 +121,7 @@ "webSearch": null }, "blendedCost": 1.3975, - "agentBlendedCost": 0.4736, + "agentBlendedCost": 0.5151, "costPer1kRequests": 4.68, "benchmarks": { "intelligenceIndex": 44.8, @@ -155,7 +155,7 @@ "webSearch": null }, "blendedCost": 1.075, - "agentBlendedCost": 0.3643, + "agentBlendedCost": 0.3962, "costPer1kRequests": 3.6, "benchmarks": { "intelligenceIndex": 44.8, @@ -163,7 +163,7 @@ "agenticIndex": 53.1, "valueScore": 5794.4 }, - "benchmarkSource": "openrouter", + "benchmarkSource": "artificial-analysis", "reasoning": null, "isFree": false, "isBatch": true, @@ -189,7 +189,7 @@ "webSearch": null }, "blendedCost": 0.14250000000000002, - "agentBlendedCost": 0.0486, + "agentBlendedCost": 0.0526, "costPer1kRequests": 0.48, "benchmarks": { "intelligenceIndex": 41.8, @@ -223,7 +223,7 @@ "webSearch": null }, "blendedCost": 0.5249999999999999, - "agentBlendedCost": 0.1329, + "agentBlendedCost": 0.1487, "costPer1kRequests": 1.8, "benchmarks": { "intelligenceIndex": 39.5, @@ -257,7 +257,7 @@ "webSearch": null }, "blendedCost": 1.98, - "agentBlendedCost": 0.5343, + "agentBlendedCost": 0.6031, "costPer1kRequests": 6.6, "benchmarks": { "intelligenceIndex": 36, @@ -265,7 +265,7 @@ "agenticIndex": 41.3, "valueScore": 2726.8 }, - "benchmarkSource": "openrouter", + "benchmarkSource": "artificial-analysis", "reasoning": null, "isFree": false, "isBatch": false, @@ -291,7 +291,7 @@ "webSearch": null }, "blendedCost": 0.99, - "agentBlendedCost": 0.2671, + "agentBlendedCost": 0.3016, "costPer1kRequests": 3.3, "benchmarks": { "intelligenceIndex": 36, @@ -299,7 +299,7 @@ "agenticIndex": 41.3, "valueScore": 5453.5 }, - "benchmarkSource": "openrouter", + "benchmarkSource": "artificial-analysis", "reasoning": null, "isFree": false, "isBatch": true, @@ -325,7 +325,7 @@ "webSearch": null }, "blendedCost": 20, - "agentBlendedCost": 5.4762, + "agentBlendedCost": 5.969, "costPer1kRequests": 70, "benchmarks": { "intelligenceIndex": 52.7, @@ -359,7 +359,7 @@ "webSearch": null }, "blendedCost": 10, - "agentBlendedCost": 2.7381, + "agentBlendedCost": 2.9845, "costPer1kRequests": 35, "benchmarks": { "intelligenceIndex": 52.7, @@ -367,7 +367,7 @@ "agenticIndex": 51, "valueScore": 642.9 }, - "benchmarkSource": "openrouter", + "benchmarkSource": "artificial-analysis", "reasoning": null, "isFree": false, "isBatch": true, @@ -393,7 +393,7 @@ "webSearch": null }, "blendedCost": 4, - "agentBlendedCost": 1.0952, + "agentBlendedCost": 1.1938, "costPer1kRequests": 14, "benchmarks": { "intelligenceIndex": 47, @@ -427,7 +427,7 @@ "webSearch": null }, "blendedCost": 3, - "agentBlendedCost": 0.9405, + "agentBlendedCost": 1.0369, "costPer1kRequests": 10, "benchmarks": { "intelligenceIndex": 45.4, @@ -435,7 +435,7 @@ "agenticIndex": 56, "valueScore": 2132.7 }, - "benchmarkSource": "openrouter", + "benchmarkSource": "artificial-analysis", "reasoning": null, "isFree": false, "isBatch": false, @@ -461,7 +461,7 @@ "webSearch": null }, "blendedCost": 3, - "agentBlendedCost": 0.9405, + "agentBlendedCost": 1.0369, "costPer1kRequests": 10, "benchmarks": { "intelligenceIndex": 39.9, @@ -495,7 +495,7 @@ "webSearch": null }, "blendedCost": 3, - "agentBlendedCost": 0.9405, + "agentBlendedCost": 1.0369, "costPer1kRequests": 10, "benchmarks": { "intelligenceIndex": 39.9, @@ -509,6 +509,40 @@ "isBatch": true, "tierClass": "frontier" }, + { + "id": "meta/muse-spark-1.3-contributor", + "name": "Meta: Muse Spark 1.3 Contributor", + "provider": "meta", + "series": "Llama", + "releasedAt": "2026-09-02", + "createdUnix": 1788381519, + "unbenchmarked": false, + "modality": "text+image+file+audio+video->text", + "contextWindow": 1048576, + "maxOutput": 943718, + "pricing": { + "input": 0.09999999999999999, + "output": 0.19999999999999998, + "cachedInput": 0.002, + "cachedInputWrite": null, + "reasoning": null, + "webSearch": null + }, + "blendedCost": 0.125, + "agentBlendedCost": 0.04, + "costPer1kRequests": 0.39999999999999997, + "benchmarks": { + "intelligenceIndex": 48.1, + "codingIndex": 75.8, + "agenticIndex": 55.5, + "valueScore": 51336 + }, + "benchmarkSource": "artificial-analysis", + "reasoning": null, + "isFree": false, + "isBatch": false, + "tierClass": "frontier" + }, { "id": "meta/muse-spark-1.2", "name": "Meta: Muse Spark 1.2", @@ -529,7 +563,7 @@ "webSearch": null }, "blendedCost": 2, - "agentBlendedCost": 0.6071, + "agentBlendedCost": 0.6677, "costPer1kRequests": 6.750000000000001, "benchmarks": { "intelligenceIndex": 39.6, @@ -563,7 +597,7 @@ "webSearch": null }, "blendedCost": 2, - "agentBlendedCost": 0.6071, + "agentBlendedCost": 0.6677, "costPer1kRequests": 6.750000000000001, "benchmarks": { "intelligenceIndex": 33.7, @@ -577,40 +611,6 @@ "isBatch": false, "tierClass": "frontier" }, - { - "id": "meta/muse-spark-1.3-contributor", - "name": "Meta: Muse Spark 1.3 Contributor", - "provider": "meta", - "series": "Llama", - "releasedAt": "2026-09-02", - "createdUnix": 1788381519, - "unbenchmarked": true, - "modality": "text+image+file+audio+video->text", - "contextWindow": 1048576, - "maxOutput": 943718, - "pricing": { - "input": 0.09999999999999999, - "output": 0.19999999999999998, - "cachedInput": 0.002, - "cachedInputWrite": null, - "reasoning": null, - "webSearch": null - }, - "blendedCost": 0.125, - "agentBlendedCost": 0.0348, - "costPer1kRequests": 0.39999999999999997, - "benchmarks": { - "intelligenceIndex": null, - "codingIndex": null, - "agenticIndex": null, - "valueScore": null - }, - "benchmarkSource": null, - "reasoning": null, - "isFree": false, - "isBatch": false, - "tierClass": "balanced" - }, { "id": "google/gemini-3.8-flash", "name": "Google: Gemini 3.8 Flash", @@ -631,7 +631,7 @@ "webSearch": null }, "blendedCost": 1.5, - "agentBlendedCost": 0.4107, + "agentBlendedCost": 0.4093, "costPer1kRequests": 5.25, "benchmarks": { "intelligenceIndex": 40.9, @@ -665,7 +665,7 @@ "webSearch": null }, "blendedCost": 0.75, - "agentBlendedCost": 0.2054, + "agentBlendedCost": 0.2055, "costPer1kRequests": 2.625, "benchmarks": { "intelligenceIndex": 40.9, @@ -673,7 +673,7 @@ "agenticIndex": 40.2, "valueScore": 7785.3 }, - "benchmarkSource": "openrouter", + "benchmarkSource": "artificial-analysis", "reasoning": null, "isFree": false, "isBatch": true, @@ -699,7 +699,7 @@ "webSearch": null }, "blendedCost": 1.5, - "agentBlendedCost": 0.4107, + "agentBlendedCost": 0.4093, "costPer1kRequests": 5.25, "benchmarks": { "intelligenceIndex": 39.1, @@ -733,7 +733,7 @@ "webSearch": null }, "blendedCost": 20, - "agentBlendedCost": 4.9405, + "agentBlendedCost": 5.4655, "costPer1kRequests": 70, "benchmarks": { "intelligenceIndex": 53.4, @@ -767,7 +767,7 @@ "webSearch": null }, "blendedCost": 10, - "agentBlendedCost": 2.4702, + "agentBlendedCost": 2.7327, "costPer1kRequests": 35, "benchmarks": { "intelligenceIndex": 53.4, @@ -775,7 +775,7 @@ "agenticIndex": 57.9, "valueScore": 688.5 }, - "benchmarkSource": "openrouter", + "benchmarkSource": "artificial-analysis", "reasoning": null, "isFree": false, "isBatch": true, @@ -801,7 +801,7 @@ "webSearch": null }, "blendedCost": 10, - "agentBlendedCost": 2.7381, + "agentBlendedCost": 2.9845, "costPer1kRequests": 35, "benchmarks": { "intelligenceIndex": 50.8, @@ -835,7 +835,7 @@ "webSearch": null }, "blendedCost": 3, - "agentBlendedCost": 1.119, + "agentBlendedCost": 1.2048, "costPer1kRequests": 10, "benchmarks": { "intelligenceIndex": 44.3, @@ -869,7 +869,7 @@ "webSearch": null }, "blendedCost": 3, - "agentBlendedCost": 0.9762, + "agentBlendedCost": 1.0705, "costPer1kRequests": 10, "benchmarks": { "intelligenceIndex": 38.8, @@ -903,7 +903,7 @@ "webSearch": null }, "blendedCost": 1.5625, - "agentBlendedCost": 0.5595, + "agentBlendedCost": 0.6179, "costPer1kRequests": 5, "benchmarks": { "intelligenceIndex": 24.9, @@ -937,7 +937,7 @@ "webSearch": null }, "blendedCost": 3.6012500000000003, - "agentBlendedCost": 0.9843, + "agentBlendedCost": 1.0657, "costPer1kRequests": 12.73, "benchmarks": { "intelligenceIndex": 43.6, @@ -971,7 +971,7 @@ "webSearch": null }, "blendedCost": 6, - "agentBlendedCost": 1.6429, + "agentBlendedCost": 1.7907, "costPer1kRequests": 21, "benchmarks": { "intelligenceIndex": 43.6, @@ -979,7 +979,7 @@ "agenticIndex": 50, "valueScore": 1030.3 }, - "benchmarkSource": "openrouter", + "benchmarkSource": "artificial-analysis", "reasoning": null, "isFree": false, "isBatch": true, @@ -1005,7 +1005,7 @@ "webSearch": null }, "blendedCost": 1.7125, - "agentBlendedCost": 0.531, + "agentBlendedCost": 0.575, "costPer1kRequests": 5.8999999999999995, "benchmarks": { "intelligenceIndex": 27, @@ -1039,7 +1039,7 @@ "webSearch": null }, "blendedCost": 3, - "agentBlendedCost": 1.7857, + "agentBlendedCost": 1.8018, "costPer1kRequests": 10.5, "benchmarks": { "intelligenceIndex": 14.2, @@ -1073,7 +1073,7 @@ "webSearch": null }, "blendedCost": 1.5, - "agentBlendedCost": 0.8929, + "agentBlendedCost": 0.9009, "costPer1kRequests": 5.25, "benchmarks": { "intelligenceIndex": 14.2, @@ -1081,7 +1081,7 @@ "agenticIndex": 7, "valueScore": 1892.7 }, - "benchmarkSource": "openrouter", + "benchmarkSource": "artificial-analysis", "reasoning": null, "isFree": false, "isBatch": true, @@ -1107,7 +1107,7 @@ "webSearch": null }, "blendedCost": 0.26249999999999996, - "agentBlendedCost": 0.075, + "agentBlendedCost": 0.0824, "costPer1kRequests": 0.9, "benchmarks": { "intelligenceIndex": 11.3, @@ -1141,7 +1141,7 @@ "webSearch": null }, "blendedCost": 0.54375, - "agentBlendedCost": 0.1476, + "agentBlendedCost": 0.1707, "costPer1kRequests": 1.74, "benchmarks": { "intelligenceIndex": 26, @@ -1175,15 +1175,15 @@ "webSearch": null }, "blendedCost": 0.17500000000000002, - "agentBlendedCost": 0.0487, + "agentBlendedCost": 0.056, "costPer1kRequests": 0.56, "benchmarks": { - "intelligenceIndex": null, - "codingIndex": 56.8, - "agenticIndex": 15.8, - "valueScore": 21304.3 + "intelligenceIndex": 26, + "codingIndex": 60.2, + "agenticIndex": 21.3, + "valueScore": 23822.9 }, - "benchmarkSource": "openrouter", + "benchmarkSource": "artificial-analysis", "reasoning": null, "isFree": false, "isBatch": false, diff --git a/public/data/usage-limits.json b/public/data/usage-limits.json index a268d75..6b9edd5 100644 --- a/public/data/usage-limits.json +++ b/public/data/usage-limits.json @@ -1,6 +1,6 @@ { "schemaVersion": "1.0.0", - "generatedAt": "2026-09-21T02:17:39.704Z", + "generatedAt": "2026-09-21T02:50:22.938Z", "constants": { "agentTurnTokens": 21000, "defaultCacheRate": 0.75, diff --git a/scripts/build-data.mjs b/scripts/build-data.mjs index d2ff797..f797cc0 100755 --- a/scripts/build-data.mjs +++ b/scripts/build-data.mjs @@ -17,6 +17,19 @@ function normalizeStr(str) { return (str || '').toLowerCase().replace(/[^a-z0-9]/g, ''); } +// Effort/variant suffixes AA appends to benchmark names ("Grok 4.6 (medium)"). +const EFFORT_SUFFIX_RE = /\s*\((?:low|medium|high|max|xhigh)\)\s*$/i; + +function normalizeBenchmarkName(name) { + return normalizeStr((name || '').replace(EFFORT_SUFFIX_RE, '')); +} + +// Minimum fraction of catalog models that must receive benchmark data. +// Below this the benchmark feed has drifted and the quality-filtered +// leaderboards silently starve (VULN-11). Set ALLOW_LOW_BENCH_MATCH=1 to +// bypass with a warning. +const MIN_BENCH_MATCH_RATE = 0.25; + function classifyModelTier(model) { const id = (model.id || '').toLowerCase(); const name = (model.name || '').toLowerCase(); @@ -61,6 +74,8 @@ async function buildData() { const { inputTokens: agentIn, outputTokens: agentOut } = estimateConstants.agentRequest; const agentRequestTokens = agentIn + agentOut; const cacheRate = estimateConstants.defaultCacheRate; + const cacheWriteShare = estimateConstants.cacheWriteShare ?? 0; + const cacheWritePremium = estimateConstants.cacheWritePremium ?? 1.25; // 1. Load models let models = []; @@ -81,18 +96,33 @@ async function buildData() { // 2. Merge benchmarks into models with enhanced matching let matchedCount = 0; + const fuzzyMatches = []; + const benchBySlug = new Map(benchmarks.map(b => [normalizeStr(b.slug || ''), b])); + const benchByName = new Map(benchmarks.map(b => [normalizeBenchmarkName(b.name), b])); + for (const model of models) { const modelIdClean = model.id.split('/').pop() || ''; const mIdNorm = normalizeStr(modelIdClean); - const mNameNorm = normalizeStr(model.name); + const mNameNorm = normalizeBenchmarkName(model.name); - const match = benchmarks.find(b => { - const bSlugClean = b.slug || ''; - const bSlugNorm = normalizeStr(bSlugClean); - const bNameNorm = normalizeStr(b.name); + // Pass 1: exact normalized slug/name equality. + let match = benchmarks.find(b => { + const bSlugNorm = normalizeStr(b.slug || ''); + const bNameNorm = normalizeBenchmarkName(b.name); + return bSlugNorm === mIdNorm || bNameNorm === mNameNorm; + }); - return bSlugNorm === mIdNorm || bNameNorm === mNameNorm; - }); + // Pass 2 (VULN-11): effort-suffix-stripped fuzzy match so "Grok 4.6 (medium)" + // still matches "grok-4.6" instead of silently starving the leaderboard. + if (!match && benchmarks.length > 0) { + match = + benchBySlug.get(mIdNorm) || + (mIdNorm.length >= 5 + ? [...benchBySlug.entries()].find(([slug]) => slug.startsWith(mIdNorm) || mIdNorm.startsWith(slug) && slug.length >= 5)?.[1] + : undefined) || + benchByName.get(mNameNorm); + if (match) fuzzyMatches.push(`${model.id} -> ${match.name ?? match.slug}`); + } if (match && match.evaluations) { matchedCount++; @@ -114,19 +144,24 @@ async function buildData() { model.benchmarkSource = 'openrouter'; } - // Cache pricing: never fabricate a discount. Use the real cachedInput from - // the upstream fetch; unknown cache prices are treated as no caching. - const inputPrice = model.pricing?.input || 0; - const outputPrice = model.pricing?.output || 0; - const cachedInput = model.pricing?.cachedInput; - // Standard Agent Request from the shared estimate constants (default // 20K input at 75% cache + 1K output). When no cache price is known, // charge full input price for cached tokens (conservative). + // Amortized cache-write share (VULN-14): keep in lockstep with + // scripts/fetch-models.mjs so both agents of record agree. + const inputPrice = model.pricing?.input || 0; + const outputPrice = model.pricing?.output || 0; + const cachedInput = model.pricing?.cachedInput; + const cachedInputWrite = model.pricing?.cachedInputWrite; + const freshIn = agentIn * (1 - cacheRate); const cachedIn = agentIn * cacheRate; const cachePrice = cachedInput !== null && cachedInput !== undefined ? cachedInput : inputPrice; - const agentReqCost = (freshIn * inputPrice / 1e6) + (cachedIn * cachePrice / 1e6) + (agentOut * outputPrice / 1e6); + const writePrice = cachedInputWrite !== null && cachedInputWrite !== undefined ? cachedInputWrite : inputPrice * cacheWritePremium; + const agentReqCost = (freshIn * inputPrice / 1e6) + + (cachedIn * (1 - cacheWriteShare) * cachePrice / 1e6) + + (cachedIn * cacheWriteShare * writePrice / 1e6) + + (agentOut * outputPrice / 1e6); model.agentBlendedCost = parseFloat(((agentReqCost / agentRequestTokens) * 1e6).toFixed(4)); // Assign tier @@ -151,7 +186,21 @@ async function buildData() { } } - console.log(`Matched ${matchedCount} models with Artificial Analysis benchmarks.`); + console.log(`Matched ${matchedCount} models with Artificial Analysis benchmarks (${fuzzyMatches.length} via fuzzy fallback).`); + if (fuzzyMatches.length > 0) { + for (const f of fuzzyMatches.slice(0, 10)) console.log(` fuzzy: ${f}`); + } + + // Match-rate floor (VULN-11): a drifted benchmark feed must not silently + // starve the quality-filtered leaderboards of coding indices. + if (models.length > 0 && benchmarks.length > 0 && matchedCount / models.length < MIN_BENCH_MATCH_RATE) { + const msg = `Benchmark match rate ${matchedCount}/${models.length} (${(matchedCount / models.length * 100).toFixed(1)}%) is below the ${MIN_BENCH_MATCH_RATE * 100}% floor — the benchmark feed has likely drifted. Fix the matching or set ALLOW_LOW_BENCH_MATCH=1 to override.`; + if (process.env.ALLOW_LOW_BENCH_MATCH === '1') { + console.warn(`WARNING: ${msg}`); + } else { + throw new Error(msg); + } + } // Authoritative series re-derive + unbenchmarked flag + age/hub re-assertion. // models.json may be stale from an older run; re-enforce the invariants here. diff --git a/scripts/fetch-models.mjs b/scripts/fetch-models.mjs index f75600c..ea5f8ac 100755 --- a/scripts/fetch-models.mjs +++ b/scripts/fetch-models.mjs @@ -27,6 +27,8 @@ async function fetchModels() { const { inputTokens: chatIn, outputTokens: chatOut } = constants.chatRequest; const { inputWeight: blendIn, outputWeight: blendOut } = constants.chatBlend; const cacheRate = constants.defaultCacheRate; + const cacheWriteShare = constants.cacheWriteShare ?? 0; + const cacheWritePremium = constants.cacheWritePremium ?? 1.25; const agentRequestTokens = agentIn + agentOut; const response = await fetch('https://openrouter.ai/api/v1/models', { headers }); @@ -108,19 +110,16 @@ async function fetchModels() { cachedInputWrite = upstreamCacheWrite * 1_000_000; } if (cachedInput === null) { + // Documented per-provider fallback ratios from data/estimate-constants.json + // (VULN-15: previously scattered literals). Unknown providers get no + // fabricated discount — cached tokens bill at full input price. + const fallbacks = constants.providerCacheReadFallback ?? {}; const provLower = provider.toLowerCase(); const idLower = id.toLowerCase(); if (inputPrice > 0) { - if (provLower.includes('anthropic') || idLower.includes('claude')) { - cachedInput = inputPrice * 0.10; - } else if (provLower.includes('deepseek') || idLower.includes('deepseek')) { - cachedInput = inputPrice * 0.10; - } else if (provLower.includes('z-ai') || idLower.includes('glm')) { - cachedInput = inputPrice * 0.10; - } else if (provLower.includes('google') || idLower.includes('gemini')) { - cachedInput = inputPrice * 0.25; - } else if (provLower.includes('openai') || idLower.includes('gpt') || idLower.includes('codex')) { - cachedInput = inputPrice * 0.50; + const fallbackKey = Object.keys(fallbacks).find(k => provLower.includes(k) || idLower.includes(k)); + if (fallbackKey) { + cachedInput = inputPrice * fallbacks[fallbackKey]; } } } @@ -128,10 +127,17 @@ async function fetchModels() { // Agentic coding calculation: standard agent request (default 20K input at // 75% cache + 1K output) from the shared estimate constants. // When no cache price is known, charge full input price for cached tokens (conservative). + // Amortized cache-write share (VULN-14): the fraction of cached context + // rewritten each turn is billed at the provider's write rate instead of + // being silently treated as a free read. const freshIn = agentIn * (1 - cacheRate); const cachedIn = agentIn * cacheRate; const cachePrice = cachedInput !== null ? cachedInput : inputPrice; - const agentReqCost = (freshIn * inputPrice / 1e6) + (cachedIn * cachePrice / 1e6) + (agentOut * outputPrice / 1e6); + const writePrice = cachedInputWrite !== null ? cachedInputWrite : inputPrice * cacheWritePremium; + const agentReqCost = (freshIn * inputPrice / 1e6) + + (cachedIn * (1 - cacheWriteShare) * cachePrice / 1e6) + + (cachedIn * cacheWriteShare * writePrice / 1e6) + + (agentOut * outputPrice / 1e6); const agentBlendedCost = (agentReqCost / agentRequestTokens) * 1e6; const createdUnix = Number.isFinite(model.created) ? model.created : null; diff --git a/scripts/validate-data.mjs b/scripts/validate-data.mjs index 9fd9df5..b9db15e 100644 --- a/scripts/validate-data.mjs +++ b/scripts/validate-data.mjs @@ -185,6 +185,12 @@ async function validateModels(models) { 'GLM', 'Kimi', 'Mistral', 'MiMo', 'other', ]); + // Price sanity bounds (VULN-01): structural validation alone let an + // upstream regression or a poisoned fetch auto-deploy absurd yields. + // Sane ceiling: no per-million list price above $1,000 in/$2,000 out. + const MAX_INPUT_PER_M = 1_000; + const MAX_OUTPUT_PER_M = 2_000; + const SERIES_CAP = 3; const AGE_WINDOW_DAYS = 365; const cutoffUnix = Math.floor(Date.now() / 1000) - AGE_WINDOW_DAYS * 86400; @@ -197,6 +203,31 @@ async function validateModels(models) { continue; } + // --- price plausibility bounds (VULN-01) --- + const pricing = m.pricing ?? {}; + const inP = pricing.input; + const outP = pricing.output; + if (typeof inP !== 'number' || !Number.isFinite(inP) || inP < 0) { + err(`${where}: input price must be a finite non-negative number, got ${inP}`); + } else if (!m.isFree && !m.isBatch && inP === 0) { + err(`${where}: non-free model has $0 input price (upstream data regression?)`); + } else if (inP > MAX_INPUT_PER_M) { + err(`${where}: input price $${inP}/M exceeds sanity ceiling $${MAX_INPUT_PER_M}/M`); + } + if (typeof outP !== 'number' || !Number.isFinite(outP) || outP < 0) { + err(`${where}: output price must be a finite non-negative number, got ${outP}`); + } else if (!m.isFree && !m.isBatch && outP === 0) { + err(`${where}: non-free model has $0 output price (upstream data regression?)`); + } else if (outP > MAX_OUTPUT_PER_M) { + err(`${where}: output price $${outP}/M exceeds sanity ceiling $${MAX_OUTPUT_PER_M}/M`); + } + if (pricing.cachedInput != null && (!Number.isFinite(pricing.cachedInput) || pricing.cachedInput < 0)) { + err(`${where}: cachedInput must be null or a finite non-negative number`); + } + if (typeof m.blendedCost === 'number' && m.blendedCost < 0) { + err(`${where}: blendedCost must not be negative`); + } + if (typeof m.series !== 'string' || m.series.length === 0) { err(`${where}: missing canonical "series" field`); } else if (!KNOWN_SERIES.has(m.series)) { diff --git a/src/components/budget/WorkflowCalculator.tsx b/src/components/budget/WorkflowCalculator.tsx index 570b7af..5d47d41 100644 --- a/src/components/budget/WorkflowCalculator.tsx +++ b/src/components/budget/WorkflowCalculator.tsx @@ -2,7 +2,7 @@ import { useState, useMemo } from 'react'; import { useSearchParams } from 'react-router-dom'; import type { CodingPlan, NormalizedModel, CacheRate, WorkloadItem } from '../../lib/types'; import { getEffectiveCacheMultiplier, calculatePoolDrain, resolveTierModelBudget, QUALITY } from '../../lib/pricing'; -import { DEFAULT_CACHE_RATE } from '../../lib/estimate-constants'; +import { DEFAULT_CACHE_RATE, SESSION_CACHE_BUST_RESIDUAL } from '../../lib/estimate-constants'; import { Workflow, Sparkles, @@ -111,7 +111,7 @@ function sessionCost( const baseFresh = Math.min(sessionInputSum, finalContextTokens); const reReadTokens = Math.max(0, sessionInputSum - baseFresh); const uncachedOverhead = Math.max(0, 1 - cacheRate); - const freshInput = baseFresh + reReadTokens * uncachedOverhead * 0.15; + const freshInput = baseFresh + reReadTokens * uncachedOverhead * SESSION_CACHE_BUST_RESIDUAL; const cachedInput = Math.max(0, sessionInputSum - freshInput); return ( diff --git a/src/lib/estimate-constants.ts b/src/lib/estimate-constants.ts index 1da686c..6d4b362 100644 --- a/src/lib/estimate-constants.ts +++ b/src/lib/estimate-constants.ts @@ -20,6 +20,10 @@ export const AGENT_TASK_OPTIMISTIC_TOKENS = constants.agentTask.optimisticTokens export const DEFAULT_CACHE_RATE = constants.defaultCacheRate as CacheRate; /** Anthropic cache-write premium (5-minute TTL) used when a provider price is missing. */ export const CACHE_WRITE_PREMIUM = constants.cacheWritePremium; +/** Default fraction of cached context rewritten per turn (see data/estimate-constants.json _cacheWriteShare). */ +export const CACHE_WRITE_SHARE = constants.cacheWriteShare as number; export const TIME_OF_DAY_BLEND = constants.timeOfDayBlend; export const AGENT_CACHE_PRESETS = constants.agentCachePresets; +/** Residual full-price share of re-read tokens inside one multi-turn session (cache invalidations). */ +export const SESSION_CACHE_BUST_RESIDUAL = constants.sessionCacheBustResidual as number; diff --git a/src/lib/hardware.test.ts b/src/lib/hardware.test.ts index f0c7046..38387ed 100644 --- a/src/lib/hardware.test.ts +++ b/src/lib/hardware.test.ts @@ -129,3 +129,62 @@ describe('hardware economics engine', () => { expect(result.monthlySavingsVsApi).toBeLessThan(0); }); }); + +describe('Physical workload feasibility (VULN-04)', () => { + it('flags a workload that exceeds the configured duty cycle as infeasible', () => { + // Mac Mini flagship decodes at 24 tok/s. 40M tokens/mo needs ~463h of pure + // decode, but 1h/day x 30.4 only affords ~30h — physically impossible. + const result = computeHardwareEconomics({ + presetId: 'mac-mini-m4-pro-64gb', + amortizationMonths: 24, + salvageValuePercent: 20, + electricityKwhCost: 0.16, + dailyInferenceHours: 1, + dailyIdleHours: 9, + monthlyTokenVolumeM: 40, + inputRatio: 0.8, + cloudApiInputPerM: 3.0, + cloudApiOutputPerM: 15.0, + }); + + expect(result.infeasibleWorkload).toBe(true); + expect(result.requiredInferenceHours).toBeGreaterThan(400); + expect(result.feasibleMonthlyVolumeM).toBeLessThan(40); + }); + + it('marks a workload achievable within the duty cycle as feasible', () => { + const result = computeHardwareEconomics({ + presetId: 'mac-mini-m4-pro-64gb', + amortizationMonths: 24, + salvageValuePercent: 20, + electricityKwhCost: 0.16, + dailyInferenceHours: 3, + dailyIdleHours: 9, + monthlyTokenVolumeM: 5, + inputRatio: 0.8, + cloudApiInputPerM: 3.0, + cloudApiOutputPerM: 15.0, + }); + + expect(result.infeasibleWorkload).toBe(false); + expect(result.feasibleMonthlyVolumeM).toBeGreaterThanOrEqual(5); + }); + + it('only quotes payback when the amortized monthly alpha is positive', () => { + // Cloud cost below amortized local cost must suppress payback entirely so + // the headline metrics can never contradict each other. + const result = computeHardwareEconomics({ + presetId: 'custom-dual-rtx-5090', + amortizationMonths: 24, + salvageValuePercent: 10, + electricityKwhCost: 0.2, + dailyInferenceHours: 6, + dailyIdleHours: 12, + monthlyTokenVolumeM: 1, + inputRatio: 0.8, + cloudApiInputPerM: 0.14, + cloudApiOutputPerM: 0.28, + }); + expect(result.paybackMonthsVsApi).toBeNull(); + }); +}); diff --git a/src/lib/hardware.ts b/src/lib/hardware.ts index 9c13ff7..c59ce3e 100644 --- a/src/lib/hardware.ts +++ b/src/lib/hardware.ts @@ -34,7 +34,7 @@ export interface HardwareEconomicsInputs { presetId: string; customCapexUsd?: number; amortizationMonths: number; // e.g. 12, 24, 36, 48 - salvageValuePercent: number; // e.g. 0 to 40% + salvageValuePercent: number; // e.g. 0 to 50% (single clamp constant shared with the UI slider) electricityKwhCost: number; // default $0.16/kWh dailyInferenceHours: number; // default 3 hrs dailyIdleHours: number; // default 9 hrs @@ -73,8 +73,17 @@ export interface HardwareEconomicsResult { paybackMonthsVsPlan: number | null; crossoverSeries: CrossoverPoint[]; cumulativeTcoSeries: CumulativePoint[]; + /** Hours of pure decode needed per month to generate the requested volume on the preset's flagship model. */ + requiredInferenceHours: number; + /** True when requiredInferenceHours exceeds the configured daily inference budget. */ + infeasibleWorkload: boolean; + /** Largest monthly volume (M tokens) the configured duty cycle can physically generate. */ + feasibleMonthlyVolumeM: number; } +/** Single salvage-value clamp shared by the engine and the UI slider (VULN-14). */ +export const SALVAGE_CLAMP_MAX = 50; + export const HARDWARE_PRESETS: HardwarePreset[] = [ { id: 'mac-mini-m4-pro-64gb', @@ -381,7 +390,7 @@ export function computeHardwareEconomics(inputs: HardwareEconomicsInputs): Hardw const capex = inputs.customCapexUsd ?? preset.capexUsd; // 1. Amortization with salvage value - const netDepreciationCapEx = capex * (1 - Math.max(0, Math.min(60, inputs.salvageValuePercent)) / 100); + const netDepreciationCapEx = capex * (1 - Math.max(0, Math.min(SALVAGE_CLAMP_MAX, inputs.salvageValuePercent)) / 100); const months = Math.max(1, inputs.amortizationMonths); const monthlyAmortizationCost = netDepreciationCapEx / months; @@ -416,24 +425,35 @@ export function computeHardwareEconomics(inputs: HardwareEconomicsInputs): Hardw // volumeM = totalMonthlyLocalCost / blendedApiRatePerM const crossoverVolumeM = blendedApiRatePerM > 0 ? totalMonthlyLocalCost / blendedApiRatePerM : 0; - // 7. Payback period in months - // Capital outlay / monthly operational savings vs cloud - // Monthly cloud spend avoided - ongoing monthly electricity cost = net cash flow benefit - const netMonthlyBenefitVsApi = monthlyCloudApiCost - monthlyElectricityCost; + // 7. Physical feasibility: generating the requested volume requires + // volume / throughput seconds of pure decode (VULN-04). Compare against + // the configured daily inference budget so infeasible workloads are + // flagged instead of silently "winning" the breakeven verdict. + const decodeTokensPerSec = preset.recommendedModels[0]?.tokensPerSec ?? 1; + const requiredInferenceHours = (Math.max(0, inputs.monthlyTokenVolumeM) * 1e6) / (decodeTokensPerSec * 3600); + const availableInferenceHours = Math.max(0, inputs.dailyInferenceHours) * 30.4375; + const feasibleMonthlyVolumeM = (availableInferenceHours * decodeTokensPerSec * 3600) / 1e6; + const infeasibleWorkload = requiredInferenceHours > availableInferenceHours * 1.001; + + // 8. Payback period in months + // Capital outlay / monthly operational savings vs cloud. Both payback and + // headline alpha use the same amortized cost basis so the two metrics can + // never contradict each other (VULN-04). + const netMonthlyBenefitVsApi = monthlyCloudApiCost - monthlyElectricityCost - monthlyAmortizationCost; let paybackMonthsVsApi: number | null = null; if (netMonthlyBenefitVsApi > 0) { - paybackMonthsVsApi = capex / netMonthlyBenefitVsApi; + paybackMonthsVsApi = capex / (monthlyCloudApiCost - monthlyElectricityCost); } let paybackMonthsVsPlan: number | null = null; if (monthlyCloudPlanCost !== null) { - const netMonthlyBenefitVsPlan = monthlyCloudPlanCost - monthlyElectricityCost; - if (netMonthlyBenefitVsPlan > 0) { - paybackMonthsVsPlan = capex / netMonthlyBenefitVsPlan; + const grossMonthlyBenefitVsPlan = monthlyCloudPlanCost - monthlyElectricityCost; + if (grossMonthlyBenefitVsPlan > 0) { + paybackMonthsVsPlan = capex / grossMonthlyBenefitVsPlan; } } - // 8. Generate Crossover Volume series (0 to 30M tokens in steps) + // 10. Generate Crossover Volume series (0 to 30M tokens in steps) const crossoverSeries: CrossoverPoint[] = []; const maxVolume = Math.max(30, Math.ceil(Math.max(crossoverVolumeM * 1.5, inputs.monthlyTokenVolumeM * 1.5))); const steps = 15; @@ -450,7 +470,7 @@ export function computeHardwareEconomics(inputs: HardwareEconomicsInputs): Hardw }); } - // 9. Generate Cumulative TCO over 36 months + // 11. Generate Cumulative TCO over 36 months const cumulativeTcoSeries: CumulativePoint[] = []; for (let m = 1; m <= 36; m++) { // Local cumulative: Capex upfront + monthly electricity @@ -480,5 +500,8 @@ export function computeHardwareEconomics(inputs: HardwareEconomicsInputs): Hardw paybackMonthsVsPlan: paybackMonthsVsPlan !== null ? Number(paybackMonthsVsPlan.toFixed(1)) : null, crossoverSeries, cumulativeTcoSeries, + requiredInferenceHours: Number(requiredInferenceHours.toFixed(1)), + infeasibleWorkload, + feasibleMonthlyVolumeM: Number(feasibleMonthlyVolumeM.toFixed(1)), }; } diff --git a/src/lib/log-parser.test.ts b/src/lib/log-parser.test.ts index 0383afa..ccf5b10 100644 --- a/src/lib/log-parser.test.ts +++ b/src/lib/log-parser.test.ts @@ -148,3 +148,37 @@ describe('Agent Log Parser & Session Receipt Engine', () => { expect(() => parseAgentLog(hugeLog, 'huge.json')).toThrow('25MB'); }); }); + +describe('Token accounting provenance (VULN-05)', () => { + it('marks JSONL transcripts with real token fields as measured', () => { + const jsonl = [ + JSON.stringify({ input_tokens: 12000, cached_tokens: 9000, output_tokens: 700 }), + JSON.stringify({ input_tokens: 15000, cached_tokens: 11000, output_tokens: 900 }), + ].join('\n'); + const session = parseAgentLog(jsonl, 'measured.jsonl'); + expect(session.measuredTokens).toBe(true); + }); + + it('marks token-less transcripts as estimated so receipts cannot pass them off as bills', () => { + const jsonl = [ + JSON.stringify({ type: 'USER_INPUT', message: 'fix the tests' }), + JSON.stringify({ type: 'ASSISTANT', message: 'done' }), + ].join('\n'); + const session = parseAgentLog(jsonl, 'no-tokens.jsonl'); + expect(session.measuredTokens).toBe(false); + + const report = calculateSessionReceipt(session, []); + expect(report.tokensMeasured).toBe(false); + expect(report.headlineSummary).toContain('ESTIMATED'); + }); + + it('marks generic text fallbacks as estimated', () => { + const session = parseAgentLog('some random markdown\nwith lines\nbut no json', 'notes.md'); + expect(session.format).toBe('generic-agent'); + expect(session.measuredTokens).toBe(false); + }); + + it('keeps the built-in sample session measured', () => { + expect(SAMPLE_AGENT_SESSION.measuredTokens).toBe(true); + }); +}); diff --git a/src/lib/log-parser.ts b/src/lib/log-parser.ts index 274df13..c47231b 100644 --- a/src/lib/log-parser.ts +++ b/src/lib/log-parser.ts @@ -21,6 +21,12 @@ export interface ParsedAgentSession { effectiveCacheHitRate: number; // 0 to 1 toolInvocationsCount: number; turns: ParsedSessionTurn[]; + /** + * True when the transcript contained real token accounting. + * False means the parser synthesized the numbers (VULN-05) — receipts must + * flag these as estimates, never present them as measured cost. + */ + measuredTokens: boolean; } /** @@ -38,6 +44,7 @@ export const SAMPLE_AGENT_SESSION: ParsedAgentSession = { totalTokens: 1392400, effectiveCacheHitRate: 0.78, // 78% cache hit rate toolInvocationsCount: 46, + measuredTokens: true, turns: Array.from({ length: 38 }, (_, i) => { const isToolHeavy = i % 2 === 1; const input = 32000 + i * 800; @@ -102,6 +109,7 @@ function parseClineJson(data: any, filename: string): ParsedAgentSession { let totalCachedTokens = 0; let totalOutputTokens = 0; let toolInvocationsCount = 0; + let sawMeasuredTokens = false; const turns: ParsedSessionTurn[] = []; messages.forEach((msg, idx) => { @@ -110,6 +118,8 @@ function parseClineJson(data: any, filename: string): ParsedAgentSession { const outTok = safeTokenNumber(msg.tokensOut ?? msg.outputTokens ?? msg.usage?.completion_tokens); const tools = Array.isArray(msg.tool_calls || msg.tools) ? (msg.tool_calls || msg.tools).length : 0; + if (inTok > 0 || outTok > 0) sawMeasuredTokens = true; + if (inTok > 0 || outTok > 0 || tools > 0) { totalInputTokens += inTok; totalCachedTokens += cachedTok; @@ -145,6 +155,7 @@ function parseClineJson(data: any, filename: string): ParsedAgentSession { effectiveCacheHitRate: effectiveCacheHitRate || 0.72, toolInvocationsCount: toolInvocationsCount > 0 ? toolInvocationsCount : Math.round(totalTurns * 1.2), turns, + measuredTokens: sawMeasuredTokens, }; } @@ -154,7 +165,8 @@ function parseJsonlSession(lines: string[], filename: string): ParsedAgentSessio let totalCachedTokens = 0; let totalOutputTokens = 0; let toolInvocationsCount = 0; - const turns: ParsedSessionTurn[] = []; + let sawMeasuredTokens = false; + const turns: ParsedAgentSession['turns'] = []; for (const line of lines) { try { @@ -175,6 +187,8 @@ function parseJsonlSession(lines: string[], filename: string): ParsedAgentSessio inTok = 20000 + stepIndex * 600; cachedTok = Math.round(inTok * 0.75); outTok = 850; + } else { + sawMeasuredTokens = true; } totalInputTokens += inTok; @@ -214,6 +228,7 @@ function parseJsonlSession(lines: string[], filename: string): ParsedAgentSessio effectiveCacheHitRate, toolInvocationsCount, turns, + measuredTokens: sawMeasuredTokens, }; } @@ -236,6 +251,7 @@ function parseGenericSession(text: string, filename: string): ParsedAgentSession totalTokens: totalInputTokens + totalOutputTokens, effectiveCacheHitRate: 0.75, toolInvocationsCount: Math.round(turnsCount * 1.3), + measuredTokens: false, turns: Array.from({ length: turnsCount }, (_, i) => ({ stepIndex: i + 1, role: 'assistant', diff --git a/src/lib/params.ts b/src/lib/params.ts new file mode 100644 index 0000000..df36496 --- /dev/null +++ b/src/lib/params.ts @@ -0,0 +1,19 @@ +/** + * Zero-safe URL parameter parsing (VULN-13). + * + * `Number(x) || default` collapses legitimate `0` values to the default, + * which silently diverges shared configs from rendered economics + * (e.g. `?salvage=0` or `?idleHours=0` rendering 20% / 9h instead). + * This helper honors 0 and only falls back when the value is absent or NaN. + */ +export function numParam( + raw: string | null, + fallback: number, + min: number, + max: number +): number { + if (raw === null || raw.trim() === '') return fallback; + const n = Number(raw); + if (!Number.isFinite(n)) return fallback; + return Math.min(max, Math.max(min, n)); +} diff --git a/src/lib/pricing.test.ts b/src/lib/pricing.test.ts index 65abd0d..83b3786 100644 --- a/src/lib/pricing.test.ts +++ b/src/lib/pricing.test.ts @@ -589,7 +589,8 @@ describe('Adversarial Edge Cases & Guardrails', () => { expect(parseTierRequestLimit({ requests: '~15K mix estimate (~26K with DeepSeek V4 Flash at typical cache)' })).toBe(15000); expect(parseTierRequestLimit({ requests: '~100K mix estimate' })).toBe(100000); expect(parseTierRequestLimit({ agenticRequests: '50 agentic requests/mo (chat + agentic coding)' })).toBe(50); - expect(parseTierRequestLimit({ requests: '1,000 requests per 7-day sliding window' })).toBe(4000); + // Weekly quotas scale by 52/12 weeks per month (VULN-12), not a flat x4. + expect(parseTierRequestLimit({ requests: '1,000 requests per 7-day sliding window' })).toBe(4333); expect(parseTierRequestLimit({ fastRequests: 500 })).toBe(500); // Ignores pure dollar credit strings expect(parseTierRequestLimit({ monthlyCredits: '$10/mo compute credits' })).toBe(null); @@ -845,7 +846,7 @@ describe('Adversarial Edge Cases & Guardrails', () => { expect(result.savings).toBeLessThan(0); }); - it('computeApplesToApples applies symmetric cache scaling to subscription yields', () => { + it('does NOT cache-scale usage-metered subscription yields (cache rate is irrelevant to fixed quotas)', () => { const testModel = model({ id: 'z-ai/glm-5.3', name: 'GLM-5.3', @@ -873,42 +874,84 @@ describe('Adversarial Edge Cases & Guardrails', () => { const testPlan = plan({ id: 'z-ai', name: 'Z.ai', tiers: [testTier] }); - // When cacheRate is 75% (lower than 95% plan assumption), subscription tokens scale down symmetrically + // Metered quota: the vendor delivers the same tokens no matter what the + // user's own cache hit rate is (VULN-02 regression guard). const result75 = computeApplesToApples([testModel], [testPlan], 'all', 100, 0, 0.75); const sub75 = result75.options.find(o => o.type === 'subscription'); - - // When cacheRate is 0% (fresh tokens, no cache), subscription tokens scale down even further const result0 = computeApplesToApples([testModel], [testPlan], 'all', 100, 0, 0.0); const sub0 = result0.options.find(o => o.type === 'subscription'); expect(sub75).toBeDefined(); expect(sub0).toBeDefined(); - // 0% cache delivers fewer tokens than 75% cache for the subscription + expect(sub75!.monthlyTokens).toBe(1000); + expect(sub0!.monthlyTokens).toBe(1000); + }); + + it('cache-scales dollar-credit pools whose yield depends on per-turn cost', () => { + const testModel = model({ + id: 'z-ai/glm-5.3', + name: 'GLM-5.3', + pricing: { input: 1.0, output: 2.0, cachedInput: 0.1, cachedInputWrite: null, reasoning: null, webSearch: null }, + blendedCost: 1.25, + benchmarks: { codingIndex: 75, intelligenceIndex: 75, agenticIndex: 75, valueScore: 200 }, + }); + + const creditTier = tier({ + name: 'Pool', + monthlyPrice: 80, + limits: { monthlyCredits: '$80/mo usage value credits' }, + models: ['GLM-5.3'], + estimatedTokenBudget: { estimatedMillionTokens: 1000, assumptions: '95% cache hit rate' }, + }); + const creditPlan = plan({ id: 'credit-test', name: 'CreditPool', tiers: [creditTier] }); + + const result0 = computeApplesToApples([testModel], [creditPlan], 'all', 100, 0, 0.0); + const result75 = computeApplesToApples([testModel], [creditPlan], 'all', 100, 0, 0.75); + const sub0 = result0.options.find(o => o.type === 'subscription'); + const sub75 = result75.options.find(o => o.type === 'subscription'); + + expect(sub0).toBeDefined(); + expect(sub75).toBeDefined(); + // A dollar pool buys fewer tokens when turns are priced without cache discounts. expect(sub0!.monthlyTokens).toBeLessThan(sub75!.monthlyTokens); - // 75% cache delivers fewer tokens than the 95% marketing estimate (1000M) expect(sub75!.monthlyTokens).toBeLessThan(1000); }); - it('parsePlanCacheAssumption correctly parses declared cache hit percentages', () => { - const tier95 = tier({ + it('parsePlanCacheAssumption ignores discount percentages and anchors on cache context', () => { + const discountFirst = tier({ estimatedTokenBudget: { estimatedMillionTokens: 100, - assumptions: '95% cache hit rate', - estimateMeta: { cacheAssumption: '95% cache hit rate', sourceUrl: 'test', sourceType: 'official', confidence: 'high', verifiedAt: '2026-09-19' }, + assumptions: 'At 90% discount, 75% cache hit rate', + estimateMeta: { cacheAssumption: 'At 90% discount, 75% cache hit rate', sourceUrl: 'test', sourceType: 'official', confidence: 'high', verifiedAt: '2026-09-19' }, }, }); - expect(parsePlanCacheAssumption(tier95)).toBe(0.95); + expect(parsePlanCacheAssumption(discountFirst)).toBe(0.75); - const tier75 = tier({ + const discountOnly = tier({ estimatedTokenBudget: { estimatedMillionTokens: 100, - assumptions: '75% cache hit rate', + assumptions: '50% off-peak discount applies', }, }); - expect(parsePlanCacheAssumption(tier75)).toBe(0.75); + // No cache-context percent: fall back to the documented default, not 0.5. + expect(parsePlanCacheAssumption(discountOnly)).toBe(0.75); + }); - const tierDefault = tier(); - expect(parsePlanCacheAssumption(tierDefault)).toBe(0.75); + it('discloses spend asymmetry when the API leg outspends a capped subscription in the arbitrage callout', () => { + const sonnetModel = model({ + id: 'anthropic/claude-sonnet-5', + name: 'Claude Sonnet 5', + benchmarks: { codingIndex: 85, intelligenceIndex: null, agenticIndex: null, valueScore: null }, + }); + const cheapSub = plan({ + id: 'cheap-sub', + name: 'Cheap Sub', + tiers: [tier({ monthlyPrice: 20, models: ['Claude Sonnet 5'], estimatedTokenBudget: { estimatedMillionTokens: 1, assumptions: 'test' } })], + }); + + const result = computeApplesToApples([sonnetModel], [cheapSub], 'all', 200, 0, 0.75); + expect(result.arbitrageCallout).not.toBeNull(); + expect(result.arbitrageCallout!.description).toContain('spends $20/mo vs $200/mo on the API leg'); }); }); diff --git a/src/lib/pricing.ts b/src/lib/pricing.ts index aefc658..7df7415 100644 --- a/src/lib/pricing.ts +++ b/src/lib/pricing.ts @@ -89,7 +89,9 @@ export function calculateBudgetResults( // yields keep the legacy 3K (2K-in/1K-out) request definition. const requests1k = blendMode === 'agentic' ? (millionTokens * 1_000_000) / STANDARD_AGENT_REQUEST_TOKENS - : (budget / (m.costPer1kRequests || 1)) * 1000; + // Nullish coalescing (never `||`): a legitimate $0/M request cost must + // not be replaced by the fallback divisor (invariant I). + : (budget / (m.costPer1kRequests ?? 1)) * 1000; return { modelId: m.id, @@ -265,20 +267,43 @@ export function matchesPlanModel(planModelName: string, model: NormalizedModel): */ export function parsePlanCacheAssumption(tier: PlanTier): number { const metaStr = tier.estimatedTokenBudget?.estimateMeta?.cacheAssumption; + // Anchor the percent to explicit cache context so discount percentages + // ("90% discount", "50% off-peak") cannot hijack the cache rate. if (metaStr) { - const match = metaStr.match(/([0-9]+)%/); - if (match) { - return parseInt(match[1], 10) / 100; + const anchored = metaStr.match(/([0-9]+)\s*%\s*(?:cache|hit)/i) || metaStr.match(/cache[^.]*?([0-9]+)\s*%/i); + if (anchored) { + return clampCacheAssumption(parseInt(anchored[1], 10) / 100); } } const assumptions = tier.estimatedTokenBudget?.assumptions; if (assumptions) { - const match = assumptions.match(/([0-9]+)%\s*cache/i); - if (match) { - return parseInt(match[1], 10) / 100; + const anchored = assumptions.match(/([0-9]+)\s*%\s*(?:cache|hit)/i) || assumptions.match(/cache[^.]*?([0-9]+)\s*%/i); + if (anchored) { + return clampCacheAssumption(parseInt(anchored[1], 10) / 100); } } - return DEFAULT_CACHE_RATE; + return clampCacheAssumption(DEFAULT_CACHE_RATE); +} + +/** + * Plans whose tiers meter a fixed dollar-credit pool scale their token yield + * with the effective cost of the drained model (spend X credits -> fewer + * tokens on pricier turns). Usage/request-metered quotas grant the same + * tokens regardless of the user's cache hit rate, so they must NOT be scaled. + */ +export function isDollarPoolTier(tier: PlanTier): boolean { + if (tier.perModelTokenBudgets) { + const bases = Object.values(tier.perModelTokenBudgets).map(e => e?.basis); + if (bases.some(b => b === 'list-price-credit' || b === 'equal-rate')) return true; + } + const limits = tier.limits ?? {}; + return Object.keys(limits).some(k => k.toLowerCase().includes('credit')); +} + +function clampCacheAssumption(rate: number): number { + if (!Number.isFinite(rate) || rate < 0) return DEFAULT_CACHE_RATE; + // Deep-agent workloads rarely sustain > 95% cache hits; treat anything above as data noise. + return Math.min(rate, 0.95); } export function calculateAgentRequestCost( @@ -430,11 +455,18 @@ export function computeApplesToApples( const resolved = resolveTierModelBudget(tier, model.name, model.id, planModelName); const rawBaseTokens = resolved.tokens || 0; - // Symmetric cache scaling: dynamically adjust subscription capacity with active cacheRate + // Cache scaling only applies to dollar-credit pools: the same monthly + // spend buys fewer tokens when turns are priced at lower cache-hit + // rates. Usage/request-metered quotas deliver fixed tokens regardless + // of the user's cache rate, so they are left unscaled (VULN-02). + const cacheScaleApplies = isDollarPoolTier(tier); const planAssumedCache = parsePlanCacheAssumption(tier); - const turnCostAssumed = calculateAgentRequestCost(model, planAssumedCache as CacheRate).costPerRequest; - const turnCostUser = calculateAgentRequestCost(model, cacheRate).costPerRequest; - const cacheScaleFactor = (turnCostAssumed > 0 && turnCostUser > 0) ? (turnCostAssumed / turnCostUser) : 1.0; + let cacheScaleFactor = 1.0; + if (cacheScaleApplies) { + const turnCostAssumed = calculateAgentRequestCost(model, planAssumedCache as CacheRate).costPerRequest; + const turnCostUser = calculateAgentRequestCost(model, cacheRate).costPerRequest; + cacheScaleFactor = (turnCostAssumed > 0 && turnCostUser > 0) ? (turnCostAssumed / turnCostUser) : 1.0; + } const baseTokens = Math.round(rawBaseTokens * cacheScaleFactor); const rawRequests = tierRawRequests(tier, baseTokens); @@ -499,8 +531,18 @@ export function computeApplesToApples( if (tier.monthlyPrice > budget) continue; const rawBaseTokens = tier.estimatedTokenBudget?.estimatedMillionTokens || 0; + // Same dollar-pool gate as matched tiers so a plan never scales in + // opposite directions depending on model-name matching (VULN-08). const planAssumedCache = parsePlanCacheAssumption(tier); - const cacheScaleFactor = (1 - planAssumedCache * 0.8) / (1 - cacheRate * 0.8); + let cacheScaleFactor = 1.0; + if (isDollarPoolTier(tier)) { + const basisModel = cleanModels.find(m => detectModelLab(m) === (detectPlanLabs(plan, tier)[0] ?? 'all')) ?? cleanModels[0]; + if (basisModel) { + const turnCostAssumed = calculateAgentRequestCost(basisModel, planAssumedCache as CacheRate).costPerRequest; + const turnCostUser = calculateAgentRequestCost(basisModel, cacheRate).costPerRequest; + cacheScaleFactor = (turnCostAssumed > 0 && turnCostUser > 0) ? (turnCostAssumed / turnCostUser) : 1.0; + } + } const baseTokens = Math.round(rawBaseTokens * (cacheScaleFactor > 0 ? cacheScaleFactor : 1.0)); const rawRequests = tierRawRequests(tier, baseTokens); const vendorQuota = parseTierRequestLimit(tier.limits); @@ -610,13 +652,26 @@ export function computeApplesToApples( ? `nearest-quality API match (CI ±${qualityGap.toFixed(1)})` : 'nearest available API match'; + // Spend parity: a single-seat subscription spends only its tier price + // while the API leg spends the full budget. Surface the asymmetry + // whenever the API leg spends meaningfully more (VULN-09). + const subSpend = bestSubscription.monthlyCost; + const apiSpend = apiOption.monthlyCost; + const spendAsymmetry = apiSpend > subSpend * 1.25 + ? ` Subscription spends $${subSpend.toFixed(0)}/mo vs $${apiSpend.toFixed(0)}/mo on the API leg${ + isFinite(apiSpend / (subSpend || 1)) + ? ` (${(apiSpend / (subSpend || 1)).toFixed(1)}x the spend — use Dave Mode or Mix & Match to equalize)` + : '' + }` + : ''; + if (subTokens >= apiTokens * 1.25) { const multiplier = Number((subTokens / (apiTokens || 1)).toFixed(1)); arbitrageCallout = { winnerType: 'subscription', multiplier, headline: `Coding Subscription Wins for ${labName} (${multiplier}x Compute)`, - description: `${bestSubscription.planName} (${bestSubscription.tierName}) running ${bestSubscription.name} yields ~${formatMillionTokens(subTokens)} normalized tokens (~${bestSubscription.monthlyRequests.toLocaleString()} requests), beating direct ${apiOption.name} API (~${formatMillionTokens(apiTokens)} tokens) for $${budget}/mo. Compared as ${qualityNote}.`, + description: `${bestSubscription.planName} (${bestSubscription.tierName}) running ${bestSubscription.name} yields ~${formatMillionTokens(subTokens)} normalized tokens (~${bestSubscription.monthlyRequests.toLocaleString()} requests), beating direct ${apiOption.name} API (~${formatMillionTokens(apiTokens)} tokens) for $${budget}/mo. Compared as ${qualityNote}.${spendAsymmetry}`, }; } else if (apiTokens >= subTokens * 1.25) { const multiplier = Number((apiTokens / (subTokens || 1)).toFixed(1)); @@ -624,14 +679,14 @@ export function computeApplesToApples( winnerType: 'api', multiplier, headline: `Direct API Wins for ${labName} (${multiplier}x Compute)`, - description: `Direct pay-per-token API for ${apiOption.name} delivers ~${formatMillionTokens(apiTokens)} tokens (~${apiOption.monthlyRequests.toLocaleString()} requests), outperforming coding subscriptions (~${formatMillionTokens(subTokens)} normalized tokens) for $${budget}/mo. Compared as ${qualityNote}.`, + description: `Direct pay-per-token API for ${apiOption.name} delivers ~${formatMillionTokens(apiTokens)} tokens (~${apiOption.monthlyRequests.toLocaleString()} requests), outperforming coding subscriptions (~${formatMillionTokens(subTokens)} normalized tokens) for $${budget}/mo. Compared as ${qualityNote}.${spendAsymmetry}`, }; } else { arbitrageCallout = { winnerType: 'even', multiplier: 1.0, headline: `Balanced Value for ${labName}`, - description: `Both direct API (${apiOption.name}) and subscriptions (${bestSubscription.planName} running ${bestSubscription.name}) offer comparable compute near ${formatMillionTokens(subTokens)} tokens for $${budget}/mo. Compared as ${qualityNote}.`, + description: `Both direct API (${apiOption.name}) and subscriptions (${bestSubscription.planName} running ${bestSubscription.name}) offer comparable compute near ${formatMillionTokens(subTokens)} tokens for $${budget}/mo. Compared as ${qualityNote}.${spendAsymmetry}`, }; } } @@ -686,11 +741,13 @@ export function parseTierRequestLimit(limits?: Record): number return Math.round(parseFloat(kMixMatch[1]) * 1000); } - // 3. Weekly requests scaled x4: e.g. "12,000/week", "1,000 requests per 7-day" + // 3. Weekly requests scaled to a monthly equivalent (52/12 weeks per + // month). Weekly quotas usually coexist with undocumented monthly + // ceilings, so scale conservatively (VULN-12). const weekMatch = val.match(/(?:≈|~)?\s*([0-9]+(?:,[0-9]+)*(?:\.[0-9]+)?)\s*(?:agentic\s*)?requests?\s*(?:\/|\s*per\s*)\s*(?:week|7-day)/i) || val.match(/([0-9]+(?:,[0-9]+)*)\s*\/\s*week/i); if (weekMatch) { - return Math.round(parseFloat(weekMatch[1].replace(/,/g, '')) * 4); + return Math.round(parseFloat(weekMatch[1].replace(/,/g, '')) * (52 / 12)); } // 4. Simple leading count: e.g. "50 agentic requests/mo", "50 requests" diff --git a/src/lib/reasoning.test.ts b/src/lib/reasoning.test.ts index d467429..46ce690 100644 --- a/src/lib/reasoning.test.ts +++ b/src/lib/reasoning.test.ts @@ -136,3 +136,56 @@ describe('Extended Thinking & Reasoning Token Engine', () => { expect(anthropicApi?.absorptionScore).toBeLessThanOrEqual(40); }); }); + +describe('Reasoning feasibility guard (VULN-06)', () => { + const baseModel: NormalizedModel = { + id: 'anthropic/claude-sonnet-5', + name: 'Claude Sonnet 5', + provider: 'anthropic', + series: 'Claude', + modality: 'text->text', + contextWindow: 200000, + maxOutput: 64000, + pricing: { input: 3.0, output: 15.0, cachedInput: 0.3, cachedInputWrite: 3.75, reasoning: null, webSearch: null }, + blendedCost: 6.0, + agentBlendedCost: 1.5, + costPer1kRequests: 30.0, + benchmarks: { intelligenceIndex: 88, codingIndex: 85, agenticIndex: 84, valueScore: 78 }, + reasoning: null, + isFree: false, + isBatch: false, + }; + + const tinyModel: NormalizedModel = { + ...baseModel, + id: 'tiny/small-model', + name: 'Tiny Small Model', + contextWindow: 32000, + maxOutput: 8000, + }; + + it('rejects max effort on a 32K-context model as an impossible combination', () => { + const turn = calculateReasoningCost(tinyModel, 0.75, 'max'); + expect(turn.feasible).toBe(false); + expect(turn.infeasibilityReason).toContain('32,000'); + }); + + it('rejects reasoning that exceeds the max completion cap even when context fits', () => { + const cappedModel: NormalizedModel = { ...baseModel, contextWindow: 200000, maxOutput: 8000 }; + const turn = calculateReasoningCost(cappedModel, 0.75, 'high'); + expect(turn.feasible).toBe(false); + expect(turn.infeasibilityReason).toContain('caps output'); + }); + + it('keeps medium effort feasible on the frontier model', () => { + const turn = calculateReasoningCost(baseModel, 0.75, 'medium'); + expect(turn.feasible).toBe(true); + expect(turn.infeasibilityReason).toBeNull(); + }); + + it('treats unknown windows (0) as feasible rather than fabricating a constraint', () => { + const unknown = { ...baseModel, contextWindow: 0, maxOutput: 0 }; + const turn = calculateReasoningCost(unknown, 0.75, 'max'); + expect(turn.feasible).toBe(true); + }); +}); diff --git a/src/lib/reasoning.ts b/src/lib/reasoning.ts index 52ba353..c1c3f00 100644 --- a/src/lib/reasoning.ts +++ b/src/lib/reasoning.ts @@ -56,6 +56,10 @@ export interface ReasoningTurnCost { totalCost: number; reasoningTokens: number; inflationMultiplier: number; // totalCost / baseCost + /** True when input + reasoning + output fit the model's context and output caps. */ + feasible: boolean; + /** Human-readable reason when infeasible, else null. */ + infeasibilityReason: string | null; } export interface MonthlyReasoningWorkload { @@ -214,6 +218,22 @@ export function calculateReasoningCost( const totalCost = baseCost + reasoningCost; const inflationMultiplier = baseCost > 0 ? totalCost / baseCost : 1.0; + // Feasibility guard (VULN-06): input + reasoning + output must fit inside + // the model's context window, and reasoning + output must fit its max + // completion cap. Unknown caps (0) are treated as feasible. + let feasible = true; + let infeasibilityReason: string | null = null; + const requiredContext = AGENT_REQUEST_INPUT_TOKENS + reasoningTokens + AGENT_REQUEST_OUTPUT_TOKENS; + if (model.contextWindow > 0 && requiredContext > model.contextWindow) { + feasible = false; + infeasibilityReason = `${effortSpec.label} reasoning needs ${requiredContext.toLocaleString()} context tokens but ${model.name} only supports ${model.contextWindow.toLocaleString()}.`; + } + const requiredOutput = reasoningTokens + AGENT_REQUEST_OUTPUT_TOKENS; + if (feasible && model.maxOutput > 0 && requiredOutput > model.maxOutput) { + feasible = false; + infeasibilityReason = `${effortSpec.label} reasoning generates ${requiredOutput.toLocaleString()} completion tokens but ${model.name} caps output at ${model.maxOutput.toLocaleString()}.`; + } + return { baseCost, inputCost, @@ -222,6 +242,8 @@ export function calculateReasoningCost( totalCost, reasoningTokens, inflationMultiplier: Number(inflationMultiplier.toFixed(2)), + feasible, + infeasibilityReason, }; } diff --git a/src/lib/receipt-math.ts b/src/lib/receipt-math.ts index 1aca8e1..6cff4c2 100644 --- a/src/lib/receipt-math.ts +++ b/src/lib/receipt-math.ts @@ -38,6 +38,11 @@ export interface SessionReceiptReport { modelRepricings: ModelRepricingItem[]; subscriptionImpacts: SubscriptionImpactItem[]; headlineSummary: string; + /** + * False when the parser synthesized the token counts (VULN-05): the receipt + * must be presented as an estimate, not a measured bill. + */ + tokensMeasured: boolean; } /** @@ -173,9 +178,11 @@ export function calculateSessionReceipt( ]; const cheapest = modelRepricings[0]; - const headlineSummary = cheapest && cheapest.sessionCost < baselineTotalCost + const tokensMeasured = session.measuredTokens !== false; + const estimatePrefix = tokensMeasured ? '' : 'ESTIMATED (transcript contained no token accounting — parser assumed ~21K in / 900 out per turn at 75% cache). '; + const headlineSummary = estimatePrefix + (cheapest && cheapest.sessionCost < baselineTotalCost ? `This session cost $${baselineTotalCost.toFixed(2)} on ${baselineModel?.name || 'Sonnet 5'}. It would have cost $${cheapest.sessionCost.toFixed(2)} on ${cheapest.modelName} (${cheapest.savingsPercentVsBaseline}% cheaper).` - : `This session executed ${session.totalTurns} turns across ${(session.totalTokens / 1e6).toFixed(2)}M tokens at a ${Math.round(session.effectiveCacheHitRate * 100)}% prompt cache hit rate.`; + : `This session executed ${session.totalTurns} turns across ${(session.totalTokens / 1e6).toFixed(2)}M tokens at a ${Math.round(session.effectiveCacheHitRate * 100)}% prompt cache hit rate.`); return { session, @@ -185,5 +192,6 @@ export function calculateSessionReceipt( modelRepricings, subscriptionImpacts, headlineSummary, + tokensMeasured, }; } diff --git a/src/lib/teams.ts b/src/lib/teams.ts index 9ffc254..ccbc70b 100644 --- a/src/lib/teams.ts +++ b/src/lib/teams.ts @@ -175,7 +175,10 @@ function computePersonaApiCost(p: DeveloperPersona, model?: NormalizedModel): nu if (!model) return p.estimatedDirectApiCost; const inPrice = model.pricing.input ?? model.blendedCost * 0.75; const outPrice = model.pricing.output ?? model.blendedCost * 1.75; - const cachedPrice = model.pricing.cachedInput ?? inPrice * 0.1; + // Never fabricate a discount for an unknown cache price (invariant I): + // an unknown cache rate is priced at full input, matching + // getEffectiveCacheMultiplier's "no caching" convention. + const cachedPrice = model.pricing.cachedInput ?? inPrice; const { costPerRequest } = calculateAgentRequestCost(model, 0.75); const agentCost = p.monthlyAgentTurns * costPerRequest; diff --git a/src/lib/throttle-profiles.ts b/src/lib/throttle-profiles.ts index 6ef3321..5d4be71 100644 --- a/src/lib/throttle-profiles.ts +++ b/src/lib/throttle-profiles.ts @@ -1,5 +1,6 @@ export type ThrottleExhaustionBehavior = 'hard-block' | 'slow-queue' | 'payg-overage'; -export type ThrottleStatus = 'smooth' | 'queued' | 'blocked'; +/** 'overage' = served beyond quota as paid overage (payg-overage profiles, VULN-03). */ +export type ThrottleStatus = 'smooth' | 'queued' | 'overage' | 'blocked'; export interface ThrottleProfile { id: string; diff --git a/src/lib/throttle.test.ts b/src/lib/throttle.test.ts index 3fa7abb..c46f187 100644 --- a/src/lib/throttle.test.ts +++ b/src/lib/throttle.test.ts @@ -136,3 +136,36 @@ describe('Throttle Simulation Engine', () => { } }); }); + +describe('Payg-overage simulation (VULN-03)', () => { + const getProfile = (id: string): ThrottleProfile => { + const found = THROTTLE_PROFILES.find((p) => p.id === id); + if (!found) throw new Error(`Profile ${id} not found`); + return found; + }; + + it('flags Google AI Pro as paid overage instead of infinitely smooth under saturating load', () => { + const googlePro = getProfile('google-ai-pro'); + // 8 agents at rapid cadence burn through the 90-turn / 5h window fast. + const result = simulateSprintThrottle(googlePro, { + concurrency: 8, + sprintDurationHours: 4, + turnPace: 'rapid', + }); + + expect(result.status).toBe('overage'); + expect(result.overageTurns).toBeGreaterThan(0); + expect(result.throttleReason).not.toBeNull(); + }); + + it('never shows overage for plans that simply survive the sprint', () => { + const claudeCode = getProfile('claude-code-pro'); + const result = simulateSprintThrottle(claudeCode, { + concurrency: 1, + sprintDurationHours: 2, + turnPace: 'deep', + }); + expect(result.status).toBe('smooth'); + expect(result.overageTurns).toBe(0); + }); +}); diff --git a/src/lib/throttle.ts b/src/lib/throttle.ts index bde1b33..520570c 100644 --- a/src/lib/throttle.ts +++ b/src/lib/throttle.ts @@ -65,6 +65,8 @@ export interface SimulationResult { fastTurnsCompleted: number; slowTurnsCompleted: number; blockedTurns: number; + /** Turns billed as paid overage when a payg-overage profile crossed its quota ceiling. */ + overageTurns: number; effectiveThroughputPercent: number; headroomScore: number; // 0 to 100 timeline: SimulationTimelinePoint[]; @@ -78,6 +80,9 @@ function formatMinutesToHours(minutes: number): string { return `${h}h ${m}m`; } +/** Severity rank so the sprint's overall status always reflects the worst outcome. */ +const STATUS_SEVERITY: Record = { smooth: 0, queued: 1, overage: 2, blocked: 3 }; + /** * Runs a discrete time-step simulation (step = 5 minutes) of an intensive coding sprint * against a given throttle profile. @@ -118,6 +123,11 @@ export function simulateSprintThrottle( let slowTurnsCompleted = 0; let blockedTurns = 0; let cumulativeTurns = 0; + // Deferred turns from prior queue delays (VULN-12): a queued step cannot + // magically complete its full turn allocation; the backlog spills into + // later steps and reduces throughput realistically. + let deferredTurns = 0; + let overageTurns = 0; const timeline: SimulationTimelinePoint[] = [ { @@ -134,10 +144,28 @@ export function simulateSprintThrottle( const hasConcurrencyViolation = safeConcurrency > profile.maxConcurrency; for (let currentMinute = stepMinutes; currentMinute <= totalMinutes; currentMinute += stepMinutes) { - const turnsAttempted = turnsPerStep; + // Queue-delay backlog reduces the turns actually executable this step. + // A 30s/turn slow-queue delay over a 5-minute step caps the step at + // floor(step / delay) full-speed-equivalent allocations; the deficit is + // deferred, not silently completed (VULN-12). + let turnsAttempted = turnsPerStep; let stepStatus: ThrottleStatus = 'smooth'; let stepDelay = 0; + if (deferredTurns > 0) { + if (profile.exhaustionBehavior === 'hard-block') { + // Blocked turns never recover; drop the backlog. + deferredTurns = 0; + } else if (profile.slowQueueDelaySec > 0) { + const executableRatio = Math.min(1, (stepMinutes * 60) / (profile.slowQueueDelaySec * turnsPerStepPerAgent * safeConcurrency * safeConcurrency)); + const executable = Math.max(1, Math.floor(turnsPerStep * executableRatio)); + turnsAttempted = Math.min(turnsPerStep, executable); + deferredTurns = Math.max(0, deferredTurns - (turnsPerStep - turnsAttempted)); + stepStatus = 'queued'; + stepDelay = profile.slowQueueDelaySec; + } + } + // 1. Check instantaneous concurrency if (hasConcurrencyViolation) { if (profile.exhaustionBehavior === 'hard-block') { @@ -176,6 +204,12 @@ export function simulateSprintThrottle( if (!firstThrottleReason) { firstThrottleReason = `Rolling ${profile.rollingWindowHours}h window exceeded: degraded to slow queue`; } + } else if (profile.exhaustionBehavior === 'payg-overage') { + // PAYG profiles never block; excess turns bill as paid overage (VULN-03). + stepStatus = 'overage'; + if (!firstThrottleReason) { + firstThrottleReason = `Rolling ${profile.rollingWindowHours}h window exceeded: excess turns continue as paid overage`; + } } } } @@ -194,6 +228,11 @@ export function simulateSprintThrottle( if (!firstThrottleReason) { firstThrottleReason = `Monthly fast pool exhausted (${profile.monthlyFastRequests} reqs): sprint blocked`; } + } else if (profile.exhaustionBehavior === 'payg-overage') { + if (!firstThrottleReason) { + firstThrottleReason = `Monthly pool exhausted (${profile.monthlyFastRequests} reqs): excess turns continue as paid overage`; + } + stepStatus = 'overage'; } } } @@ -201,25 +240,53 @@ export function simulateSprintThrottle( // Process turns for this step if (stepStatus === 'blocked') { blockedTurns += turnsAttempted; - if (firstThrottleMinute === null) { - firstThrottleMinute = currentMinute; + if (STATUS_SEVERITY['blocked'] > STATUS_SEVERITY[overallStatus]) { overallStatus = 'blocked'; } - } else if (stepStatus === 'queued') { - slowTurnsCompleted += turnsAttempted; + if (firstThrottleMinute === null) { + firstThrottleMinute = currentMinute; + } + } else if (stepStatus === 'overage') { + // Served beyond quota — turns complete but bill as paid overage (VULN-03). + overageTurns += turnsAttempted; cumulativeTurns += turnsAttempted; rollingWindowHistory.push({ minute: currentMinute, turns: turnsAttempted }); if (remainingMonthlyFast !== null) { remainingMonthlyFast = Math.max(0, remainingMonthlyFast - turnsAttempted); } + if (STATUS_SEVERITY['overage'] > STATUS_SEVERITY[overallStatus]) { + overallStatus = 'overage'; + } + if (firstThrottleMinute === null) { + firstThrottleMinute = currentMinute; + } + } else if (stepStatus === 'queued') { + // Queue degradation defers the un-executable share of this step's turns. + const deferred = Math.max(0, turnsAttempted - Math.max(1, Math.floor(turnsPerStep * 0.7))); + deferredTurns += deferred; + slowTurnsCompleted += turnsAttempted; + cumulativeTurns += turnsAttempted; + if (profile.exhaustionBehavior === 'payg-overage') { + overageTurns += turnsAttempted; + } + rollingWindowHistory.push({ minute: currentMinute, turns: turnsAttempted }); + if (remainingMonthlyFast !== null) { + remainingMonthlyFast = Math.max(0, remainingMonthlyFast - turnsAttempted); + } + if (STATUS_SEVERITY['queued'] > STATUS_SEVERITY[overallStatus]) { + overallStatus = 'queued'; + } if (firstThrottleMinute === null) { firstThrottleMinute = currentMinute; - overallStatus = overallStatus === 'blocked' ? 'blocked' : 'queued'; } } else { // Smooth step fastTurnsCompleted += turnsAttempted; cumulativeTurns += turnsAttempted; + if (profile.exhaustionBehavior === 'payg-overage' && firstThrottleReason !== null) { + // Once a PAYG ceiling was crossed, all subsequent turns bill as overage. + overageTurns += turnsAttempted; + } rollingWindowHistory.push({ minute: currentMinute, turns: turnsAttempted }); if (remainingMonthlyFast !== null) { remainingMonthlyFast = Math.max(0, remainingMonthlyFast - turnsAttempted); @@ -278,6 +345,7 @@ export function simulateSprintThrottle( blockedTurns: Math.round(blockedTurns), effectiveThroughputPercent: Math.min(100, Math.max(0, effectiveThroughputPercent)), headroomScore: Math.min(100, Math.max(0, headroomScore)), + overageTurns: Math.round(overageTurns), timeline, }; } @@ -289,12 +357,12 @@ export function simulateAllProfiles(params: SprintSimulationParams): SimulationR const results = THROTTLE_PROFILES.map((profile) => simulateSprintThrottle(profile, params)); // Sort order: - // 1. Smooth before Queued before Blocked + // 1. Smooth before Queued before Overage before Blocked // 2. Higher headroom score // 3. Longer survival hours // 4. Lower monthly price return results.sort((a, b) => { - const statusWeight = { smooth: 3, queued: 2, blocked: 1 }; + const statusWeight = { smooth: 4, queued: 3, overage: 2, blocked: 1 }; if (statusWeight[a.status] !== statusWeight[b.status]) { return statusWeight[b.status] - statusWeight[a.status]; } diff --git a/src/lib/tos.test.ts b/src/lib/tos.test.ts index 7ee75be..f9b7634 100644 --- a/src/lib/tos.test.ts +++ b/src/lib/tos.test.ts @@ -58,16 +58,21 @@ describe('classifyTraining', () => { }); describe('getIndemnityInfo', () => { - it('labels boolean true as full indemnity', () => { - expect(getIndemnityInfo(plan({ ipIndemnity: true })).label).toBe('Full Indemnity'); + it('never labels boolean true as unconditional "Full Indemnity" (VULN-10)', () => { + const info = getIndemnityInfo(plan({ ipIndemnity: true })); + expect(info.label).toBe('Indemnity Offered'); + expect(info.tone).toBe('warning'); + expect(info.description).toContain('carve-outs'); }); it('labels z-ai-style strings as enterprise only', () => { expect(getIndemnityInfo(plan({ ipIndemnity: 'Enterprise Team contracts only' })).label).toBe('Enterprise Only'); }); - it('labels an unknown string as unclear rather than none', () => { - expect(getIndemnityInfo(plan({ ipIndemnity: 'Full indemnity on all paid plans' })).label).toBe('Full Indemnity'); + it('flags indemnity strings with a verify-scope caveat rather than promising full coverage', () => { + const info = getIndemnityInfo(plan({ ipIndemnity: 'Full indemnity on all paid plans' })); + expect(info.label).toBe('Indemnity Stated (verify scope)'); + expect(info.tone).toBe('warning'); }); }); diff --git a/src/lib/tos.ts b/src/lib/tos.ts index 47f01d8..3b9e343 100644 --- a/src/lib/tos.ts +++ b/src/lib/tos.ts @@ -48,7 +48,8 @@ const TRAINS_PHRASES = [ 'used for model improvement', 'interactions are recorded and used', 'may be used for model improvement', - 'no guarantee', + // 'no guarantee' removed (VULN-10): hedges about uptime/confidentiality are + // not evidence that a vendor trains on customer data. ]; const OPT_OUT_PHRASES = ['opt-out', 'opt out', 'privacy mode', 'training opt-out', 'in-settings opt-out']; @@ -152,9 +153,13 @@ export function getIndemnityInfo(plan: CodingPlan): IndemnityInfo { const ip = plan.ipIndemnity; if (ip === true) { return { - label: 'Full Indemnity', - tone: 'success', - description: 'Full legal IP indemnification covered for copyright infringement on generated code.', + // VULN-10: a boolean cannot establish unconditional coverage. Real + // indemnities carry carve-outs (modification, combination, breach of + // usage policy) and liability caps — never present this as "Full". + label: 'Indemnity Offered', + tone: 'warning', + description: + 'Vendor advertises IP indemnification. Scope, carve-outs and liability caps are defined in the vendor’s legal terms — verify before relying on this for compliance.', }; } if (typeof ip === 'string') { @@ -163,7 +168,11 @@ export function getIndemnityInfo(plan: CodingPlan): IndemnityInfo { return { label: 'Enterprise Only', tone: 'warning', description: ip }; } if (lower.includes('full') || lower.includes('all plans')) { - return { label: 'Full Indemnity', tone: 'success', description: ip }; + return { + label: 'Indemnity Stated (verify scope)', + tone: 'warning', + description: ip, + }; } return { label: 'Unclear', tone: 'muted', description: ip }; } @@ -182,7 +191,10 @@ export interface GotchaClassification { } const RESTRICTION_WORDS = ['ban', 'banned', 'suspend', 'suspension', 'bulk', 'multiple accounts', 'multi-account', 'account sharing', 'unauthorized', 'tool-only', 'lock-in', 'error 1113', 'circumvent', 'breach', 'prohibited']; -const PRIVACY_WORDS = ['train', 'training', 'privacy', 'telemetry', 'human review', 'data retention', 'zero retention']; +/** Strong privacy signals that warrant CRITICAL severity (VULN-10). */ +const PRIVACY_STRONG_WORDS = ['train on', 'trains on', 'trained on', 'used for training', 'used to train', 'human review', 'data retention', 'zero retention', 'not used for training']; +/** Weak privacy mentions (e.g. "training" inside a caching sentence) only justify a warning. */ +const PRIVACY_WEAK_WORDS = ['train', 'training', 'privacy', 'telemetry']; const LEGAL_WORDS = ['indemnity', 'indemnification', 'copyright', 'liability', 'ip ', 'legal']; const BILLING_WORDS = ['bill', 'billing', 'fee', 'fees', 'charge', 'overage', 'refund', 'refunds', 'cost', 'expire', 'expires', 'rate', '$', 'price', 'non-refundable']; const LIMIT_WORDS = ['limit', 'limits', 'throttle', 'cooldown', 'hard limit', 'quota', 'cap', 'resets', 'reset']; @@ -200,9 +212,14 @@ export function classifyGotcha(text: string): GotchaClassification { if (matchesWord(lower, RESTRICTION_WORDS)) { return { category: 'restrictions', severity: 'critical' }; } - if (matchesWord(lower, PRIVACY_WORDS)) { + if (matchesWord(lower, PRIVACY_STRONG_WORDS)) { return { category: 'data-privacy', severity: 'critical' }; } + if (matchesWord(lower, PRIVACY_WEAK_WORDS)) { + // Weak signal: the word "training"/"privacy" appears without an actual + // training-on-data claim (e.g. "Privacy mode keeps code out of training"). + return { category: 'data-privacy', severity: 'warning' }; + } if (matchesWord(lower, LEGAL_WORDS)) { return { category: 'ip-legal', severity: 'warning' }; } diff --git a/src/pages/BurstSimulator.tsx b/src/pages/BurstSimulator.tsx index 1df57df..9c23ce9 100644 --- a/src/pages/BurstSimulator.tsx +++ b/src/pages/BurstSimulator.tsx @@ -6,6 +6,7 @@ import { type TurnCadence, } from '../lib/throttle'; import type { ThrottleStatus } from '../lib/throttle-profiles'; +import { numParam } from '../lib/params'; import { Zap, Gauge, @@ -20,6 +21,7 @@ import { RotateCcw, Sparkles, Info, + DollarSign, } from 'lucide-react'; import { ResponsiveContainer, @@ -74,12 +76,13 @@ const PRESETS: Preset[] = [ export default function BurstSimulator() { const [searchParams, setSearchParams] = useSearchParams(); - // Read initial parameters from URL or use defaults - const initialConcurrency = Math.min(10, Math.max(1, Number(searchParams.get('agents')) || 2)); - const initialDuration = Math.min(8, Math.max(1, Number(searchParams.get('duration')) || 4)); + // Read initial parameters from URL or use defaults (zero-safe: `?prior=0` + // and `?agents=0`-style values must not collapse to defaults, VULN-13) + const initialConcurrency = numParam(searchParams.get('agents'), 2, 1, 10); + const initialDuration = numParam(searchParams.get('duration'), 4, 1, 8); const rawCadence = searchParams.get('cadence') as TurnCadence; const initialCadence: TurnCadence = (rawCadence && CADENCE_DEFINITIONS[rawCadence]) ? rawCadence : 'standard'; - const initialPrior = Math.min(90, Math.max(0, Number(searchParams.get('prior')) || 20)); + const initialPrior = numParam(searchParams.get('prior'), 20, 0, 90); const [concurrency, setConcurrency] = useState(initialConcurrency); const [sprintDuration, setSprintDuration] = useState(initialDuration); @@ -152,6 +155,7 @@ export default function BurstSimulator() { const estimatedTokensM = Number(((totalTurnsAttempted * 21000) / 1000000).toFixed(1)); const smoothCount = simulationResults.filter((r) => r.status === 'smooth').length; const queuedCount = simulationResults.filter((r) => r.status === 'queued').length; + const overageCount = simulationResults.filter((r) => r.status === 'overage').length; const blockedCount = simulationResults.filter((r) => r.status === 'blocked').length; const filteredResults = useMemo(() => { @@ -489,7 +493,7 @@ export default function BurstSimulator() {
Status: {data.status} @@ -509,6 +513,8 @@ export default function BurstSimulator() { ? '#10b981' : entry.status === 'queued' ? '#f59e0b' + : entry.status === 'overage' + ? '#f97316' : '#ef4444' } /> @@ -565,6 +571,17 @@ export default function BurstSimulator() { Queued ({queuedCount}) + ); })}
+ + {/* Infeasible effort warning (VULN-06) */} + {effortFeasibility && !effortFeasibility[effort].feasible && ( +
+ +
+ Impossible combination.{' '} + {effortFeasibility[effort].infeasibilityReason} +
+
+ )} {/* Interactive Controls & Cost KPIs */} diff --git a/src/pages/SessionReceipt.tsx b/src/pages/SessionReceipt.tsx index d2b4025..015785f 100644 --- a/src/pages/SessionReceipt.tsx +++ b/src/pages/SessionReceipt.tsx @@ -21,6 +21,7 @@ import { RotateCcw, Zap, TrendingDown, + AlertTriangle, } from 'lucide-react'; export default function SessionReceipt() { @@ -85,6 +86,7 @@ export default function SessionReceipt() { `================================================`, ` TOKEN-MAX AGENT SESSION RECEIPT `, `================================================`, + receiptReport.tokensMeasured ? null : `*** ESTIMATED: transcript had no token accounting. ***`, `Session: ${activeSession.sessionTitle}`, `Turns Executed: ${activeSession.totalTurns}`, `Tool Invocations: ${activeSession.toolInvocationsCount}`, @@ -212,6 +214,18 @@ export default function SessionReceipt() { {errorMsg} )} + + {!receiptReport.tokensMeasured && ( +
+ +
+ Estimated receipt. This transcript contained no token + accounting — token counts and costs below are synthesized from the standard 21K agent + request shape, not measured from your session. Treat every dollar figure as a rough + planning estimate. +
+
+ )} {/* Session KPI Summary Cards */}