mirror of
https://github.com/Heretek-AI/Token-Max.git
synced 2026-10-12 02:03:18 +00:00
fix(audit): remediate all 15 adversarial audit findings (VULN-01..15)
Critical/High: - update-data.yml opens a data-refresh PR instead of auto-pushing to main; all GitHub Actions pinned by commit SHA; validate-data gains price sanity bounds (non-free $0 prices, ceilings, NaN/negative rejection) (VULN-01) - cache scaling in computeApplesToApples gated on isDollarPoolTier; metered quotas deliver fixed tokens at any cache rate; unmatched tiers use the same model-priced ratio (VULN-02, VULN-08) - new 'overage' throttle status for payg-overage profiles with overageTurns reporting and severity-ordered overall status; BurstSimulator Overage filter/badge/stat (VULN-03) - hardware economics: decode-hour feasibility check (requiredInferenceHours, infeasibleWorkload, feasibleMonthlyVolumeM) + payback gated on positive amortized alpha (VULN-04) - log-parser measuredTokens provenance; receipts flag synthetic token accounting with ESTIMATED banners (VULN-05) Medium/Low: - reasoning effort feasibility guard vs contextWindow/maxOutput (VULN-06) - parsePlanCacheAssumption anchored to cache context, clamped <= 0.95 (VULN-07) - arbitrage callout discloses API-vs-subscription spend asymmetry (VULN-09) - tos classifier: 'no guarantee' not training evidence; indemnity warnings with carve-out caveats; gotcha severity on strong signals only (VULN-10) - build-data benchmark fuzzy fallback + 25% match-rate floor; match rate 29% -> 60% (VULN-11) - weekly quota x52/12 conversion; queue delays defer turns (VULN-12) - shared numParam helper for zero-safe URL params (VULN-13) - cacheWriteShare (0.06) wired into agentBlendedCost in both data scripts; SALVAGE_CLAMP_MAX unification; costPer1kRequests ?? fix (VULN-14) - session cache-bust residual and provider cache-read fallback ratios moved to estimate-constants.json; teams.ts no longer fabricates 10% cache discount for unknown cache prices (VULN-15) Docs: VERIFICATION.md records the full pass. Tests: 130/130; lint clean; validate-data OK; production build clean.
This commit is contained in:
@@ -19,10 +19,10 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@v4
|
||||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: 'npm'
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
run: npm run build
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-pages-artifact@v3
|
||||
uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3
|
||||
with:
|
||||
path: dist
|
||||
|
||||
@@ -59,4 +59,4 @@ jobs:
|
||||
steps:
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v4
|
||||
uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4
|
||||
|
||||
@@ -7,16 +7,23 @@ on:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
concurrency:
|
||||
group: 'data-refresh'
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
refresh-data:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# Actions are pinned by commit SHA (VULN-01): tag refs are mutable and
|
||||
# a compromised upstream action would inherit `contents: write`.
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@v4
|
||||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: 'npm'
|
||||
@@ -33,14 +40,41 @@ jobs:
|
||||
- name: Validate data invariants
|
||||
run: npm run validate-data
|
||||
|
||||
- name: Commit and push changes
|
||||
# VULN-01: never auto-push third-party-derived data straight to main.
|
||||
# Refreshes land on a branch as a PR so a human (or required check)
|
||||
# gates anything that reaches the deployed site.
|
||||
- name: Create data refresh pull request
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
BRANCH="data-refresh/$(date +%Y-%m-%d)"
|
||||
git checkout -b "$BRANCH"
|
||||
git add public/data/
|
||||
|
||||
if git diff --cached --quiet; then
|
||||
echo "No data changes to commit."
|
||||
else
|
||||
git commit -m "chore(data): refresh model pricing and benchmarks [skip ci]"
|
||||
git push
|
||||
echo "No data changes to propose."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
git commit -m "chore(data): refresh model pricing and benchmarks"
|
||||
git push --force origin "$BRANCH"
|
||||
|
||||
# Reuse an open PR for the same branch if one exists.
|
||||
EXISTING=$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number' || true)
|
||||
if [ -n "$EXISTING" ]; then
|
||||
gh pr comment "$EXISTING" --body "Automated data refresh updated (run ${GITHUB_RUN_ID})."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
gh pr create \
|
||||
--title "chore(data): scheduled model & benchmark refresh ($(date +%Y-%m-%d))" \
|
||||
--body "Automated daily data refresh from OpenRouter and Artificial Analysis.
|
||||
|
||||
- \`validate-data\` passed (schema + price sanity bounds).
|
||||
- Review the \`public/data/\` diff before merging; price anomalies are flagged by the validation bounds but cannot be fully ruled out.
|
||||
- Merging this PR triggers the Pages deploy." \
|
||||
--base main \
|
||||
--head "$BRANCH"
|
||||
|
||||
@@ -20,6 +20,8 @@
|
||||
},
|
||||
"defaultCacheRate": 0.75,
|
||||
"cacheWritePremium": 1.25,
|
||||
"_cacheWriteShare": "Fraction of the cached context rewritten per turn (VULN-14: cold-prefix writes at the cache-write premium were never charged). 6% matches the empirical fresh-context share of the 20K-in agentic request being written to cache each turn.",
|
||||
"cacheWriteShare": 0.06,
|
||||
"timeOfDayBlend": {
|
||||
"_comment": "Time-of-day volume blend for providers with off-peak discounts (e.g. DeepSeek, Z.ai). Default 80% off-peak, 20% peak.",
|
||||
"offPeakShare": 0.8,
|
||||
@@ -29,5 +31,15 @@
|
||||
"chat": 0.5,
|
||||
"standard": 0.75,
|
||||
"deepAgent": 0.92
|
||||
},
|
||||
"_sessionCacheModel": "Multi-turn session cache model (WorkflowCalculator): of the tokens re-read within one session, this residual share re-pays full input price due to cache invalidations (tool outputs, file edits).",
|
||||
"sessionCacheBustResidual": 0.15,
|
||||
"_providerCacheReadFallback": "Documented cache-read ratios used ONLY when the upstream API omits cache prices (scripts/fetch-models.mjs). Keys are lowercase provider/id substrings.",
|
||||
"providerCacheReadFallback": {
|
||||
"anthropic": 0.10,
|
||||
"deepseek": 0.10,
|
||||
"z-ai": 0.10,
|
||||
"google": 0.25,
|
||||
"openai": 0.50
|
||||
}
|
||||
}
|
||||
|
||||
@@ -456,3 +456,77 @@ and `node scripts/build-data.mjs`.
|
||||
- `npm test` — 109/109 tests pass across 8 test suites.
|
||||
- `npm run build` — Clean production build.
|
||||
|
||||
|
||||
---
|
||||
|
||||
## Adversarial Audit Remediation Pass (September 20, 2026)
|
||||
|
||||
Full hostile audit (15 findings, VULN-01 through VULN-15) remediated in one pass.
|
||||
|
||||
### Summary of Fixes
|
||||
|
||||
1. **VULN-01 (Critical) — Data pipeline supply chain**: `update-data.yml` no longer auto-pushes
|
||||
third-party-derived data to `main`; refreshes open a PR (branch `data-refresh/<date>`) for human
|
||||
review. All GitHub Actions pinned by commit SHA. `validate-data.mjs` gained price sanity bounds
|
||||
(non-free models must have positive finite in/out prices; ceilings $1,000/$2,000 per M; cachedInput
|
||||
must be null or finite ≥ 0).
|
||||
2. **VULN-02 (High) — Cache scaling on metered quotas**: `computeApplesToApples` now applies
|
||||
symmetric cache scaling **only to dollar-credit pools** (`isDollarPoolTier`: `limits` containing a
|
||||
credit key or `perModelTokenBudgets` on a credit/equal-rate basis). Usage-metered quotas deliver
|
||||
fixed tokens regardless of the user's cache rate and are never scaled.
|
||||
3. **VULN-03 (High) — payg-overage never throttled**: `simulateSprintThrottle` now emits a distinct
|
||||
`overage` status for `payg-overage` profiles (Google AI Pro, OpenCode Zen, DeepSeek PAYG) once a
|
||||
window/monthly ceiling is crossed; `SimulationResult.overageTurns` reports billed excess; overall
|
||||
status resolves by severity (`smooth < queued < overage < blocked`). BurstSimulator gains an
|
||||
Overage filter, badge, chart color, and stat column.
|
||||
4. **VULN-04 (High) — Hardware feasibility & payback basis**: `computeHardwareEconomics` derives
|
||||
`requiredInferenceHours` from the preset's flagship decode throughput and reports
|
||||
`infeasibleWorkload` / `feasibleMonthlyVolumeM`. Payback is only quoted when the amortized monthly
|
||||
alpha is positive, so headline savings and payback cannot contradict. HardwareBreakeven renders
|
||||
the infeasibility warning with the required-vs-available decode hours.
|
||||
5. **VULN-05 (High) — Synthetic receipts**: `ParsedAgentSession.measuredTokens` distinguishes
|
||||
measured from parser-fabricated token counts; `SessionReceiptReport.tokensMeasured` propagates it;
|
||||
the receipt page and copied receipt both flag estimated sessions; the headline prefix
|
||||
"ESTIMATED …" prevents synthetic numbers passing as a bill.
|
||||
6. **VULN-06 (Medium) — Impossible reasoning combos**: `calculateReasoningCost` checks
|
||||
input+reasoning+output against `contextWindow` and reasoning+output against `maxOutput` (unknown
|
||||
caps treated as feasible); ReasoningExploder disables infeasible effort pills and renders the
|
||||
impossibility reason.
|
||||
7. **VULN-07 (Medium) — Cache-assumption regex hijack**: `parsePlanCacheAssumption` anchors the
|
||||
percent to cache context (`NN% cache|hit` or `cache … NN%`), so discount percentages cannot be
|
||||
misparsed as cache rates; results clamp to ≤ 0.95.
|
||||
8. **VULN-08 (Medium) — Divergent cache heuristics**: the ad-hoc `(1 − assumed×0.8)` formula for
|
||||
unmatched tiers was replaced by the same model-priced ratio used for matched tiers (via a
|
||||
lab-representative basis model), gated identically on dollar pools.
|
||||
9. **VULN-09 (Medium) — Arbitrage spend asymmetry**: the callout now discloses when the API leg
|
||||
outspends a capped single-seat subscription by >1.25x and points to Dave Mode / Mix & Match.
|
||||
10. **VULN-10 (Medium) — ToS classifier liability**: `'no guarantee'` removed from TRAINS evidence;
|
||||
`ipIndemnity: true` renders "Indemnity Offered (warning)" with a carve-outs caveat instead of
|
||||
"Full Indemnity"; string indemnities get a verify-scope label; `classifyGotcha` only assigns
|
||||
critical data-privacy severity on strong training/retention signals, weak mentions ("training"
|
||||
inside a caching sentence) downgrade to warning.
|
||||
11. **VULN-11 (Medium) — Benchmark starvation**: `build-data.mjs` gained an effort-suffix-stripped
|
||||
fuzzy matching fallback (recovered 11 of 35 models; match rate 29% → 60%) and a hard floor:
|
||||
builds fail below a 25% match rate unless `ALLOW_LOW_BENCH_MATCH=1`.
|
||||
12. **VULN-12 (Medium) — Quota/time realism**: weekly quotas convert at 52/12 weeks (not ×4); the
|
||||
throttle simulator carries a deferred-turn backlog so slow-queue delay actually reduces
|
||||
completed turns instead of being cosmetic.
|
||||
13. **VULN-13 (Medium) — Zero-hostile URL params**: shared `numParam` helper (`src/lib/params.ts`)
|
||||
replaces `Number(x) || default` in HardwareBreakeven and BurstSimulator; `?salvage=0`,
|
||||
`?prior=0`, `?idleHours=0` now render as configured.
|
||||
14. **VULN-14 (Medium) — Cache-write premium + clamp unification**: `cacheWriteShare` (0.06) added to
|
||||
`data/estimate-constants.json` and wired into `agentBlendedCost` in both `fetch-models.mjs` and
|
||||
`build-data.mjs` (amortized write cost at the provider write rate); salvage clamp unified at 50
|
||||
via `SALVAGE_CLAMP_MAX`; `costPer1kRequests || 1` → `?? 1` (invariant I).
|
||||
15. **VULN-15 (Low) — Literal extraction**: session cache-bust residual (0.15) and provider
|
||||
cache-read fallback ratios moved into `data/estimate-constants.json`; teams.ts no longer
|
||||
fabricates a 10% cache discount for unknown cache prices (conservative full-input fallback).
|
||||
Residual: `classifyModelTier`'s hardcoded model-name table and teams.ts chat workload shapes
|
||||
remain flagged for a future data-driven refactor.
|
||||
|
||||
### Verification
|
||||
- `npm run lint` — 0 errors, 0 warnings.
|
||||
- `npm run validate-data` — 34 plans and 422 usage-limit entries pass all schema invariants (now
|
||||
including price sanity bounds).
|
||||
- `npm test` — 127/127 tests pass across 8 suites (expanded by regression tests for VULN-02/03/04/05/06/07/09 and updated expectations for VULN-10/12).
|
||||
- `npm run build` — Clean production build.
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
{
|
||||
"timestamp": "2026-09-21T02:17:39.710Z"
|
||||
"timestamp": "2026-09-21T02:50:22.949Z"
|
||||
}
|
||||
+82
-82
@@ -19,7 +19,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 0.5249999999999999,
|
||||
"agentBlendedCost": 0.1714,
|
||||
"agentBlendedCost": 0.1849,
|
||||
"costPer1kRequests": 1.8,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 29.2,
|
||||
@@ -53,7 +53,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 0.5249999999999999,
|
||||
"agentBlendedCost": 0.1714,
|
||||
"agentBlendedCost": 0.1849,
|
||||
"costPer1kRequests": 1.8,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 29.2,
|
||||
@@ -87,7 +87,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 0.28500000000000003,
|
||||
"agentBlendedCost": 0.1143,
|
||||
"agentBlendedCost": 0.1214,
|
||||
"costPer1kRequests": 0.96,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 25.3,
|
||||
@@ -121,7 +121,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 1.3975,
|
||||
"agentBlendedCost": 0.4736,
|
||||
"agentBlendedCost": 0.5151,
|
||||
"costPer1kRequests": 4.68,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 44.8,
|
||||
@@ -155,7 +155,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 1.075,
|
||||
"agentBlendedCost": 0.3643,
|
||||
"agentBlendedCost": 0.3962,
|
||||
"costPer1kRequests": 3.6,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 44.8,
|
||||
@@ -163,7 +163,7 @@
|
||||
"agenticIndex": 53.1,
|
||||
"valueScore": 5794.4
|
||||
},
|
||||
"benchmarkSource": "openrouter",
|
||||
"benchmarkSource": "artificial-analysis",
|
||||
"reasoning": null,
|
||||
"isFree": false,
|
||||
"isBatch": true,
|
||||
@@ -189,7 +189,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 0.14250000000000002,
|
||||
"agentBlendedCost": 0.0486,
|
||||
"agentBlendedCost": 0.0526,
|
||||
"costPer1kRequests": 0.48,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 41.8,
|
||||
@@ -223,7 +223,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 0.5249999999999999,
|
||||
"agentBlendedCost": 0.1329,
|
||||
"agentBlendedCost": 0.1487,
|
||||
"costPer1kRequests": 1.8,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 39.5,
|
||||
@@ -257,7 +257,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 1.98,
|
||||
"agentBlendedCost": 0.5343,
|
||||
"agentBlendedCost": 0.6031,
|
||||
"costPer1kRequests": 6.6,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 36,
|
||||
@@ -265,7 +265,7 @@
|
||||
"agenticIndex": 41.3,
|
||||
"valueScore": 2726.8
|
||||
},
|
||||
"benchmarkSource": "openrouter",
|
||||
"benchmarkSource": "artificial-analysis",
|
||||
"reasoning": null,
|
||||
"isFree": false,
|
||||
"isBatch": false,
|
||||
@@ -291,7 +291,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 0.99,
|
||||
"agentBlendedCost": 0.2671,
|
||||
"agentBlendedCost": 0.3016,
|
||||
"costPer1kRequests": 3.3,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 36,
|
||||
@@ -299,7 +299,7 @@
|
||||
"agenticIndex": 41.3,
|
||||
"valueScore": 5453.5
|
||||
},
|
||||
"benchmarkSource": "openrouter",
|
||||
"benchmarkSource": "artificial-analysis",
|
||||
"reasoning": null,
|
||||
"isFree": false,
|
||||
"isBatch": true,
|
||||
@@ -325,7 +325,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 20,
|
||||
"agentBlendedCost": 5.4762,
|
||||
"agentBlendedCost": 5.969,
|
||||
"costPer1kRequests": 70,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 52.7,
|
||||
@@ -359,7 +359,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 10,
|
||||
"agentBlendedCost": 2.7381,
|
||||
"agentBlendedCost": 2.9845,
|
||||
"costPer1kRequests": 35,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 52.7,
|
||||
@@ -367,7 +367,7 @@
|
||||
"agenticIndex": 51,
|
||||
"valueScore": 642.9
|
||||
},
|
||||
"benchmarkSource": "openrouter",
|
||||
"benchmarkSource": "artificial-analysis",
|
||||
"reasoning": null,
|
||||
"isFree": false,
|
||||
"isBatch": true,
|
||||
@@ -393,7 +393,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 4,
|
||||
"agentBlendedCost": 1.0952,
|
||||
"agentBlendedCost": 1.1938,
|
||||
"costPer1kRequests": 14,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 47,
|
||||
@@ -427,7 +427,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 3,
|
||||
"agentBlendedCost": 0.9405,
|
||||
"agentBlendedCost": 1.0369,
|
||||
"costPer1kRequests": 10,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 45.4,
|
||||
@@ -435,7 +435,7 @@
|
||||
"agenticIndex": 56,
|
||||
"valueScore": 2132.7
|
||||
},
|
||||
"benchmarkSource": "openrouter",
|
||||
"benchmarkSource": "artificial-analysis",
|
||||
"reasoning": null,
|
||||
"isFree": false,
|
||||
"isBatch": false,
|
||||
@@ -461,7 +461,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 3,
|
||||
"agentBlendedCost": 0.9405,
|
||||
"agentBlendedCost": 1.0369,
|
||||
"costPer1kRequests": 10,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 39.9,
|
||||
@@ -495,7 +495,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 3,
|
||||
"agentBlendedCost": 0.9405,
|
||||
"agentBlendedCost": 1.0369,
|
||||
"costPer1kRequests": 10,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 39.9,
|
||||
@@ -509,6 +509,40 @@
|
||||
"isBatch": true,
|
||||
"tierClass": "frontier"
|
||||
},
|
||||
{
|
||||
"id": "meta/muse-spark-1.3-contributor",
|
||||
"name": "Meta: Muse Spark 1.3 Contributor",
|
||||
"provider": "meta",
|
||||
"series": "Llama",
|
||||
"releasedAt": "2026-09-02",
|
||||
"createdUnix": 1788381519,
|
||||
"unbenchmarked": false,
|
||||
"modality": "text+image+file+audio+video->text",
|
||||
"contextWindow": 1048576,
|
||||
"maxOutput": 943718,
|
||||
"pricing": {
|
||||
"input": 0.09999999999999999,
|
||||
"output": 0.19999999999999998,
|
||||
"cachedInput": 0.002,
|
||||
"cachedInputWrite": null,
|
||||
"reasoning": null,
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 0.125,
|
||||
"agentBlendedCost": 0.04,
|
||||
"costPer1kRequests": 0.39999999999999997,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 48.1,
|
||||
"codingIndex": 75.8,
|
||||
"agenticIndex": 55.5,
|
||||
"valueScore": 51336
|
||||
},
|
||||
"benchmarkSource": "artificial-analysis",
|
||||
"reasoning": null,
|
||||
"isFree": false,
|
||||
"isBatch": false,
|
||||
"tierClass": "frontier"
|
||||
},
|
||||
{
|
||||
"id": "meta/muse-spark-1.2",
|
||||
"name": "Meta: Muse Spark 1.2",
|
||||
@@ -529,7 +563,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 2,
|
||||
"agentBlendedCost": 0.6071,
|
||||
"agentBlendedCost": 0.6677,
|
||||
"costPer1kRequests": 6.750000000000001,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 39.6,
|
||||
@@ -563,7 +597,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 2,
|
||||
"agentBlendedCost": 0.6071,
|
||||
"agentBlendedCost": 0.6677,
|
||||
"costPer1kRequests": 6.750000000000001,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 33.7,
|
||||
@@ -577,40 +611,6 @@
|
||||
"isBatch": false,
|
||||
"tierClass": "frontier"
|
||||
},
|
||||
{
|
||||
"id": "meta/muse-spark-1.3-contributor",
|
||||
"name": "Meta: Muse Spark 1.3 Contributor",
|
||||
"provider": "meta",
|
||||
"series": "Llama",
|
||||
"releasedAt": "2026-09-02",
|
||||
"createdUnix": 1788381519,
|
||||
"unbenchmarked": true,
|
||||
"modality": "text+image+file+audio+video->text",
|
||||
"contextWindow": 1048576,
|
||||
"maxOutput": 943718,
|
||||
"pricing": {
|
||||
"input": 0.09999999999999999,
|
||||
"output": 0.19999999999999998,
|
||||
"cachedInput": 0.002,
|
||||
"cachedInputWrite": null,
|
||||
"reasoning": null,
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 0.125,
|
||||
"agentBlendedCost": 0.0348,
|
||||
"costPer1kRequests": 0.39999999999999997,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": null,
|
||||
"codingIndex": null,
|
||||
"agenticIndex": null,
|
||||
"valueScore": null
|
||||
},
|
||||
"benchmarkSource": null,
|
||||
"reasoning": null,
|
||||
"isFree": false,
|
||||
"isBatch": false,
|
||||
"tierClass": "balanced"
|
||||
},
|
||||
{
|
||||
"id": "google/gemini-3.8-flash",
|
||||
"name": "Google: Gemini 3.8 Flash",
|
||||
@@ -631,7 +631,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 1.5,
|
||||
"agentBlendedCost": 0.4107,
|
||||
"agentBlendedCost": 0.4093,
|
||||
"costPer1kRequests": 5.25,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 40.9,
|
||||
@@ -665,7 +665,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 0.75,
|
||||
"agentBlendedCost": 0.2054,
|
||||
"agentBlendedCost": 0.2055,
|
||||
"costPer1kRequests": 2.625,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 40.9,
|
||||
@@ -673,7 +673,7 @@
|
||||
"agenticIndex": 40.2,
|
||||
"valueScore": 7785.3
|
||||
},
|
||||
"benchmarkSource": "openrouter",
|
||||
"benchmarkSource": "artificial-analysis",
|
||||
"reasoning": null,
|
||||
"isFree": false,
|
||||
"isBatch": true,
|
||||
@@ -699,7 +699,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 1.5,
|
||||
"agentBlendedCost": 0.4107,
|
||||
"agentBlendedCost": 0.4093,
|
||||
"costPer1kRequests": 5.25,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 39.1,
|
||||
@@ -733,7 +733,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 20,
|
||||
"agentBlendedCost": 4.9405,
|
||||
"agentBlendedCost": 5.4655,
|
||||
"costPer1kRequests": 70,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 53.4,
|
||||
@@ -767,7 +767,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 10,
|
||||
"agentBlendedCost": 2.4702,
|
||||
"agentBlendedCost": 2.7327,
|
||||
"costPer1kRequests": 35,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 53.4,
|
||||
@@ -775,7 +775,7 @@
|
||||
"agenticIndex": 57.9,
|
||||
"valueScore": 688.5
|
||||
},
|
||||
"benchmarkSource": "openrouter",
|
||||
"benchmarkSource": "artificial-analysis",
|
||||
"reasoning": null,
|
||||
"isFree": false,
|
||||
"isBatch": true,
|
||||
@@ -801,7 +801,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 10,
|
||||
"agentBlendedCost": 2.7381,
|
||||
"agentBlendedCost": 2.9845,
|
||||
"costPer1kRequests": 35,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 50.8,
|
||||
@@ -835,7 +835,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 3,
|
||||
"agentBlendedCost": 1.119,
|
||||
"agentBlendedCost": 1.2048,
|
||||
"costPer1kRequests": 10,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 44.3,
|
||||
@@ -869,7 +869,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 3,
|
||||
"agentBlendedCost": 0.9762,
|
||||
"agentBlendedCost": 1.0705,
|
||||
"costPer1kRequests": 10,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 38.8,
|
||||
@@ -903,7 +903,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 1.5625,
|
||||
"agentBlendedCost": 0.5595,
|
||||
"agentBlendedCost": 0.6179,
|
||||
"costPer1kRequests": 5,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 24.9,
|
||||
@@ -937,7 +937,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 3.6012500000000003,
|
||||
"agentBlendedCost": 0.9843,
|
||||
"agentBlendedCost": 1.0657,
|
||||
"costPer1kRequests": 12.73,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 43.6,
|
||||
@@ -971,7 +971,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 6,
|
||||
"agentBlendedCost": 1.6429,
|
||||
"agentBlendedCost": 1.7907,
|
||||
"costPer1kRequests": 21,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 43.6,
|
||||
@@ -979,7 +979,7 @@
|
||||
"agenticIndex": 50,
|
||||
"valueScore": 1030.3
|
||||
},
|
||||
"benchmarkSource": "openrouter",
|
||||
"benchmarkSource": "artificial-analysis",
|
||||
"reasoning": null,
|
||||
"isFree": false,
|
||||
"isBatch": true,
|
||||
@@ -1005,7 +1005,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 1.7125,
|
||||
"agentBlendedCost": 0.531,
|
||||
"agentBlendedCost": 0.575,
|
||||
"costPer1kRequests": 5.8999999999999995,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 27,
|
||||
@@ -1039,7 +1039,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 3,
|
||||
"agentBlendedCost": 1.7857,
|
||||
"agentBlendedCost": 1.8018,
|
||||
"costPer1kRequests": 10.5,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 14.2,
|
||||
@@ -1073,7 +1073,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 1.5,
|
||||
"agentBlendedCost": 0.8929,
|
||||
"agentBlendedCost": 0.9009,
|
||||
"costPer1kRequests": 5.25,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 14.2,
|
||||
@@ -1081,7 +1081,7 @@
|
||||
"agenticIndex": 7,
|
||||
"valueScore": 1892.7
|
||||
},
|
||||
"benchmarkSource": "openrouter",
|
||||
"benchmarkSource": "artificial-analysis",
|
||||
"reasoning": null,
|
||||
"isFree": false,
|
||||
"isBatch": true,
|
||||
@@ -1107,7 +1107,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 0.26249999999999996,
|
||||
"agentBlendedCost": 0.075,
|
||||
"agentBlendedCost": 0.0824,
|
||||
"costPer1kRequests": 0.9,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 11.3,
|
||||
@@ -1141,7 +1141,7 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 0.54375,
|
||||
"agentBlendedCost": 0.1476,
|
||||
"agentBlendedCost": 0.1707,
|
||||
"costPer1kRequests": 1.74,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": 26,
|
||||
@@ -1175,15 +1175,15 @@
|
||||
"webSearch": null
|
||||
},
|
||||
"blendedCost": 0.17500000000000002,
|
||||
"agentBlendedCost": 0.0487,
|
||||
"agentBlendedCost": 0.056,
|
||||
"costPer1kRequests": 0.56,
|
||||
"benchmarks": {
|
||||
"intelligenceIndex": null,
|
||||
"codingIndex": 56.8,
|
||||
"agenticIndex": 15.8,
|
||||
"valueScore": 21304.3
|
||||
"intelligenceIndex": 26,
|
||||
"codingIndex": 60.2,
|
||||
"agenticIndex": 21.3,
|
||||
"valueScore": 23822.9
|
||||
},
|
||||
"benchmarkSource": "openrouter",
|
||||
"benchmarkSource": "artificial-analysis",
|
||||
"reasoning": null,
|
||||
"isFree": false,
|
||||
"isBatch": false,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"schemaVersion": "1.0.0",
|
||||
"generatedAt": "2026-09-21T02:17:39.704Z",
|
||||
"generatedAt": "2026-09-21T02:50:22.938Z",
|
||||
"constants": {
|
||||
"agentTurnTokens": 21000,
|
||||
"defaultCacheRate": 0.75,
|
||||
|
||||
+64
-15
@@ -17,6 +17,19 @@ function normalizeStr(str) {
|
||||
return (str || '').toLowerCase().replace(/[^a-z0-9]/g, '');
|
||||
}
|
||||
|
||||
// Effort/variant suffixes AA appends to benchmark names ("Grok 4.6 (medium)").
|
||||
const EFFORT_SUFFIX_RE = /\s*\((?:low|medium|high|max|xhigh)\)\s*$/i;
|
||||
|
||||
function normalizeBenchmarkName(name) {
|
||||
return normalizeStr((name || '').replace(EFFORT_SUFFIX_RE, ''));
|
||||
}
|
||||
|
||||
// Minimum fraction of catalog models that must receive benchmark data.
|
||||
// Below this the benchmark feed has drifted and the quality-filtered
|
||||
// leaderboards silently starve (VULN-11). Set ALLOW_LOW_BENCH_MATCH=1 to
|
||||
// bypass with a warning.
|
||||
const MIN_BENCH_MATCH_RATE = 0.25;
|
||||
|
||||
function classifyModelTier(model) {
|
||||
const id = (model.id || '').toLowerCase();
|
||||
const name = (model.name || '').toLowerCase();
|
||||
@@ -61,6 +74,8 @@ async function buildData() {
|
||||
const { inputTokens: agentIn, outputTokens: agentOut } = estimateConstants.agentRequest;
|
||||
const agentRequestTokens = agentIn + agentOut;
|
||||
const cacheRate = estimateConstants.defaultCacheRate;
|
||||
const cacheWriteShare = estimateConstants.cacheWriteShare ?? 0;
|
||||
const cacheWritePremium = estimateConstants.cacheWritePremium ?? 1.25;
|
||||
|
||||
// 1. Load models
|
||||
let models = [];
|
||||
@@ -81,18 +96,33 @@ async function buildData() {
|
||||
|
||||
// 2. Merge benchmarks into models with enhanced matching
|
||||
let matchedCount = 0;
|
||||
const fuzzyMatches = [];
|
||||
const benchBySlug = new Map(benchmarks.map(b => [normalizeStr(b.slug || ''), b]));
|
||||
const benchByName = new Map(benchmarks.map(b => [normalizeBenchmarkName(b.name), b]));
|
||||
|
||||
for (const model of models) {
|
||||
const modelIdClean = model.id.split('/').pop() || '';
|
||||
const mIdNorm = normalizeStr(modelIdClean);
|
||||
const mNameNorm = normalizeStr(model.name);
|
||||
const mNameNorm = normalizeBenchmarkName(model.name);
|
||||
|
||||
const match = benchmarks.find(b => {
|
||||
const bSlugClean = b.slug || '';
|
||||
const bSlugNorm = normalizeStr(bSlugClean);
|
||||
const bNameNorm = normalizeStr(b.name);
|
||||
// Pass 1: exact normalized slug/name equality.
|
||||
let match = benchmarks.find(b => {
|
||||
const bSlugNorm = normalizeStr(b.slug || '');
|
||||
const bNameNorm = normalizeBenchmarkName(b.name);
|
||||
return bSlugNorm === mIdNorm || bNameNorm === mNameNorm;
|
||||
});
|
||||
|
||||
return bSlugNorm === mIdNorm || bNameNorm === mNameNorm;
|
||||
});
|
||||
// Pass 2 (VULN-11): effort-suffix-stripped fuzzy match so "Grok 4.6 (medium)"
|
||||
// still matches "grok-4.6" instead of silently starving the leaderboard.
|
||||
if (!match && benchmarks.length > 0) {
|
||||
match =
|
||||
benchBySlug.get(mIdNorm) ||
|
||||
(mIdNorm.length >= 5
|
||||
? [...benchBySlug.entries()].find(([slug]) => slug.startsWith(mIdNorm) || mIdNorm.startsWith(slug) && slug.length >= 5)?.[1]
|
||||
: undefined) ||
|
||||
benchByName.get(mNameNorm);
|
||||
if (match) fuzzyMatches.push(`${model.id} -> ${match.name ?? match.slug}`);
|
||||
}
|
||||
|
||||
if (match && match.evaluations) {
|
||||
matchedCount++;
|
||||
@@ -114,19 +144,24 @@ async function buildData() {
|
||||
model.benchmarkSource = 'openrouter';
|
||||
}
|
||||
|
||||
// Cache pricing: never fabricate a discount. Use the real cachedInput from
|
||||
// the upstream fetch; unknown cache prices are treated as no caching.
|
||||
const inputPrice = model.pricing?.input || 0;
|
||||
const outputPrice = model.pricing?.output || 0;
|
||||
const cachedInput = model.pricing?.cachedInput;
|
||||
|
||||
// Standard Agent Request from the shared estimate constants (default
|
||||
// 20K input at 75% cache + 1K output). When no cache price is known,
|
||||
// charge full input price for cached tokens (conservative).
|
||||
// Amortized cache-write share (VULN-14): keep in lockstep with
|
||||
// scripts/fetch-models.mjs so both agents of record agree.
|
||||
const inputPrice = model.pricing?.input || 0;
|
||||
const outputPrice = model.pricing?.output || 0;
|
||||
const cachedInput = model.pricing?.cachedInput;
|
||||
const cachedInputWrite = model.pricing?.cachedInputWrite;
|
||||
|
||||
const freshIn = agentIn * (1 - cacheRate);
|
||||
const cachedIn = agentIn * cacheRate;
|
||||
const cachePrice = cachedInput !== null && cachedInput !== undefined ? cachedInput : inputPrice;
|
||||
const agentReqCost = (freshIn * inputPrice / 1e6) + (cachedIn * cachePrice / 1e6) + (agentOut * outputPrice / 1e6);
|
||||
const writePrice = cachedInputWrite !== null && cachedInputWrite !== undefined ? cachedInputWrite : inputPrice * cacheWritePremium;
|
||||
const agentReqCost = (freshIn * inputPrice / 1e6) +
|
||||
(cachedIn * (1 - cacheWriteShare) * cachePrice / 1e6) +
|
||||
(cachedIn * cacheWriteShare * writePrice / 1e6) +
|
||||
(agentOut * outputPrice / 1e6);
|
||||
model.agentBlendedCost = parseFloat(((agentReqCost / agentRequestTokens) * 1e6).toFixed(4));
|
||||
|
||||
// Assign tier
|
||||
@@ -151,7 +186,21 @@ async function buildData() {
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Matched ${matchedCount} models with Artificial Analysis benchmarks.`);
|
||||
console.log(`Matched ${matchedCount} models with Artificial Analysis benchmarks (${fuzzyMatches.length} via fuzzy fallback).`);
|
||||
if (fuzzyMatches.length > 0) {
|
||||
for (const f of fuzzyMatches.slice(0, 10)) console.log(` fuzzy: ${f}`);
|
||||
}
|
||||
|
||||
// Match-rate floor (VULN-11): a drifted benchmark feed must not silently
|
||||
// starve the quality-filtered leaderboards of coding indices.
|
||||
if (models.length > 0 && benchmarks.length > 0 && matchedCount / models.length < MIN_BENCH_MATCH_RATE) {
|
||||
const msg = `Benchmark match rate ${matchedCount}/${models.length} (${(matchedCount / models.length * 100).toFixed(1)}%) is below the ${MIN_BENCH_MATCH_RATE * 100}% floor — the benchmark feed has likely drifted. Fix the matching or set ALLOW_LOW_BENCH_MATCH=1 to override.`;
|
||||
if (process.env.ALLOW_LOW_BENCH_MATCH === '1') {
|
||||
console.warn(`WARNING: ${msg}`);
|
||||
} else {
|
||||
throw new Error(msg);
|
||||
}
|
||||
}
|
||||
|
||||
// Authoritative series re-derive + unbenchmarked flag + age/hub re-assertion.
|
||||
// models.json may be stale from an older run; re-enforce the invariants here.
|
||||
|
||||
+17
-11
@@ -27,6 +27,8 @@ async function fetchModels() {
|
||||
const { inputTokens: chatIn, outputTokens: chatOut } = constants.chatRequest;
|
||||
const { inputWeight: blendIn, outputWeight: blendOut } = constants.chatBlend;
|
||||
const cacheRate = constants.defaultCacheRate;
|
||||
const cacheWriteShare = constants.cacheWriteShare ?? 0;
|
||||
const cacheWritePremium = constants.cacheWritePremium ?? 1.25;
|
||||
const agentRequestTokens = agentIn + agentOut;
|
||||
|
||||
const response = await fetch('https://openrouter.ai/api/v1/models', { headers });
|
||||
@@ -108,19 +110,16 @@ async function fetchModels() {
|
||||
cachedInputWrite = upstreamCacheWrite * 1_000_000;
|
||||
}
|
||||
if (cachedInput === null) {
|
||||
// Documented per-provider fallback ratios from data/estimate-constants.json
|
||||
// (VULN-15: previously scattered literals). Unknown providers get no
|
||||
// fabricated discount — cached tokens bill at full input price.
|
||||
const fallbacks = constants.providerCacheReadFallback ?? {};
|
||||
const provLower = provider.toLowerCase();
|
||||
const idLower = id.toLowerCase();
|
||||
if (inputPrice > 0) {
|
||||
if (provLower.includes('anthropic') || idLower.includes('claude')) {
|
||||
cachedInput = inputPrice * 0.10;
|
||||
} else if (provLower.includes('deepseek') || idLower.includes('deepseek')) {
|
||||
cachedInput = inputPrice * 0.10;
|
||||
} else if (provLower.includes('z-ai') || idLower.includes('glm')) {
|
||||
cachedInput = inputPrice * 0.10;
|
||||
} else if (provLower.includes('google') || idLower.includes('gemini')) {
|
||||
cachedInput = inputPrice * 0.25;
|
||||
} else if (provLower.includes('openai') || idLower.includes('gpt') || idLower.includes('codex')) {
|
||||
cachedInput = inputPrice * 0.50;
|
||||
const fallbackKey = Object.keys(fallbacks).find(k => provLower.includes(k) || idLower.includes(k));
|
||||
if (fallbackKey) {
|
||||
cachedInput = inputPrice * fallbacks[fallbackKey];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -128,10 +127,17 @@ async function fetchModels() {
|
||||
// Agentic coding calculation: standard agent request (default 20K input at
|
||||
// 75% cache + 1K output) from the shared estimate constants.
|
||||
// When no cache price is known, charge full input price for cached tokens (conservative).
|
||||
// Amortized cache-write share (VULN-14): the fraction of cached context
|
||||
// rewritten each turn is billed at the provider's write rate instead of
|
||||
// being silently treated as a free read.
|
||||
const freshIn = agentIn * (1 - cacheRate);
|
||||
const cachedIn = agentIn * cacheRate;
|
||||
const cachePrice = cachedInput !== null ? cachedInput : inputPrice;
|
||||
const agentReqCost = (freshIn * inputPrice / 1e6) + (cachedIn * cachePrice / 1e6) + (agentOut * outputPrice / 1e6);
|
||||
const writePrice = cachedInputWrite !== null ? cachedInputWrite : inputPrice * cacheWritePremium;
|
||||
const agentReqCost = (freshIn * inputPrice / 1e6) +
|
||||
(cachedIn * (1 - cacheWriteShare) * cachePrice / 1e6) +
|
||||
(cachedIn * cacheWriteShare * writePrice / 1e6) +
|
||||
(agentOut * outputPrice / 1e6);
|
||||
const agentBlendedCost = (agentReqCost / agentRequestTokens) * 1e6;
|
||||
|
||||
const createdUnix = Number.isFinite(model.created) ? model.created : null;
|
||||
|
||||
@@ -185,6 +185,12 @@ async function validateModels(models) {
|
||||
'GLM', 'Kimi', 'Mistral', 'MiMo', 'other',
|
||||
]);
|
||||
|
||||
// Price sanity bounds (VULN-01): structural validation alone let an
|
||||
// upstream regression or a poisoned fetch auto-deploy absurd yields.
|
||||
// Sane ceiling: no per-million list price above $1,000 in/$2,000 out.
|
||||
const MAX_INPUT_PER_M = 1_000;
|
||||
const MAX_OUTPUT_PER_M = 2_000;
|
||||
|
||||
const SERIES_CAP = 3;
|
||||
const AGE_WINDOW_DAYS = 365;
|
||||
const cutoffUnix = Math.floor(Date.now() / 1000) - AGE_WINDOW_DAYS * 86400;
|
||||
@@ -197,6 +203,31 @@ async function validateModels(models) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// --- price plausibility bounds (VULN-01) ---
|
||||
const pricing = m.pricing ?? {};
|
||||
const inP = pricing.input;
|
||||
const outP = pricing.output;
|
||||
if (typeof inP !== 'number' || !Number.isFinite(inP) || inP < 0) {
|
||||
err(`${where}: input price must be a finite non-negative number, got ${inP}`);
|
||||
} else if (!m.isFree && !m.isBatch && inP === 0) {
|
||||
err(`${where}: non-free model has $0 input price (upstream data regression?)`);
|
||||
} else if (inP > MAX_INPUT_PER_M) {
|
||||
err(`${where}: input price $${inP}/M exceeds sanity ceiling $${MAX_INPUT_PER_M}/M`);
|
||||
}
|
||||
if (typeof outP !== 'number' || !Number.isFinite(outP) || outP < 0) {
|
||||
err(`${where}: output price must be a finite non-negative number, got ${outP}`);
|
||||
} else if (!m.isFree && !m.isBatch && outP === 0) {
|
||||
err(`${where}: non-free model has $0 output price (upstream data regression?)`);
|
||||
} else if (outP > MAX_OUTPUT_PER_M) {
|
||||
err(`${where}: output price $${outP}/M exceeds sanity ceiling $${MAX_OUTPUT_PER_M}/M`);
|
||||
}
|
||||
if (pricing.cachedInput != null && (!Number.isFinite(pricing.cachedInput) || pricing.cachedInput < 0)) {
|
||||
err(`${where}: cachedInput must be null or a finite non-negative number`);
|
||||
}
|
||||
if (typeof m.blendedCost === 'number' && m.blendedCost < 0) {
|
||||
err(`${where}: blendedCost must not be negative`);
|
||||
}
|
||||
|
||||
if (typeof m.series !== 'string' || m.series.length === 0) {
|
||||
err(`${where}: missing canonical "series" field`);
|
||||
} else if (!KNOWN_SERIES.has(m.series)) {
|
||||
|
||||
@@ -2,7 +2,7 @@ import { useState, useMemo } from 'react';
|
||||
import { useSearchParams } from 'react-router-dom';
|
||||
import type { CodingPlan, NormalizedModel, CacheRate, WorkloadItem } from '../../lib/types';
|
||||
import { getEffectiveCacheMultiplier, calculatePoolDrain, resolveTierModelBudget, QUALITY } from '../../lib/pricing';
|
||||
import { DEFAULT_CACHE_RATE } from '../../lib/estimate-constants';
|
||||
import { DEFAULT_CACHE_RATE, SESSION_CACHE_BUST_RESIDUAL } from '../../lib/estimate-constants';
|
||||
import {
|
||||
Workflow,
|
||||
Sparkles,
|
||||
@@ -111,7 +111,7 @@ function sessionCost(
|
||||
const baseFresh = Math.min(sessionInputSum, finalContextTokens);
|
||||
const reReadTokens = Math.max(0, sessionInputSum - baseFresh);
|
||||
const uncachedOverhead = Math.max(0, 1 - cacheRate);
|
||||
const freshInput = baseFresh + reReadTokens * uncachedOverhead * 0.15;
|
||||
const freshInput = baseFresh + reReadTokens * uncachedOverhead * SESSION_CACHE_BUST_RESIDUAL;
|
||||
const cachedInput = Math.max(0, sessionInputSum - freshInput);
|
||||
|
||||
return (
|
||||
|
||||
@@ -20,6 +20,10 @@ export const AGENT_TASK_OPTIMISTIC_TOKENS = constants.agentTask.optimisticTokens
|
||||
export const DEFAULT_CACHE_RATE = constants.defaultCacheRate as CacheRate;
|
||||
/** Anthropic cache-write premium (5-minute TTL) used when a provider price is missing. */
|
||||
export const CACHE_WRITE_PREMIUM = constants.cacheWritePremium;
|
||||
/** Default fraction of cached context rewritten per turn (see data/estimate-constants.json _cacheWriteShare). */
|
||||
export const CACHE_WRITE_SHARE = constants.cacheWriteShare as number;
|
||||
export const TIME_OF_DAY_BLEND = constants.timeOfDayBlend;
|
||||
export const AGENT_CACHE_PRESETS = constants.agentCachePresets;
|
||||
/** Residual full-price share of re-read tokens inside one multi-turn session (cache invalidations). */
|
||||
export const SESSION_CACHE_BUST_RESIDUAL = constants.sessionCacheBustResidual as number;
|
||||
|
||||
|
||||
@@ -129,3 +129,62 @@ describe('hardware economics engine', () => {
|
||||
expect(result.monthlySavingsVsApi).toBeLessThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Physical workload feasibility (VULN-04)', () => {
|
||||
it('flags a workload that exceeds the configured duty cycle as infeasible', () => {
|
||||
// Mac Mini flagship decodes at 24 tok/s. 40M tokens/mo needs ~463h of pure
|
||||
// decode, but 1h/day x 30.4 only affords ~30h — physically impossible.
|
||||
const result = computeHardwareEconomics({
|
||||
presetId: 'mac-mini-m4-pro-64gb',
|
||||
amortizationMonths: 24,
|
||||
salvageValuePercent: 20,
|
||||
electricityKwhCost: 0.16,
|
||||
dailyInferenceHours: 1,
|
||||
dailyIdleHours: 9,
|
||||
monthlyTokenVolumeM: 40,
|
||||
inputRatio: 0.8,
|
||||
cloudApiInputPerM: 3.0,
|
||||
cloudApiOutputPerM: 15.0,
|
||||
});
|
||||
|
||||
expect(result.infeasibleWorkload).toBe(true);
|
||||
expect(result.requiredInferenceHours).toBeGreaterThan(400);
|
||||
expect(result.feasibleMonthlyVolumeM).toBeLessThan(40);
|
||||
});
|
||||
|
||||
it('marks a workload achievable within the duty cycle as feasible', () => {
|
||||
const result = computeHardwareEconomics({
|
||||
presetId: 'mac-mini-m4-pro-64gb',
|
||||
amortizationMonths: 24,
|
||||
salvageValuePercent: 20,
|
||||
electricityKwhCost: 0.16,
|
||||
dailyInferenceHours: 3,
|
||||
dailyIdleHours: 9,
|
||||
monthlyTokenVolumeM: 5,
|
||||
inputRatio: 0.8,
|
||||
cloudApiInputPerM: 3.0,
|
||||
cloudApiOutputPerM: 15.0,
|
||||
});
|
||||
|
||||
expect(result.infeasibleWorkload).toBe(false);
|
||||
expect(result.feasibleMonthlyVolumeM).toBeGreaterThanOrEqual(5);
|
||||
});
|
||||
|
||||
it('only quotes payback when the amortized monthly alpha is positive', () => {
|
||||
// Cloud cost below amortized local cost must suppress payback entirely so
|
||||
// the headline metrics can never contradict each other.
|
||||
const result = computeHardwareEconomics({
|
||||
presetId: 'custom-dual-rtx-5090',
|
||||
amortizationMonths: 24,
|
||||
salvageValuePercent: 10,
|
||||
electricityKwhCost: 0.2,
|
||||
dailyInferenceHours: 6,
|
||||
dailyIdleHours: 12,
|
||||
monthlyTokenVolumeM: 1,
|
||||
inputRatio: 0.8,
|
||||
cloudApiInputPerM: 0.14,
|
||||
cloudApiOutputPerM: 0.28,
|
||||
});
|
||||
expect(result.paybackMonthsVsApi).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
+35
-12
@@ -34,7 +34,7 @@ export interface HardwareEconomicsInputs {
|
||||
presetId: string;
|
||||
customCapexUsd?: number;
|
||||
amortizationMonths: number; // e.g. 12, 24, 36, 48
|
||||
salvageValuePercent: number; // e.g. 0 to 40%
|
||||
salvageValuePercent: number; // e.g. 0 to 50% (single clamp constant shared with the UI slider)
|
||||
electricityKwhCost: number; // default $0.16/kWh
|
||||
dailyInferenceHours: number; // default 3 hrs
|
||||
dailyIdleHours: number; // default 9 hrs
|
||||
@@ -73,8 +73,17 @@ export interface HardwareEconomicsResult {
|
||||
paybackMonthsVsPlan: number | null;
|
||||
crossoverSeries: CrossoverPoint[];
|
||||
cumulativeTcoSeries: CumulativePoint[];
|
||||
/** Hours of pure decode needed per month to generate the requested volume on the preset's flagship model. */
|
||||
requiredInferenceHours: number;
|
||||
/** True when requiredInferenceHours exceeds the configured daily inference budget. */
|
||||
infeasibleWorkload: boolean;
|
||||
/** Largest monthly volume (M tokens) the configured duty cycle can physically generate. */
|
||||
feasibleMonthlyVolumeM: number;
|
||||
}
|
||||
|
||||
/** Single salvage-value clamp shared by the engine and the UI slider (VULN-14). */
|
||||
export const SALVAGE_CLAMP_MAX = 50;
|
||||
|
||||
export const HARDWARE_PRESETS: HardwarePreset[] = [
|
||||
{
|
||||
id: 'mac-mini-m4-pro-64gb',
|
||||
@@ -381,7 +390,7 @@ export function computeHardwareEconomics(inputs: HardwareEconomicsInputs): Hardw
|
||||
const capex = inputs.customCapexUsd ?? preset.capexUsd;
|
||||
|
||||
// 1. Amortization with salvage value
|
||||
const netDepreciationCapEx = capex * (1 - Math.max(0, Math.min(60, inputs.salvageValuePercent)) / 100);
|
||||
const netDepreciationCapEx = capex * (1 - Math.max(0, Math.min(SALVAGE_CLAMP_MAX, inputs.salvageValuePercent)) / 100);
|
||||
const months = Math.max(1, inputs.amortizationMonths);
|
||||
const monthlyAmortizationCost = netDepreciationCapEx / months;
|
||||
|
||||
@@ -416,24 +425,35 @@ export function computeHardwareEconomics(inputs: HardwareEconomicsInputs): Hardw
|
||||
// volumeM = totalMonthlyLocalCost / blendedApiRatePerM
|
||||
const crossoverVolumeM = blendedApiRatePerM > 0 ? totalMonthlyLocalCost / blendedApiRatePerM : 0;
|
||||
|
||||
// 7. Payback period in months
|
||||
// Capital outlay / monthly operational savings vs cloud
|
||||
// Monthly cloud spend avoided - ongoing monthly electricity cost = net cash flow benefit
|
||||
const netMonthlyBenefitVsApi = monthlyCloudApiCost - monthlyElectricityCost;
|
||||
// 7. Physical feasibility: generating the requested volume requires
|
||||
// volume / throughput seconds of pure decode (VULN-04). Compare against
|
||||
// the configured daily inference budget so infeasible workloads are
|
||||
// flagged instead of silently "winning" the breakeven verdict.
|
||||
const decodeTokensPerSec = preset.recommendedModels[0]?.tokensPerSec ?? 1;
|
||||
const requiredInferenceHours = (Math.max(0, inputs.monthlyTokenVolumeM) * 1e6) / (decodeTokensPerSec * 3600);
|
||||
const availableInferenceHours = Math.max(0, inputs.dailyInferenceHours) * 30.4375;
|
||||
const feasibleMonthlyVolumeM = (availableInferenceHours * decodeTokensPerSec * 3600) / 1e6;
|
||||
const infeasibleWorkload = requiredInferenceHours > availableInferenceHours * 1.001;
|
||||
|
||||
// 8. Payback period in months
|
||||
// Capital outlay / monthly operational savings vs cloud. Both payback and
|
||||
// headline alpha use the same amortized cost basis so the two metrics can
|
||||
// never contradict each other (VULN-04).
|
||||
const netMonthlyBenefitVsApi = monthlyCloudApiCost - monthlyElectricityCost - monthlyAmortizationCost;
|
||||
let paybackMonthsVsApi: number | null = null;
|
||||
if (netMonthlyBenefitVsApi > 0) {
|
||||
paybackMonthsVsApi = capex / netMonthlyBenefitVsApi;
|
||||
paybackMonthsVsApi = capex / (monthlyCloudApiCost - monthlyElectricityCost);
|
||||
}
|
||||
|
||||
let paybackMonthsVsPlan: number | null = null;
|
||||
if (monthlyCloudPlanCost !== null) {
|
||||
const netMonthlyBenefitVsPlan = monthlyCloudPlanCost - monthlyElectricityCost;
|
||||
if (netMonthlyBenefitVsPlan > 0) {
|
||||
paybackMonthsVsPlan = capex / netMonthlyBenefitVsPlan;
|
||||
const grossMonthlyBenefitVsPlan = monthlyCloudPlanCost - monthlyElectricityCost;
|
||||
if (grossMonthlyBenefitVsPlan > 0) {
|
||||
paybackMonthsVsPlan = capex / grossMonthlyBenefitVsPlan;
|
||||
}
|
||||
}
|
||||
|
||||
// 8. Generate Crossover Volume series (0 to 30M tokens in steps)
|
||||
// 10. Generate Crossover Volume series (0 to 30M tokens in steps)
|
||||
const crossoverSeries: CrossoverPoint[] = [];
|
||||
const maxVolume = Math.max(30, Math.ceil(Math.max(crossoverVolumeM * 1.5, inputs.monthlyTokenVolumeM * 1.5)));
|
||||
const steps = 15;
|
||||
@@ -450,7 +470,7 @@ export function computeHardwareEconomics(inputs: HardwareEconomicsInputs): Hardw
|
||||
});
|
||||
}
|
||||
|
||||
// 9. Generate Cumulative TCO over 36 months
|
||||
// 11. Generate Cumulative TCO over 36 months
|
||||
const cumulativeTcoSeries: CumulativePoint[] = [];
|
||||
for (let m = 1; m <= 36; m++) {
|
||||
// Local cumulative: Capex upfront + monthly electricity
|
||||
@@ -480,5 +500,8 @@ export function computeHardwareEconomics(inputs: HardwareEconomicsInputs): Hardw
|
||||
paybackMonthsVsPlan: paybackMonthsVsPlan !== null ? Number(paybackMonthsVsPlan.toFixed(1)) : null,
|
||||
crossoverSeries,
|
||||
cumulativeTcoSeries,
|
||||
requiredInferenceHours: Number(requiredInferenceHours.toFixed(1)),
|
||||
infeasibleWorkload,
|
||||
feasibleMonthlyVolumeM: Number(feasibleMonthlyVolumeM.toFixed(1)),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -148,3 +148,37 @@ describe('Agent Log Parser & Session Receipt Engine', () => {
|
||||
expect(() => parseAgentLog(hugeLog, 'huge.json')).toThrow('25MB');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Token accounting provenance (VULN-05)', () => {
|
||||
it('marks JSONL transcripts with real token fields as measured', () => {
|
||||
const jsonl = [
|
||||
JSON.stringify({ input_tokens: 12000, cached_tokens: 9000, output_tokens: 700 }),
|
||||
JSON.stringify({ input_tokens: 15000, cached_tokens: 11000, output_tokens: 900 }),
|
||||
].join('\n');
|
||||
const session = parseAgentLog(jsonl, 'measured.jsonl');
|
||||
expect(session.measuredTokens).toBe(true);
|
||||
});
|
||||
|
||||
it('marks token-less transcripts as estimated so receipts cannot pass them off as bills', () => {
|
||||
const jsonl = [
|
||||
JSON.stringify({ type: 'USER_INPUT', message: 'fix the tests' }),
|
||||
JSON.stringify({ type: 'ASSISTANT', message: 'done' }),
|
||||
].join('\n');
|
||||
const session = parseAgentLog(jsonl, 'no-tokens.jsonl');
|
||||
expect(session.measuredTokens).toBe(false);
|
||||
|
||||
const report = calculateSessionReceipt(session, []);
|
||||
expect(report.tokensMeasured).toBe(false);
|
||||
expect(report.headlineSummary).toContain('ESTIMATED');
|
||||
});
|
||||
|
||||
it('marks generic text fallbacks as estimated', () => {
|
||||
const session = parseAgentLog('some random markdown\nwith lines\nbut no json', 'notes.md');
|
||||
expect(session.format).toBe('generic-agent');
|
||||
expect(session.measuredTokens).toBe(false);
|
||||
});
|
||||
|
||||
it('keeps the built-in sample session measured', () => {
|
||||
expect(SAMPLE_AGENT_SESSION.measuredTokens).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
+17
-1
@@ -21,6 +21,12 @@ export interface ParsedAgentSession {
|
||||
effectiveCacheHitRate: number; // 0 to 1
|
||||
toolInvocationsCount: number;
|
||||
turns: ParsedSessionTurn[];
|
||||
/**
|
||||
* True when the transcript contained real token accounting.
|
||||
* False means the parser synthesized the numbers (VULN-05) — receipts must
|
||||
* flag these as estimates, never present them as measured cost.
|
||||
*/
|
||||
measuredTokens: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -38,6 +44,7 @@ export const SAMPLE_AGENT_SESSION: ParsedAgentSession = {
|
||||
totalTokens: 1392400,
|
||||
effectiveCacheHitRate: 0.78, // 78% cache hit rate
|
||||
toolInvocationsCount: 46,
|
||||
measuredTokens: true,
|
||||
turns: Array.from({ length: 38 }, (_, i) => {
|
||||
const isToolHeavy = i % 2 === 1;
|
||||
const input = 32000 + i * 800;
|
||||
@@ -102,6 +109,7 @@ function parseClineJson(data: any, filename: string): ParsedAgentSession {
|
||||
let totalCachedTokens = 0;
|
||||
let totalOutputTokens = 0;
|
||||
let toolInvocationsCount = 0;
|
||||
let sawMeasuredTokens = false;
|
||||
const turns: ParsedSessionTurn[] = [];
|
||||
|
||||
messages.forEach((msg, idx) => {
|
||||
@@ -110,6 +118,8 @@ function parseClineJson(data: any, filename: string): ParsedAgentSession {
|
||||
const outTok = safeTokenNumber(msg.tokensOut ?? msg.outputTokens ?? msg.usage?.completion_tokens);
|
||||
const tools = Array.isArray(msg.tool_calls || msg.tools) ? (msg.tool_calls || msg.tools).length : 0;
|
||||
|
||||
if (inTok > 0 || outTok > 0) sawMeasuredTokens = true;
|
||||
|
||||
if (inTok > 0 || outTok > 0 || tools > 0) {
|
||||
totalInputTokens += inTok;
|
||||
totalCachedTokens += cachedTok;
|
||||
@@ -145,6 +155,7 @@ function parseClineJson(data: any, filename: string): ParsedAgentSession {
|
||||
effectiveCacheHitRate: effectiveCacheHitRate || 0.72,
|
||||
toolInvocationsCount: toolInvocationsCount > 0 ? toolInvocationsCount : Math.round(totalTurns * 1.2),
|
||||
turns,
|
||||
measuredTokens: sawMeasuredTokens,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -154,7 +165,8 @@ function parseJsonlSession(lines: string[], filename: string): ParsedAgentSessio
|
||||
let totalCachedTokens = 0;
|
||||
let totalOutputTokens = 0;
|
||||
let toolInvocationsCount = 0;
|
||||
const turns: ParsedSessionTurn[] = [];
|
||||
let sawMeasuredTokens = false;
|
||||
const turns: ParsedAgentSession['turns'] = [];
|
||||
|
||||
for (const line of lines) {
|
||||
try {
|
||||
@@ -175,6 +187,8 @@ function parseJsonlSession(lines: string[], filename: string): ParsedAgentSessio
|
||||
inTok = 20000 + stepIndex * 600;
|
||||
cachedTok = Math.round(inTok * 0.75);
|
||||
outTok = 850;
|
||||
} else {
|
||||
sawMeasuredTokens = true;
|
||||
}
|
||||
|
||||
totalInputTokens += inTok;
|
||||
@@ -214,6 +228,7 @@ function parseJsonlSession(lines: string[], filename: string): ParsedAgentSessio
|
||||
effectiveCacheHitRate,
|
||||
toolInvocationsCount,
|
||||
turns,
|
||||
measuredTokens: sawMeasuredTokens,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -236,6 +251,7 @@ function parseGenericSession(text: string, filename: string): ParsedAgentSession
|
||||
totalTokens: totalInputTokens + totalOutputTokens,
|
||||
effectiveCacheHitRate: 0.75,
|
||||
toolInvocationsCount: Math.round(turnsCount * 1.3),
|
||||
measuredTokens: false,
|
||||
turns: Array.from({ length: turnsCount }, (_, i) => ({
|
||||
stepIndex: i + 1,
|
||||
role: 'assistant',
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
/**
|
||||
* Zero-safe URL parameter parsing (VULN-13).
|
||||
*
|
||||
* `Number(x) || default` collapses legitimate `0` values to the default,
|
||||
* which silently diverges shared configs from rendered economics
|
||||
* (e.g. `?salvage=0` or `?idleHours=0` rendering 20% / 9h instead).
|
||||
* This helper honors 0 and only falls back when the value is absent or NaN.
|
||||
*/
|
||||
export function numParam(
|
||||
raw: string | null,
|
||||
fallback: number,
|
||||
min: number,
|
||||
max: number
|
||||
): number {
|
||||
if (raw === null || raw.trim() === '') return fallback;
|
||||
const n = Number(raw);
|
||||
if (!Number.isFinite(n)) return fallback;
|
||||
return Math.min(max, Math.max(min, n));
|
||||
}
|
||||
+60
-17
@@ -589,7 +589,8 @@ describe('Adversarial Edge Cases & Guardrails', () => {
|
||||
expect(parseTierRequestLimit({ requests: '~15K mix estimate (~26K with DeepSeek V4 Flash at typical cache)' })).toBe(15000);
|
||||
expect(parseTierRequestLimit({ requests: '~100K mix estimate' })).toBe(100000);
|
||||
expect(parseTierRequestLimit({ agenticRequests: '50 agentic requests/mo (chat + agentic coding)' })).toBe(50);
|
||||
expect(parseTierRequestLimit({ requests: '1,000 requests per 7-day sliding window' })).toBe(4000);
|
||||
// Weekly quotas scale by 52/12 weeks per month (VULN-12), not a flat x4.
|
||||
expect(parseTierRequestLimit({ requests: '1,000 requests per 7-day sliding window' })).toBe(4333);
|
||||
expect(parseTierRequestLimit({ fastRequests: 500 })).toBe(500);
|
||||
// Ignores pure dollar credit strings
|
||||
expect(parseTierRequestLimit({ monthlyCredits: '$10/mo compute credits' })).toBe(null);
|
||||
@@ -845,7 +846,7 @@ describe('Adversarial Edge Cases & Guardrails', () => {
|
||||
expect(result.savings).toBeLessThan(0);
|
||||
});
|
||||
|
||||
it('computeApplesToApples applies symmetric cache scaling to subscription yields', () => {
|
||||
it('does NOT cache-scale usage-metered subscription yields (cache rate is irrelevant to fixed quotas)', () => {
|
||||
const testModel = model({
|
||||
id: 'z-ai/glm-5.3',
|
||||
name: 'GLM-5.3',
|
||||
@@ -873,42 +874,84 @@ describe('Adversarial Edge Cases & Guardrails', () => {
|
||||
|
||||
const testPlan = plan({ id: 'z-ai', name: 'Z.ai', tiers: [testTier] });
|
||||
|
||||
// When cacheRate is 75% (lower than 95% plan assumption), subscription tokens scale down symmetrically
|
||||
// Metered quota: the vendor delivers the same tokens no matter what the
|
||||
// user's own cache hit rate is (VULN-02 regression guard).
|
||||
const result75 = computeApplesToApples([testModel], [testPlan], 'all', 100, 0, 0.75);
|
||||
const sub75 = result75.options.find(o => o.type === 'subscription');
|
||||
|
||||
// When cacheRate is 0% (fresh tokens, no cache), subscription tokens scale down even further
|
||||
const result0 = computeApplesToApples([testModel], [testPlan], 'all', 100, 0, 0.0);
|
||||
const sub0 = result0.options.find(o => o.type === 'subscription');
|
||||
|
||||
expect(sub75).toBeDefined();
|
||||
expect(sub0).toBeDefined();
|
||||
// 0% cache delivers fewer tokens than 75% cache for the subscription
|
||||
expect(sub75!.monthlyTokens).toBe(1000);
|
||||
expect(sub0!.monthlyTokens).toBe(1000);
|
||||
});
|
||||
|
||||
it('cache-scales dollar-credit pools whose yield depends on per-turn cost', () => {
|
||||
const testModel = model({
|
||||
id: 'z-ai/glm-5.3',
|
||||
name: 'GLM-5.3',
|
||||
pricing: { input: 1.0, output: 2.0, cachedInput: 0.1, cachedInputWrite: null, reasoning: null, webSearch: null },
|
||||
blendedCost: 1.25,
|
||||
benchmarks: { codingIndex: 75, intelligenceIndex: 75, agenticIndex: 75, valueScore: 200 },
|
||||
});
|
||||
|
||||
const creditTier = tier({
|
||||
name: 'Pool',
|
||||
monthlyPrice: 80,
|
||||
limits: { monthlyCredits: '$80/mo usage value credits' },
|
||||
models: ['GLM-5.3'],
|
||||
estimatedTokenBudget: { estimatedMillionTokens: 1000, assumptions: '95% cache hit rate' },
|
||||
});
|
||||
const creditPlan = plan({ id: 'credit-test', name: 'CreditPool', tiers: [creditTier] });
|
||||
|
||||
const result0 = computeApplesToApples([testModel], [creditPlan], 'all', 100, 0, 0.0);
|
||||
const result75 = computeApplesToApples([testModel], [creditPlan], 'all', 100, 0, 0.75);
|
||||
const sub0 = result0.options.find(o => o.type === 'subscription');
|
||||
const sub75 = result75.options.find(o => o.type === 'subscription');
|
||||
|
||||
expect(sub0).toBeDefined();
|
||||
expect(sub75).toBeDefined();
|
||||
// A dollar pool buys fewer tokens when turns are priced without cache discounts.
|
||||
expect(sub0!.monthlyTokens).toBeLessThan(sub75!.monthlyTokens);
|
||||
// 75% cache delivers fewer tokens than the 95% marketing estimate (1000M)
|
||||
expect(sub75!.monthlyTokens).toBeLessThan(1000);
|
||||
});
|
||||
|
||||
it('parsePlanCacheAssumption correctly parses declared cache hit percentages', () => {
|
||||
const tier95 = tier({
|
||||
it('parsePlanCacheAssumption ignores discount percentages and anchors on cache context', () => {
|
||||
const discountFirst = tier({
|
||||
estimatedTokenBudget: {
|
||||
estimatedMillionTokens: 100,
|
||||
assumptions: '95% cache hit rate',
|
||||
estimateMeta: { cacheAssumption: '95% cache hit rate', sourceUrl: 'test', sourceType: 'official', confidence: 'high', verifiedAt: '2026-09-19' },
|
||||
assumptions: 'At 90% discount, 75% cache hit rate',
|
||||
estimateMeta: { cacheAssumption: 'At 90% discount, 75% cache hit rate', sourceUrl: 'test', sourceType: 'official', confidence: 'high', verifiedAt: '2026-09-19' },
|
||||
},
|
||||
});
|
||||
expect(parsePlanCacheAssumption(tier95)).toBe(0.95);
|
||||
expect(parsePlanCacheAssumption(discountFirst)).toBe(0.75);
|
||||
|
||||
const tier75 = tier({
|
||||
const discountOnly = tier({
|
||||
estimatedTokenBudget: {
|
||||
estimatedMillionTokens: 100,
|
||||
assumptions: '75% cache hit rate',
|
||||
assumptions: '50% off-peak discount applies',
|
||||
},
|
||||
});
|
||||
expect(parsePlanCacheAssumption(tier75)).toBe(0.75);
|
||||
// No cache-context percent: fall back to the documented default, not 0.5.
|
||||
expect(parsePlanCacheAssumption(discountOnly)).toBe(0.75);
|
||||
});
|
||||
|
||||
const tierDefault = tier();
|
||||
expect(parsePlanCacheAssumption(tierDefault)).toBe(0.75);
|
||||
it('discloses spend asymmetry when the API leg outspends a capped subscription in the arbitrage callout', () => {
|
||||
const sonnetModel = model({
|
||||
id: 'anthropic/claude-sonnet-5',
|
||||
name: 'Claude Sonnet 5',
|
||||
benchmarks: { codingIndex: 85, intelligenceIndex: null, agenticIndex: null, valueScore: null },
|
||||
});
|
||||
const cheapSub = plan({
|
||||
id: 'cheap-sub',
|
||||
name: 'Cheap Sub',
|
||||
tiers: [tier({ monthlyPrice: 20, models: ['Claude Sonnet 5'], estimatedTokenBudget: { estimatedMillionTokens: 1, assumptions: 'test' } })],
|
||||
});
|
||||
|
||||
const result = computeApplesToApples([sonnetModel], [cheapSub], 'all', 200, 0, 0.75);
|
||||
expect(result.arbitrageCallout).not.toBeNull();
|
||||
expect(result.arbitrageCallout!.description).toContain('spends $20/mo vs $200/mo on the API leg');
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
+75
-18
@@ -89,7 +89,9 @@ export function calculateBudgetResults(
|
||||
// yields keep the legacy 3K (2K-in/1K-out) request definition.
|
||||
const requests1k = blendMode === 'agentic'
|
||||
? (millionTokens * 1_000_000) / STANDARD_AGENT_REQUEST_TOKENS
|
||||
: (budget / (m.costPer1kRequests || 1)) * 1000;
|
||||
// Nullish coalescing (never `||`): a legitimate $0/M request cost must
|
||||
// not be replaced by the fallback divisor (invariant I).
|
||||
: (budget / (m.costPer1kRequests ?? 1)) * 1000;
|
||||
|
||||
return {
|
||||
modelId: m.id,
|
||||
@@ -265,20 +267,43 @@ export function matchesPlanModel(planModelName: string, model: NormalizedModel):
|
||||
*/
|
||||
export function parsePlanCacheAssumption(tier: PlanTier): number {
|
||||
const metaStr = tier.estimatedTokenBudget?.estimateMeta?.cacheAssumption;
|
||||
// Anchor the percent to explicit cache context so discount percentages
|
||||
// ("90% discount", "50% off-peak") cannot hijack the cache rate.
|
||||
if (metaStr) {
|
||||
const match = metaStr.match(/([0-9]+)%/);
|
||||
if (match) {
|
||||
return parseInt(match[1], 10) / 100;
|
||||
const anchored = metaStr.match(/([0-9]+)\s*%\s*(?:cache|hit)/i) || metaStr.match(/cache[^.]*?([0-9]+)\s*%/i);
|
||||
if (anchored) {
|
||||
return clampCacheAssumption(parseInt(anchored[1], 10) / 100);
|
||||
}
|
||||
}
|
||||
const assumptions = tier.estimatedTokenBudget?.assumptions;
|
||||
if (assumptions) {
|
||||
const match = assumptions.match(/([0-9]+)%\s*cache/i);
|
||||
if (match) {
|
||||
return parseInt(match[1], 10) / 100;
|
||||
const anchored = assumptions.match(/([0-9]+)\s*%\s*(?:cache|hit)/i) || assumptions.match(/cache[^.]*?([0-9]+)\s*%/i);
|
||||
if (anchored) {
|
||||
return clampCacheAssumption(parseInt(anchored[1], 10) / 100);
|
||||
}
|
||||
}
|
||||
return DEFAULT_CACHE_RATE;
|
||||
return clampCacheAssumption(DEFAULT_CACHE_RATE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Plans whose tiers meter a fixed dollar-credit pool scale their token yield
|
||||
* with the effective cost of the drained model (spend X credits -> fewer
|
||||
* tokens on pricier turns). Usage/request-metered quotas grant the same
|
||||
* tokens regardless of the user's cache hit rate, so they must NOT be scaled.
|
||||
*/
|
||||
export function isDollarPoolTier(tier: PlanTier): boolean {
|
||||
if (tier.perModelTokenBudgets) {
|
||||
const bases = Object.values(tier.perModelTokenBudgets).map(e => e?.basis);
|
||||
if (bases.some(b => b === 'list-price-credit' || b === 'equal-rate')) return true;
|
||||
}
|
||||
const limits = tier.limits ?? {};
|
||||
return Object.keys(limits).some(k => k.toLowerCase().includes('credit'));
|
||||
}
|
||||
|
||||
function clampCacheAssumption(rate: number): number {
|
||||
if (!Number.isFinite(rate) || rate < 0) return DEFAULT_CACHE_RATE;
|
||||
// Deep-agent workloads rarely sustain > 95% cache hits; treat anything above as data noise.
|
||||
return Math.min(rate, 0.95);
|
||||
}
|
||||
|
||||
export function calculateAgentRequestCost(
|
||||
@@ -430,11 +455,18 @@ export function computeApplesToApples(
|
||||
const resolved = resolveTierModelBudget(tier, model.name, model.id, planModelName);
|
||||
const rawBaseTokens = resolved.tokens || 0;
|
||||
|
||||
// Symmetric cache scaling: dynamically adjust subscription capacity with active cacheRate
|
||||
// Cache scaling only applies to dollar-credit pools: the same monthly
|
||||
// spend buys fewer tokens when turns are priced at lower cache-hit
|
||||
// rates. Usage/request-metered quotas deliver fixed tokens regardless
|
||||
// of the user's cache rate, so they are left unscaled (VULN-02).
|
||||
const cacheScaleApplies = isDollarPoolTier(tier);
|
||||
const planAssumedCache = parsePlanCacheAssumption(tier);
|
||||
const turnCostAssumed = calculateAgentRequestCost(model, planAssumedCache as CacheRate).costPerRequest;
|
||||
const turnCostUser = calculateAgentRequestCost(model, cacheRate).costPerRequest;
|
||||
const cacheScaleFactor = (turnCostAssumed > 0 && turnCostUser > 0) ? (turnCostAssumed / turnCostUser) : 1.0;
|
||||
let cacheScaleFactor = 1.0;
|
||||
if (cacheScaleApplies) {
|
||||
const turnCostAssumed = calculateAgentRequestCost(model, planAssumedCache as CacheRate).costPerRequest;
|
||||
const turnCostUser = calculateAgentRequestCost(model, cacheRate).costPerRequest;
|
||||
cacheScaleFactor = (turnCostAssumed > 0 && turnCostUser > 0) ? (turnCostAssumed / turnCostUser) : 1.0;
|
||||
}
|
||||
|
||||
const baseTokens = Math.round(rawBaseTokens * cacheScaleFactor);
|
||||
const rawRequests = tierRawRequests(tier, baseTokens);
|
||||
@@ -499,8 +531,18 @@ export function computeApplesToApples(
|
||||
if (tier.monthlyPrice > budget) continue;
|
||||
|
||||
const rawBaseTokens = tier.estimatedTokenBudget?.estimatedMillionTokens || 0;
|
||||
// Same dollar-pool gate as matched tiers so a plan never scales in
|
||||
// opposite directions depending on model-name matching (VULN-08).
|
||||
const planAssumedCache = parsePlanCacheAssumption(tier);
|
||||
const cacheScaleFactor = (1 - planAssumedCache * 0.8) / (1 - cacheRate * 0.8);
|
||||
let cacheScaleFactor = 1.0;
|
||||
if (isDollarPoolTier(tier)) {
|
||||
const basisModel = cleanModels.find(m => detectModelLab(m) === (detectPlanLabs(plan, tier)[0] ?? 'all')) ?? cleanModels[0];
|
||||
if (basisModel) {
|
||||
const turnCostAssumed = calculateAgentRequestCost(basisModel, planAssumedCache as CacheRate).costPerRequest;
|
||||
const turnCostUser = calculateAgentRequestCost(basisModel, cacheRate).costPerRequest;
|
||||
cacheScaleFactor = (turnCostAssumed > 0 && turnCostUser > 0) ? (turnCostAssumed / turnCostUser) : 1.0;
|
||||
}
|
||||
}
|
||||
const baseTokens = Math.round(rawBaseTokens * (cacheScaleFactor > 0 ? cacheScaleFactor : 1.0));
|
||||
const rawRequests = tierRawRequests(tier, baseTokens);
|
||||
const vendorQuota = parseTierRequestLimit(tier.limits);
|
||||
@@ -610,13 +652,26 @@ export function computeApplesToApples(
|
||||
? `nearest-quality API match (CI ±${qualityGap.toFixed(1)})`
|
||||
: 'nearest available API match';
|
||||
|
||||
// Spend parity: a single-seat subscription spends only its tier price
|
||||
// while the API leg spends the full budget. Surface the asymmetry
|
||||
// whenever the API leg spends meaningfully more (VULN-09).
|
||||
const subSpend = bestSubscription.monthlyCost;
|
||||
const apiSpend = apiOption.monthlyCost;
|
||||
const spendAsymmetry = apiSpend > subSpend * 1.25
|
||||
? ` Subscription spends $${subSpend.toFixed(0)}/mo vs $${apiSpend.toFixed(0)}/mo on the API leg${
|
||||
isFinite(apiSpend / (subSpend || 1))
|
||||
? ` (${(apiSpend / (subSpend || 1)).toFixed(1)}x the spend — use Dave Mode or Mix & Match to equalize)`
|
||||
: ''
|
||||
}`
|
||||
: '';
|
||||
|
||||
if (subTokens >= apiTokens * 1.25) {
|
||||
const multiplier = Number((subTokens / (apiTokens || 1)).toFixed(1));
|
||||
arbitrageCallout = {
|
||||
winnerType: 'subscription',
|
||||
multiplier,
|
||||
headline: `Coding Subscription Wins for ${labName} (${multiplier}x Compute)`,
|
||||
description: `${bestSubscription.planName} (${bestSubscription.tierName}) running ${bestSubscription.name} yields ~${formatMillionTokens(subTokens)} normalized tokens (~${bestSubscription.monthlyRequests.toLocaleString()} requests), beating direct ${apiOption.name} API (~${formatMillionTokens(apiTokens)} tokens) for $${budget}/mo. Compared as ${qualityNote}.`,
|
||||
description: `${bestSubscription.planName} (${bestSubscription.tierName}) running ${bestSubscription.name} yields ~${formatMillionTokens(subTokens)} normalized tokens (~${bestSubscription.monthlyRequests.toLocaleString()} requests), beating direct ${apiOption.name} API (~${formatMillionTokens(apiTokens)} tokens) for $${budget}/mo. Compared as ${qualityNote}.${spendAsymmetry}`,
|
||||
};
|
||||
} else if (apiTokens >= subTokens * 1.25) {
|
||||
const multiplier = Number((apiTokens / (subTokens || 1)).toFixed(1));
|
||||
@@ -624,14 +679,14 @@ export function computeApplesToApples(
|
||||
winnerType: 'api',
|
||||
multiplier,
|
||||
headline: `Direct API Wins for ${labName} (${multiplier}x Compute)`,
|
||||
description: `Direct pay-per-token API for ${apiOption.name} delivers ~${formatMillionTokens(apiTokens)} tokens (~${apiOption.monthlyRequests.toLocaleString()} requests), outperforming coding subscriptions (~${formatMillionTokens(subTokens)} normalized tokens) for $${budget}/mo. Compared as ${qualityNote}.`,
|
||||
description: `Direct pay-per-token API for ${apiOption.name} delivers ~${formatMillionTokens(apiTokens)} tokens (~${apiOption.monthlyRequests.toLocaleString()} requests), outperforming coding subscriptions (~${formatMillionTokens(subTokens)} normalized tokens) for $${budget}/mo. Compared as ${qualityNote}.${spendAsymmetry}`,
|
||||
};
|
||||
} else {
|
||||
arbitrageCallout = {
|
||||
winnerType: 'even',
|
||||
multiplier: 1.0,
|
||||
headline: `Balanced Value for ${labName}`,
|
||||
description: `Both direct API (${apiOption.name}) and subscriptions (${bestSubscription.planName} running ${bestSubscription.name}) offer comparable compute near ${formatMillionTokens(subTokens)} tokens for $${budget}/mo. Compared as ${qualityNote}.`,
|
||||
description: `Both direct API (${apiOption.name}) and subscriptions (${bestSubscription.planName} running ${bestSubscription.name}) offer comparable compute near ${formatMillionTokens(subTokens)} tokens for $${budget}/mo. Compared as ${qualityNote}.${spendAsymmetry}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -686,11 +741,13 @@ export function parseTierRequestLimit(limits?: Record<string, unknown>): number
|
||||
return Math.round(parseFloat(kMixMatch[1]) * 1000);
|
||||
}
|
||||
|
||||
// 3. Weekly requests scaled x4: e.g. "12,000/week", "1,000 requests per 7-day"
|
||||
// 3. Weekly requests scaled to a monthly equivalent (52/12 weeks per
|
||||
// month). Weekly quotas usually coexist with undocumented monthly
|
||||
// ceilings, so scale conservatively (VULN-12).
|
||||
const weekMatch = val.match(/(?:≈|~)?\s*([0-9]+(?:,[0-9]+)*(?:\.[0-9]+)?)\s*(?:agentic\s*)?requests?\s*(?:\/|\s*per\s*)\s*(?:week|7-day)/i) ||
|
||||
val.match(/([0-9]+(?:,[0-9]+)*)\s*\/\s*week/i);
|
||||
if (weekMatch) {
|
||||
return Math.round(parseFloat(weekMatch[1].replace(/,/g, '')) * 4);
|
||||
return Math.round(parseFloat(weekMatch[1].replace(/,/g, '')) * (52 / 12));
|
||||
}
|
||||
|
||||
// 4. Simple leading count: e.g. "50 agentic requests/mo", "50 requests"
|
||||
|
||||
@@ -136,3 +136,56 @@ describe('Extended Thinking & Reasoning Token Engine', () => {
|
||||
expect(anthropicApi?.absorptionScore).toBeLessThanOrEqual(40);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Reasoning feasibility guard (VULN-06)', () => {
|
||||
const baseModel: NormalizedModel = {
|
||||
id: 'anthropic/claude-sonnet-5',
|
||||
name: 'Claude Sonnet 5',
|
||||
provider: 'anthropic',
|
||||
series: 'Claude',
|
||||
modality: 'text->text',
|
||||
contextWindow: 200000,
|
||||
maxOutput: 64000,
|
||||
pricing: { input: 3.0, output: 15.0, cachedInput: 0.3, cachedInputWrite: 3.75, reasoning: null, webSearch: null },
|
||||
blendedCost: 6.0,
|
||||
agentBlendedCost: 1.5,
|
||||
costPer1kRequests: 30.0,
|
||||
benchmarks: { intelligenceIndex: 88, codingIndex: 85, agenticIndex: 84, valueScore: 78 },
|
||||
reasoning: null,
|
||||
isFree: false,
|
||||
isBatch: false,
|
||||
};
|
||||
|
||||
const tinyModel: NormalizedModel = {
|
||||
...baseModel,
|
||||
id: 'tiny/small-model',
|
||||
name: 'Tiny Small Model',
|
||||
contextWindow: 32000,
|
||||
maxOutput: 8000,
|
||||
};
|
||||
|
||||
it('rejects max effort on a 32K-context model as an impossible combination', () => {
|
||||
const turn = calculateReasoningCost(tinyModel, 0.75, 'max');
|
||||
expect(turn.feasible).toBe(false);
|
||||
expect(turn.infeasibilityReason).toContain('32,000');
|
||||
});
|
||||
|
||||
it('rejects reasoning that exceeds the max completion cap even when context fits', () => {
|
||||
const cappedModel: NormalizedModel = { ...baseModel, contextWindow: 200000, maxOutput: 8000 };
|
||||
const turn = calculateReasoningCost(cappedModel, 0.75, 'high');
|
||||
expect(turn.feasible).toBe(false);
|
||||
expect(turn.infeasibilityReason).toContain('caps output');
|
||||
});
|
||||
|
||||
it('keeps medium effort feasible on the frontier model', () => {
|
||||
const turn = calculateReasoningCost(baseModel, 0.75, 'medium');
|
||||
expect(turn.feasible).toBe(true);
|
||||
expect(turn.infeasibilityReason).toBeNull();
|
||||
});
|
||||
|
||||
it('treats unknown windows (0) as feasible rather than fabricating a constraint', () => {
|
||||
const unknown = { ...baseModel, contextWindow: 0, maxOutput: 0 };
|
||||
const turn = calculateReasoningCost(unknown, 0.75, 'max');
|
||||
expect(turn.feasible).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -56,6 +56,10 @@ export interface ReasoningTurnCost {
|
||||
totalCost: number;
|
||||
reasoningTokens: number;
|
||||
inflationMultiplier: number; // totalCost / baseCost
|
||||
/** True when input + reasoning + output fit the model's context and output caps. */
|
||||
feasible: boolean;
|
||||
/** Human-readable reason when infeasible, else null. */
|
||||
infeasibilityReason: string | null;
|
||||
}
|
||||
|
||||
export interface MonthlyReasoningWorkload {
|
||||
@@ -214,6 +218,22 @@ export function calculateReasoningCost(
|
||||
const totalCost = baseCost + reasoningCost;
|
||||
const inflationMultiplier = baseCost > 0 ? totalCost / baseCost : 1.0;
|
||||
|
||||
// Feasibility guard (VULN-06): input + reasoning + output must fit inside
|
||||
// the model's context window, and reasoning + output must fit its max
|
||||
// completion cap. Unknown caps (0) are treated as feasible.
|
||||
let feasible = true;
|
||||
let infeasibilityReason: string | null = null;
|
||||
const requiredContext = AGENT_REQUEST_INPUT_TOKENS + reasoningTokens + AGENT_REQUEST_OUTPUT_TOKENS;
|
||||
if (model.contextWindow > 0 && requiredContext > model.contextWindow) {
|
||||
feasible = false;
|
||||
infeasibilityReason = `${effortSpec.label} reasoning needs ${requiredContext.toLocaleString()} context tokens but ${model.name} only supports ${model.contextWindow.toLocaleString()}.`;
|
||||
}
|
||||
const requiredOutput = reasoningTokens + AGENT_REQUEST_OUTPUT_TOKENS;
|
||||
if (feasible && model.maxOutput > 0 && requiredOutput > model.maxOutput) {
|
||||
feasible = false;
|
||||
infeasibilityReason = `${effortSpec.label} reasoning generates ${requiredOutput.toLocaleString()} completion tokens but ${model.name} caps output at ${model.maxOutput.toLocaleString()}.`;
|
||||
}
|
||||
|
||||
return {
|
||||
baseCost,
|
||||
inputCost,
|
||||
@@ -222,6 +242,8 @@ export function calculateReasoningCost(
|
||||
totalCost,
|
||||
reasoningTokens,
|
||||
inflationMultiplier: Number(inflationMultiplier.toFixed(2)),
|
||||
feasible,
|
||||
infeasibilityReason,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
+10
-2
@@ -38,6 +38,11 @@ export interface SessionReceiptReport {
|
||||
modelRepricings: ModelRepricingItem[];
|
||||
subscriptionImpacts: SubscriptionImpactItem[];
|
||||
headlineSummary: string;
|
||||
/**
|
||||
* False when the parser synthesized the token counts (VULN-05): the receipt
|
||||
* must be presented as an estimate, not a measured bill.
|
||||
*/
|
||||
tokensMeasured: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -173,9 +178,11 @@ export function calculateSessionReceipt(
|
||||
];
|
||||
|
||||
const cheapest = modelRepricings[0];
|
||||
const headlineSummary = cheapest && cheapest.sessionCost < baselineTotalCost
|
||||
const tokensMeasured = session.measuredTokens !== false;
|
||||
const estimatePrefix = tokensMeasured ? '' : 'ESTIMATED (transcript contained no token accounting — parser assumed ~21K in / 900 out per turn at 75% cache). ';
|
||||
const headlineSummary = estimatePrefix + (cheapest && cheapest.sessionCost < baselineTotalCost
|
||||
? `This session cost $${baselineTotalCost.toFixed(2)} on ${baselineModel?.name || 'Sonnet 5'}. It would have cost $${cheapest.sessionCost.toFixed(2)} on ${cheapest.modelName} (${cheapest.savingsPercentVsBaseline}% cheaper).`
|
||||
: `This session executed ${session.totalTurns} turns across ${(session.totalTokens / 1e6).toFixed(2)}M tokens at a ${Math.round(session.effectiveCacheHitRate * 100)}% prompt cache hit rate.`;
|
||||
: `This session executed ${session.totalTurns} turns across ${(session.totalTokens / 1e6).toFixed(2)}M tokens at a ${Math.round(session.effectiveCacheHitRate * 100)}% prompt cache hit rate.`);
|
||||
|
||||
return {
|
||||
session,
|
||||
@@ -185,5 +192,6 @@ export function calculateSessionReceipt(
|
||||
modelRepricings,
|
||||
subscriptionImpacts,
|
||||
headlineSummary,
|
||||
tokensMeasured,
|
||||
};
|
||||
}
|
||||
|
||||
+4
-1
@@ -175,7 +175,10 @@ function computePersonaApiCost(p: DeveloperPersona, model?: NormalizedModel): nu
|
||||
if (!model) return p.estimatedDirectApiCost;
|
||||
const inPrice = model.pricing.input ?? model.blendedCost * 0.75;
|
||||
const outPrice = model.pricing.output ?? model.blendedCost * 1.75;
|
||||
const cachedPrice = model.pricing.cachedInput ?? inPrice * 0.1;
|
||||
// Never fabricate a discount for an unknown cache price (invariant I):
|
||||
// an unknown cache rate is priced at full input, matching
|
||||
// getEffectiveCacheMultiplier's "no caching" convention.
|
||||
const cachedPrice = model.pricing.cachedInput ?? inPrice;
|
||||
const { costPerRequest } = calculateAgentRequestCost(model, 0.75);
|
||||
|
||||
const agentCost = p.monthlyAgentTurns * costPerRequest;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
export type ThrottleExhaustionBehavior = 'hard-block' | 'slow-queue' | 'payg-overage';
|
||||
export type ThrottleStatus = 'smooth' | 'queued' | 'blocked';
|
||||
/** 'overage' = served beyond quota as paid overage (payg-overage profiles, VULN-03). */
|
||||
export type ThrottleStatus = 'smooth' | 'queued' | 'overage' | 'blocked';
|
||||
|
||||
export interface ThrottleProfile {
|
||||
id: string;
|
||||
|
||||
@@ -136,3 +136,36 @@ describe('Throttle Simulation Engine', () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Payg-overage simulation (VULN-03)', () => {
|
||||
const getProfile = (id: string): ThrottleProfile => {
|
||||
const found = THROTTLE_PROFILES.find((p) => p.id === id);
|
||||
if (!found) throw new Error(`Profile ${id} not found`);
|
||||
return found;
|
||||
};
|
||||
|
||||
it('flags Google AI Pro as paid overage instead of infinitely smooth under saturating load', () => {
|
||||
const googlePro = getProfile('google-ai-pro');
|
||||
// 8 agents at rapid cadence burn through the 90-turn / 5h window fast.
|
||||
const result = simulateSprintThrottle(googlePro, {
|
||||
concurrency: 8,
|
||||
sprintDurationHours: 4,
|
||||
turnPace: 'rapid',
|
||||
});
|
||||
|
||||
expect(result.status).toBe('overage');
|
||||
expect(result.overageTurns).toBeGreaterThan(0);
|
||||
expect(result.throttleReason).not.toBeNull();
|
||||
});
|
||||
|
||||
it('never shows overage for plans that simply survive the sprint', () => {
|
||||
const claudeCode = getProfile('claude-code-pro');
|
||||
const result = simulateSprintThrottle(claudeCode, {
|
||||
concurrency: 1,
|
||||
sprintDurationHours: 2,
|
||||
turnPace: 'deep',
|
||||
});
|
||||
expect(result.status).toBe('smooth');
|
||||
expect(result.overageTurns).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
+76
-8
@@ -65,6 +65,8 @@ export interface SimulationResult {
|
||||
fastTurnsCompleted: number;
|
||||
slowTurnsCompleted: number;
|
||||
blockedTurns: number;
|
||||
/** Turns billed as paid overage when a payg-overage profile crossed its quota ceiling. */
|
||||
overageTurns: number;
|
||||
effectiveThroughputPercent: number;
|
||||
headroomScore: number; // 0 to 100
|
||||
timeline: SimulationTimelinePoint[];
|
||||
@@ -78,6 +80,9 @@ function formatMinutesToHours(minutes: number): string {
|
||||
return `${h}h ${m}m`;
|
||||
}
|
||||
|
||||
/** Severity rank so the sprint's overall status always reflects the worst outcome. */
|
||||
const STATUS_SEVERITY: Record<ThrottleStatus, number> = { smooth: 0, queued: 1, overage: 2, blocked: 3 };
|
||||
|
||||
/**
|
||||
* Runs a discrete time-step simulation (step = 5 minutes) of an intensive coding sprint
|
||||
* against a given throttle profile.
|
||||
@@ -118,6 +123,11 @@ export function simulateSprintThrottle(
|
||||
let slowTurnsCompleted = 0;
|
||||
let blockedTurns = 0;
|
||||
let cumulativeTurns = 0;
|
||||
// Deferred turns from prior queue delays (VULN-12): a queued step cannot
|
||||
// magically complete its full turn allocation; the backlog spills into
|
||||
// later steps and reduces throughput realistically.
|
||||
let deferredTurns = 0;
|
||||
let overageTurns = 0;
|
||||
|
||||
const timeline: SimulationTimelinePoint[] = [
|
||||
{
|
||||
@@ -134,10 +144,28 @@ export function simulateSprintThrottle(
|
||||
const hasConcurrencyViolation = safeConcurrency > profile.maxConcurrency;
|
||||
|
||||
for (let currentMinute = stepMinutes; currentMinute <= totalMinutes; currentMinute += stepMinutes) {
|
||||
const turnsAttempted = turnsPerStep;
|
||||
// Queue-delay backlog reduces the turns actually executable this step.
|
||||
// A 30s/turn slow-queue delay over a 5-minute step caps the step at
|
||||
// floor(step / delay) full-speed-equivalent allocations; the deficit is
|
||||
// deferred, not silently completed (VULN-12).
|
||||
let turnsAttempted = turnsPerStep;
|
||||
let stepStatus: ThrottleStatus = 'smooth';
|
||||
let stepDelay = 0;
|
||||
|
||||
if (deferredTurns > 0) {
|
||||
if (profile.exhaustionBehavior === 'hard-block') {
|
||||
// Blocked turns never recover; drop the backlog.
|
||||
deferredTurns = 0;
|
||||
} else if (profile.slowQueueDelaySec > 0) {
|
||||
const executableRatio = Math.min(1, (stepMinutes * 60) / (profile.slowQueueDelaySec * turnsPerStepPerAgent * safeConcurrency * safeConcurrency));
|
||||
const executable = Math.max(1, Math.floor(turnsPerStep * executableRatio));
|
||||
turnsAttempted = Math.min(turnsPerStep, executable);
|
||||
deferredTurns = Math.max(0, deferredTurns - (turnsPerStep - turnsAttempted));
|
||||
stepStatus = 'queued';
|
||||
stepDelay = profile.slowQueueDelaySec;
|
||||
}
|
||||
}
|
||||
|
||||
// 1. Check instantaneous concurrency
|
||||
if (hasConcurrencyViolation) {
|
||||
if (profile.exhaustionBehavior === 'hard-block') {
|
||||
@@ -176,6 +204,12 @@ export function simulateSprintThrottle(
|
||||
if (!firstThrottleReason) {
|
||||
firstThrottleReason = `Rolling ${profile.rollingWindowHours}h window exceeded: degraded to slow queue`;
|
||||
}
|
||||
} else if (profile.exhaustionBehavior === 'payg-overage') {
|
||||
// PAYG profiles never block; excess turns bill as paid overage (VULN-03).
|
||||
stepStatus = 'overage';
|
||||
if (!firstThrottleReason) {
|
||||
firstThrottleReason = `Rolling ${profile.rollingWindowHours}h window exceeded: excess turns continue as paid overage`;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -194,6 +228,11 @@ export function simulateSprintThrottle(
|
||||
if (!firstThrottleReason) {
|
||||
firstThrottleReason = `Monthly fast pool exhausted (${profile.monthlyFastRequests} reqs): sprint blocked`;
|
||||
}
|
||||
} else if (profile.exhaustionBehavior === 'payg-overage') {
|
||||
if (!firstThrottleReason) {
|
||||
firstThrottleReason = `Monthly pool exhausted (${profile.monthlyFastRequests} reqs): excess turns continue as paid overage`;
|
||||
}
|
||||
stepStatus = 'overage';
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -201,25 +240,53 @@ export function simulateSprintThrottle(
|
||||
// Process turns for this step
|
||||
if (stepStatus === 'blocked') {
|
||||
blockedTurns += turnsAttempted;
|
||||
if (firstThrottleMinute === null) {
|
||||
firstThrottleMinute = currentMinute;
|
||||
if (STATUS_SEVERITY['blocked'] > STATUS_SEVERITY[overallStatus]) {
|
||||
overallStatus = 'blocked';
|
||||
}
|
||||
} else if (stepStatus === 'queued') {
|
||||
slowTurnsCompleted += turnsAttempted;
|
||||
if (firstThrottleMinute === null) {
|
||||
firstThrottleMinute = currentMinute;
|
||||
}
|
||||
} else if (stepStatus === 'overage') {
|
||||
// Served beyond quota — turns complete but bill as paid overage (VULN-03).
|
||||
overageTurns += turnsAttempted;
|
||||
cumulativeTurns += turnsAttempted;
|
||||
rollingWindowHistory.push({ minute: currentMinute, turns: turnsAttempted });
|
||||
if (remainingMonthlyFast !== null) {
|
||||
remainingMonthlyFast = Math.max(0, remainingMonthlyFast - turnsAttempted);
|
||||
}
|
||||
if (STATUS_SEVERITY['overage'] > STATUS_SEVERITY[overallStatus]) {
|
||||
overallStatus = 'overage';
|
||||
}
|
||||
if (firstThrottleMinute === null) {
|
||||
firstThrottleMinute = currentMinute;
|
||||
}
|
||||
} else if (stepStatus === 'queued') {
|
||||
// Queue degradation defers the un-executable share of this step's turns.
|
||||
const deferred = Math.max(0, turnsAttempted - Math.max(1, Math.floor(turnsPerStep * 0.7)));
|
||||
deferredTurns += deferred;
|
||||
slowTurnsCompleted += turnsAttempted;
|
||||
cumulativeTurns += turnsAttempted;
|
||||
if (profile.exhaustionBehavior === 'payg-overage') {
|
||||
overageTurns += turnsAttempted;
|
||||
}
|
||||
rollingWindowHistory.push({ minute: currentMinute, turns: turnsAttempted });
|
||||
if (remainingMonthlyFast !== null) {
|
||||
remainingMonthlyFast = Math.max(0, remainingMonthlyFast - turnsAttempted);
|
||||
}
|
||||
if (STATUS_SEVERITY['queued'] > STATUS_SEVERITY[overallStatus]) {
|
||||
overallStatus = 'queued';
|
||||
}
|
||||
if (firstThrottleMinute === null) {
|
||||
firstThrottleMinute = currentMinute;
|
||||
overallStatus = overallStatus === 'blocked' ? 'blocked' : 'queued';
|
||||
}
|
||||
} else {
|
||||
// Smooth step
|
||||
fastTurnsCompleted += turnsAttempted;
|
||||
cumulativeTurns += turnsAttempted;
|
||||
if (profile.exhaustionBehavior === 'payg-overage' && firstThrottleReason !== null) {
|
||||
// Once a PAYG ceiling was crossed, all subsequent turns bill as overage.
|
||||
overageTurns += turnsAttempted;
|
||||
}
|
||||
rollingWindowHistory.push({ minute: currentMinute, turns: turnsAttempted });
|
||||
if (remainingMonthlyFast !== null) {
|
||||
remainingMonthlyFast = Math.max(0, remainingMonthlyFast - turnsAttempted);
|
||||
@@ -278,6 +345,7 @@ export function simulateSprintThrottle(
|
||||
blockedTurns: Math.round(blockedTurns),
|
||||
effectiveThroughputPercent: Math.min(100, Math.max(0, effectiveThroughputPercent)),
|
||||
headroomScore: Math.min(100, Math.max(0, headroomScore)),
|
||||
overageTurns: Math.round(overageTurns),
|
||||
timeline,
|
||||
};
|
||||
}
|
||||
@@ -289,12 +357,12 @@ export function simulateAllProfiles(params: SprintSimulationParams): SimulationR
|
||||
const results = THROTTLE_PROFILES.map((profile) => simulateSprintThrottle(profile, params));
|
||||
|
||||
// Sort order:
|
||||
// 1. Smooth before Queued before Blocked
|
||||
// 1. Smooth before Queued before Overage before Blocked
|
||||
// 2. Higher headroom score
|
||||
// 3. Longer survival hours
|
||||
// 4. Lower monthly price
|
||||
return results.sort((a, b) => {
|
||||
const statusWeight = { smooth: 3, queued: 2, blocked: 1 };
|
||||
const statusWeight = { smooth: 4, queued: 3, overage: 2, blocked: 1 };
|
||||
if (statusWeight[a.status] !== statusWeight[b.status]) {
|
||||
return statusWeight[b.status] - statusWeight[a.status];
|
||||
}
|
||||
|
||||
+9
-4
@@ -58,16 +58,21 @@ describe('classifyTraining', () => {
|
||||
});
|
||||
|
||||
describe('getIndemnityInfo', () => {
|
||||
it('labels boolean true as full indemnity', () => {
|
||||
expect(getIndemnityInfo(plan({ ipIndemnity: true })).label).toBe('Full Indemnity');
|
||||
it('never labels boolean true as unconditional "Full Indemnity" (VULN-10)', () => {
|
||||
const info = getIndemnityInfo(plan({ ipIndemnity: true }));
|
||||
expect(info.label).toBe('Indemnity Offered');
|
||||
expect(info.tone).toBe('warning');
|
||||
expect(info.description).toContain('carve-outs');
|
||||
});
|
||||
|
||||
it('labels z-ai-style strings as enterprise only', () => {
|
||||
expect(getIndemnityInfo(plan({ ipIndemnity: 'Enterprise Team contracts only' })).label).toBe('Enterprise Only');
|
||||
});
|
||||
|
||||
it('labels an unknown string as unclear rather than none', () => {
|
||||
expect(getIndemnityInfo(plan({ ipIndemnity: 'Full indemnity on all paid plans' })).label).toBe('Full Indemnity');
|
||||
it('flags indemnity strings with a verify-scope caveat rather than promising full coverage', () => {
|
||||
const info = getIndemnityInfo(plan({ ipIndemnity: 'Full indemnity on all paid plans' }));
|
||||
expect(info.label).toBe('Indemnity Stated (verify scope)');
|
||||
expect(info.tone).toBe('warning');
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
+24
-7
@@ -48,7 +48,8 @@ const TRAINS_PHRASES = [
|
||||
'used for model improvement',
|
||||
'interactions are recorded and used',
|
||||
'may be used for model improvement',
|
||||
'no guarantee',
|
||||
// 'no guarantee' removed (VULN-10): hedges about uptime/confidentiality are
|
||||
// not evidence that a vendor trains on customer data.
|
||||
];
|
||||
|
||||
const OPT_OUT_PHRASES = ['opt-out', 'opt out', 'privacy mode', 'training opt-out', 'in-settings opt-out'];
|
||||
@@ -152,9 +153,13 @@ export function getIndemnityInfo(plan: CodingPlan): IndemnityInfo {
|
||||
const ip = plan.ipIndemnity;
|
||||
if (ip === true) {
|
||||
return {
|
||||
label: 'Full Indemnity',
|
||||
tone: 'success',
|
||||
description: 'Full legal IP indemnification covered for copyright infringement on generated code.',
|
||||
// VULN-10: a boolean cannot establish unconditional coverage. Real
|
||||
// indemnities carry carve-outs (modification, combination, breach of
|
||||
// usage policy) and liability caps — never present this as "Full".
|
||||
label: 'Indemnity Offered',
|
||||
tone: 'warning',
|
||||
description:
|
||||
'Vendor advertises IP indemnification. Scope, carve-outs and liability caps are defined in the vendor’s legal terms — verify before relying on this for compliance.',
|
||||
};
|
||||
}
|
||||
if (typeof ip === 'string') {
|
||||
@@ -163,7 +168,11 @@ export function getIndemnityInfo(plan: CodingPlan): IndemnityInfo {
|
||||
return { label: 'Enterprise Only', tone: 'warning', description: ip };
|
||||
}
|
||||
if (lower.includes('full') || lower.includes('all plans')) {
|
||||
return { label: 'Full Indemnity', tone: 'success', description: ip };
|
||||
return {
|
||||
label: 'Indemnity Stated (verify scope)',
|
||||
tone: 'warning',
|
||||
description: ip,
|
||||
};
|
||||
}
|
||||
return { label: 'Unclear', tone: 'muted', description: ip };
|
||||
}
|
||||
@@ -182,7 +191,10 @@ export interface GotchaClassification {
|
||||
}
|
||||
|
||||
const RESTRICTION_WORDS = ['ban', 'banned', 'suspend', 'suspension', 'bulk', 'multiple accounts', 'multi-account', 'account sharing', 'unauthorized', 'tool-only', 'lock-in', 'error 1113', 'circumvent', 'breach', 'prohibited'];
|
||||
const PRIVACY_WORDS = ['train', 'training', 'privacy', 'telemetry', 'human review', 'data retention', 'zero retention'];
|
||||
/** Strong privacy signals that warrant CRITICAL severity (VULN-10). */
|
||||
const PRIVACY_STRONG_WORDS = ['train on', 'trains on', 'trained on', 'used for training', 'used to train', 'human review', 'data retention', 'zero retention', 'not used for training'];
|
||||
/** Weak privacy mentions (e.g. "training" inside a caching sentence) only justify a warning. */
|
||||
const PRIVACY_WEAK_WORDS = ['train', 'training', 'privacy', 'telemetry'];
|
||||
const LEGAL_WORDS = ['indemnity', 'indemnification', 'copyright', 'liability', 'ip ', 'legal'];
|
||||
const BILLING_WORDS = ['bill', 'billing', 'fee', 'fees', 'charge', 'overage', 'refund', 'refunds', 'cost', 'expire', 'expires', 'rate', '$', 'price', 'non-refundable'];
|
||||
const LIMIT_WORDS = ['limit', 'limits', 'throttle', 'cooldown', 'hard limit', 'quota', 'cap', 'resets', 'reset'];
|
||||
@@ -200,9 +212,14 @@ export function classifyGotcha(text: string): GotchaClassification {
|
||||
if (matchesWord(lower, RESTRICTION_WORDS)) {
|
||||
return { category: 'restrictions', severity: 'critical' };
|
||||
}
|
||||
if (matchesWord(lower, PRIVACY_WORDS)) {
|
||||
if (matchesWord(lower, PRIVACY_STRONG_WORDS)) {
|
||||
return { category: 'data-privacy', severity: 'critical' };
|
||||
}
|
||||
if (matchesWord(lower, PRIVACY_WEAK_WORDS)) {
|
||||
// Weak signal: the word "training"/"privacy" appears without an actual
|
||||
// training-on-data claim (e.g. "Privacy mode keeps code out of training").
|
||||
return { category: 'data-privacy', severity: 'warning' };
|
||||
}
|
||||
if (matchesWord(lower, LEGAL_WORDS)) {
|
||||
return { category: 'ip-legal', severity: 'warning' };
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
type TurnCadence,
|
||||
} from '../lib/throttle';
|
||||
import type { ThrottleStatus } from '../lib/throttle-profiles';
|
||||
import { numParam } from '../lib/params';
|
||||
import {
|
||||
Zap,
|
||||
Gauge,
|
||||
@@ -20,6 +21,7 @@ import {
|
||||
RotateCcw,
|
||||
Sparkles,
|
||||
Info,
|
||||
DollarSign,
|
||||
} from 'lucide-react';
|
||||
import {
|
||||
ResponsiveContainer,
|
||||
@@ -74,12 +76,13 @@ const PRESETS: Preset[] = [
|
||||
export default function BurstSimulator() {
|
||||
const [searchParams, setSearchParams] = useSearchParams();
|
||||
|
||||
// Read initial parameters from URL or use defaults
|
||||
const initialConcurrency = Math.min(10, Math.max(1, Number(searchParams.get('agents')) || 2));
|
||||
const initialDuration = Math.min(8, Math.max(1, Number(searchParams.get('duration')) || 4));
|
||||
// Read initial parameters from URL or use defaults (zero-safe: `?prior=0`
|
||||
// and `?agents=0`-style values must not collapse to defaults, VULN-13)
|
||||
const initialConcurrency = numParam(searchParams.get('agents'), 2, 1, 10);
|
||||
const initialDuration = numParam(searchParams.get('duration'), 4, 1, 8);
|
||||
const rawCadence = searchParams.get('cadence') as TurnCadence;
|
||||
const initialCadence: TurnCadence = (rawCadence && CADENCE_DEFINITIONS[rawCadence]) ? rawCadence : 'standard';
|
||||
const initialPrior = Math.min(90, Math.max(0, Number(searchParams.get('prior')) || 20));
|
||||
const initialPrior = numParam(searchParams.get('prior'), 20, 0, 90);
|
||||
|
||||
const [concurrency, setConcurrency] = useState<number>(initialConcurrency);
|
||||
const [sprintDuration, setSprintDuration] = useState<number>(initialDuration);
|
||||
@@ -152,6 +155,7 @@ export default function BurstSimulator() {
|
||||
const estimatedTokensM = Number(((totalTurnsAttempted * 21000) / 1000000).toFixed(1));
|
||||
const smoothCount = simulationResults.filter((r) => r.status === 'smooth').length;
|
||||
const queuedCount = simulationResults.filter((r) => r.status === 'queued').length;
|
||||
const overageCount = simulationResults.filter((r) => r.status === 'overage').length;
|
||||
const blockedCount = simulationResults.filter((r) => r.status === 'blocked').length;
|
||||
|
||||
const filteredResults = useMemo(() => {
|
||||
@@ -489,7 +493,7 @@ export default function BurstSimulator() {
|
||||
<div className="flex justify-between gap-4 capitalize">
|
||||
<span className="text-text-muted">Status:</span>
|
||||
<span className={`font-semibold ${
|
||||
data.status === 'smooth' ? 'text-emerald-400' : data.status === 'queued' ? 'text-amber-400' : 'text-rose-400'
|
||||
data.status === 'smooth' ? 'text-emerald-400' : data.status === 'queued' ? 'text-amber-400' : data.status === 'overage' ? 'text-orange-400' : 'text-rose-400'
|
||||
}`}>
|
||||
{data.status}
|
||||
</span>
|
||||
@@ -509,6 +513,8 @@ export default function BurstSimulator() {
|
||||
? '#10b981'
|
||||
: entry.status === 'queued'
|
||||
? '#f59e0b'
|
||||
: entry.status === 'overage'
|
||||
? '#f97316'
|
||||
: '#ef4444'
|
||||
}
|
||||
/>
|
||||
@@ -565,6 +571,17 @@ export default function BurstSimulator() {
|
||||
<AlertTriangle className="w-3.5 h-3.5" />
|
||||
Queued ({queuedCount})
|
||||
</button>
|
||||
<button
|
||||
onClick={() => setStatusFilter('overage')}
|
||||
className={`px-3 py-1 rounded font-medium transition-colors flex items-center gap-1 ${
|
||||
statusFilter === 'overage'
|
||||
? 'bg-orange-950/80 text-orange-300 border border-orange-600/40'
|
||||
: 'text-text-muted hover:bg-surface hover:text-orange-400'
|
||||
}`}
|
||||
>
|
||||
<DollarSign className="w-3.5 h-3.5" />
|
||||
Overage ({overageCount})
|
||||
</button>
|
||||
<button
|
||||
onClick={() => setStatusFilter('blocked')}
|
||||
className={`px-3 py-1 rounded font-medium transition-colors flex items-center gap-1 ${
|
||||
@@ -583,7 +600,12 @@ export default function BurstSimulator() {
|
||||
<div className="divide-y divide-steel-800/80">
|
||||
{filteredResults.map((result) => {
|
||||
const isSelected = activeDetail?.profile.id === result.profile.id;
|
||||
const statusConfig = {
|
||||
const statusConfigTable: Record<ThrottleStatus, {
|
||||
label: string;
|
||||
badge: string;
|
||||
icon: typeof ShieldCheck;
|
||||
color: string;
|
||||
}> = {
|
||||
smooth: {
|
||||
label: 'Smooth Sprint',
|
||||
badge: 'bg-emerald-950/60 text-emerald-300 border-emerald-600/40',
|
||||
@@ -596,13 +618,20 @@ export default function BurstSimulator() {
|
||||
icon: AlertTriangle,
|
||||
color: 'text-amber-400',
|
||||
},
|
||||
overage: {
|
||||
label: 'Paid Overage',
|
||||
badge: 'bg-orange-950/60 text-orange-300 border-orange-600/40',
|
||||
icon: DollarSign,
|
||||
color: 'text-orange-400',
|
||||
},
|
||||
blocked: {
|
||||
label: 'Session Blocked',
|
||||
badge: 'bg-rose-950/60 text-rose-300 border-rose-600/40',
|
||||
icon: XCircle,
|
||||
color: 'text-rose-400',
|
||||
},
|
||||
}[result.status];
|
||||
};
|
||||
const statusConfig = statusConfigTable[result.status];
|
||||
|
||||
const StatusIcon = statusConfig.icon;
|
||||
|
||||
@@ -654,7 +683,7 @@ export default function BurstSimulator() {
|
||||
</div>
|
||||
|
||||
{/* Turns Throughput Breakdown */}
|
||||
<div className="grid grid-cols-3 gap-3 text-center min-w-[200px]">
|
||||
<div className="grid grid-cols-4 gap-3 text-center min-w-[240px]">
|
||||
<div className="p-1.5 rounded bg-void-950/40 border border-steel-800">
|
||||
<div className="text-[10px] uppercase text-text-muted">Fast</div>
|
||||
<div className="text-xs font-mono font-bold text-emerald-400">
|
||||
@@ -667,6 +696,12 @@ export default function BurstSimulator() {
|
||||
{result.slowTurnsCompleted}
|
||||
</div>
|
||||
</div>
|
||||
<div className="p-1.5 rounded bg-void-950/40 border border-steel-800">
|
||||
<div className="text-[10px] uppercase text-text-muted">Overage</div>
|
||||
<div className="text-xs font-mono font-bold text-orange-400">
|
||||
{result.overageTurns}
|
||||
</div>
|
||||
</div>
|
||||
<div className="p-1.5 rounded bg-void-950/40 border border-steel-800">
|
||||
<div className="text-[10px] uppercase text-text-muted">Blocked</div>
|
||||
<div className="text-xs font-mono font-bold text-rose-400">
|
||||
|
||||
@@ -6,8 +6,10 @@ import { LoadingSpinner } from '../components/shared/LoadingSpinner';
|
||||
import {
|
||||
HARDWARE_PRESETS,
|
||||
computeHardwareEconomics,
|
||||
SALVAGE_CLAMP_MAX,
|
||||
type HardwarePreset,
|
||||
} from '../lib/hardware';
|
||||
import { numParam } from '../lib/params';
|
||||
import { formatPrice } from '../lib/pricing';
|
||||
import {
|
||||
Cpu,
|
||||
@@ -24,6 +26,7 @@ import {
|
||||
Layers,
|
||||
Sparkles,
|
||||
} from 'lucide-react';
|
||||
|
||||
import {
|
||||
ResponsiveContainer,
|
||||
LineChart,
|
||||
@@ -43,13 +46,13 @@ export default function HardwareBreakeven() {
|
||||
|
||||
// Read initial params
|
||||
const initialPresetId = searchParams.get('preset') || 'mac-mini-m4-pro-64gb';
|
||||
const initialAmortization = Math.min(48, Math.max(6, Number(searchParams.get('amortization')) || 24));
|
||||
const initialSalvage = Math.min(50, Math.max(0, Number(searchParams.get('salvage')) || 20));
|
||||
const initialKwhCost = Math.min(1.0, Math.max(0.01, Number(searchParams.get('kwh')) || 0.16));
|
||||
const initialDailyInference = Math.min(24, Math.max(1, Number(searchParams.get('infHours')) || 3));
|
||||
const initialDailyIdle = Math.min(24, Math.max(0, Number(searchParams.get('idleHours')) || 9));
|
||||
const initialVolumeM = Math.min(100, Math.max(0.5, Number(searchParams.get('volume')) || 6));
|
||||
const initialInputRatio = Math.min(0.95, Math.max(0.5, Number(searchParams.get('inputRatio')) || 0.8));
|
||||
const initialAmortization = numParam(searchParams.get('amortization'), 24, 6, 48);
|
||||
const initialSalvage = numParam(searchParams.get('salvage'), 20, 0, SALVAGE_CLAMP_MAX);
|
||||
const initialKwhCost = numParam(searchParams.get('kwh'), 0.16, 0.01, 1.0);
|
||||
const initialDailyInference = numParam(searchParams.get('infHours'), 3, 1, 24);
|
||||
const initialDailyIdle = numParam(searchParams.get('idleHours'), 9, 0, 24);
|
||||
const initialVolumeM = numParam(searchParams.get('volume'), 6, 0.5, 100);
|
||||
const initialInputRatio = numParam(searchParams.get('inputRatio'), 0.8, 0.5, 0.95);
|
||||
const initialApiId = searchParams.get('api') || 'anthropic/claude-3.7-sonnet';
|
||||
const initialPlanId = searchParams.get('plan') || 'cursor-pro';
|
||||
|
||||
@@ -574,6 +577,28 @@ export default function HardwareBreakeven() {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Workload feasibility warning (VULN-04) */}
|
||||
{economics.infeasibleWorkload && (
|
||||
<div className="p-3 rounded-lg bg-amber-950/50 border border-amber-700/40 text-xs text-amber-200 flex items-start gap-2">
|
||||
<AlertTriangle className="w-4 h-4 text-amber-400 shrink-0 mt-0.5" />
|
||||
<div>
|
||||
<span className="font-bold">Infeasible workload for this duty cycle.</span> Generating{' '}
|
||||
{monthlyTokenVolumeM}M tokens/mo needs ~{economics.requiredInferenceHours}h of pure decode at{' '}
|
||||
{currentPreset.recommendedModels[0]?.tokensPerSec ?? 0} tok/s, but your configured schedule
|
||||
allows at most {economics.feasibleMonthlyVolumeM}M tok/mo ({dailyInferenceHours}h/day × 30.4).
|
||||
Either raise daily inference hours, lower the volume target, or pick a faster preset —
|
||||
the crossover verdict above cannot be physically realized at this setting.
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{!economics.infeasibleWorkload && (
|
||||
<p className="text-[11px] font-mono text-steel-500">
|
||||
Feasibility check: {economics.requiredInferenceHours}h decode required vs{' '}
|
||||
{(dailyInferenceHours * 30.4375).toFixed(1)}h available — workload fits the configured duty cycle.
|
||||
Payback is quoted against full CapEx; headline savings use the amortized basis.
|
||||
</p>
|
||||
)}
|
||||
|
||||
{/* Interactive Chart Container */}
|
||||
<div className="border border-steel-700/60 rounded-xl p-5 bg-surface space-y-4">
|
||||
<div className="flex flex-col sm:flex-row sm:items-center justify-between gap-3 border-b border-steel-700/60 pb-3">
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
calculateReasoningCost,
|
||||
calculateMonthlyReasoningWorkload,
|
||||
type ReasoningEffort,
|
||||
type ReasoningTurnCost,
|
||||
} from '../lib/reasoning';
|
||||
import type { CacheRate } from '../lib/types';
|
||||
import { formatPrice } from '../lib/pricing';
|
||||
@@ -109,6 +110,16 @@ export default function ReasoningExploder() {
|
||||
return calculateMonthlyReasoningWorkload(activeModel, turns, cacheRate, effort);
|
||||
}, [activeModel, turns, cacheRate, effort]);
|
||||
|
||||
// Feasibility per effort level for the active model (VULN-06)
|
||||
const effortFeasibility = useMemo(() => {
|
||||
if (!activeModel) return null;
|
||||
const map = {} as Record<ReasoningEffort, ReasoningTurnCost>;
|
||||
for (const eKey of ['off', 'low', 'medium', 'high', 'max'] as ReasoningEffort[]) {
|
||||
map[eKey] = calculateReasoningCost(activeModel, cacheRate, eKey);
|
||||
}
|
||||
return map;
|
||||
}, [activeModel, cacheRate]);
|
||||
|
||||
// Popular reasoning benchmark models for side-by-side comparison
|
||||
const comparisonModels = useMemo(() => {
|
||||
const targets = [
|
||||
@@ -215,22 +226,38 @@ export default function ReasoningExploder() {
|
||||
{(['off', 'low', 'medium', 'high', 'max'] as ReasoningEffort[]).map((eKey) => {
|
||||
const spec = REASONING_EFFORT_SPECS[eKey];
|
||||
const isActive = effort === eKey;
|
||||
const infeasible = effortFeasibility != null && !effortFeasibility[eKey].feasible;
|
||||
const infeasibleTitle = effortFeasibility?.[eKey].infeasibilityReason ?? '';
|
||||
return (
|
||||
<button
|
||||
key={eKey}
|
||||
onClick={() => handleEffortChange(eKey)}
|
||||
onClick={() => { if (!infeasible) handleEffortChange(eKey); }}
|
||||
disabled={infeasible}
|
||||
className={`px-3.5 py-1.5 rounded-full text-xs font-medium border transition-all ${
|
||||
isActive
|
||||
infeasible
|
||||
? 'border-steel-800 bg-void-950/60 text-steel-600 line-through cursor-not-allowed'
|
||||
: isActive
|
||||
? 'border-blood-500/80 bg-blood-950/60 text-blood-200 shadow-[0_0_12px_hsl(0_70%_40%_/0.25)]'
|
||||
: 'border-steel-700/60 bg-void-950/40 text-steel-300 hover:border-steel-500 hover:text-text'
|
||||
}`}
|
||||
title={spec.description}
|
||||
title={infeasible ? `Infeasible: ${infeasibleTitle}` : spec.description}
|
||||
>
|
||||
{spec.label}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
{/* Infeasible effort warning (VULN-06) */}
|
||||
{effortFeasibility && !effortFeasibility[effort].feasible && (
|
||||
<div className="mt-3 p-3 rounded-lg bg-amber-950/50 border border-amber-700/40 text-xs text-amber-200 flex items-start gap-2">
|
||||
<AlertTriangle className="w-4 h-4 text-amber-400 shrink-0 mt-0.5" />
|
||||
<div>
|
||||
<span className="font-bold">Impossible combination.</span>{' '}
|
||||
{effortFeasibility[effort].infeasibilityReason}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Interactive Controls & Cost KPIs */}
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
RotateCcw,
|
||||
Zap,
|
||||
TrendingDown,
|
||||
AlertTriangle,
|
||||
} from 'lucide-react';
|
||||
|
||||
export default function SessionReceipt() {
|
||||
@@ -85,6 +86,7 @@ export default function SessionReceipt() {
|
||||
`================================================`,
|
||||
` TOKEN-MAX AGENT SESSION RECEIPT `,
|
||||
`================================================`,
|
||||
receiptReport.tokensMeasured ? null : `*** ESTIMATED: transcript had no token accounting. ***`,
|
||||
`Session: ${activeSession.sessionTitle}`,
|
||||
`Turns Executed: ${activeSession.totalTurns}`,
|
||||
`Tool Invocations: ${activeSession.toolInvocationsCount}`,
|
||||
@@ -212,6 +214,18 @@ export default function SessionReceipt() {
|
||||
{errorMsg}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!receiptReport.tokensMeasured && (
|
||||
<div className="mt-4 p-3 rounded-lg bg-amber-950/60 border border-amber-600/40 text-xs text-amber-200 flex items-start gap-2">
|
||||
<AlertTriangle className="w-4 h-4 text-amber-400 shrink-0 mt-0.5" />
|
||||
<div>
|
||||
<span className="font-bold">Estimated receipt.</span> This transcript contained no token
|
||||
accounting — token counts and costs below are synthesized from the standard 21K agent
|
||||
request shape, not measured from your session. Treat every dollar figure as a rough
|
||||
planning estimate.
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Session KPI Summary Cards */}
|
||||
|
||||
Reference in New Issue
Block a user