mirror of
https://github.com/codewhale-hq/Codewhale.git
synced 2026-11-08 08:04:25 -10:00
fix(pricing): price live models.dev rates instead of stripping them (#6396 item 6)
The repair checklist asks for live models.dev rates to be accepted as published-price estimates. `verified_catalog_offering` did the opposite: a models.dev live overlay had `cost` set to `None` outright, on the reasoning that models.dev is a capabilities catalog and leftover price fields must not be billed as `provider_live` (#5241). The reasoning was right about the danger and wrong about the remedy. The danger is billing a catalog claim at provider-live confidence; the remedy is the confidence, not the price. `PricingProvenance::ModelsDevBundled` already models exactly this — its own doc says "the bundled snapshot or a live refresh of it. Both are the same third-party claim about a model" — and `live_pricing_defect` returns early for anything that is not `ProviderLive`, so a models.dev row is never faulted by the endpoint-fingerprint or freshness gates. Deleting the strip therefore prices live rates at the confidence they actually carry. One substitution rather than a clean deletion. A row tagged `CatalogSource::Live` that the models.dev live partition owns is mislabelled: `Live` means "the provider published this for one endpoint", but models.dev is unauthenticated and its rows deliberately carry no endpoint fingerprint (`CatalogSource::ModelsDevLive` doc). Restamping that row's `cost_source` through the existing field — the same mechanism `complete_facts_from` uses — lets it price as `ModelsDevBundled` while dropping an endpoint scope it never earned. Genuine `ProviderLive` rows keep every gate. Deleting the block instead would also have left `provider_lake::live_catalog_origin` with no production caller, which is a hard error under the workspace's `warnings = "deny"`. `capabilities_only_offering` is deleted with no callers left. `UnpricedReason::UnverifiedLivePricing` keeps its other producers. Two existing tests asserted the old contract and now assert the new one: `models_dev_live_rates_price_the_published_estimate` (live input 99.0/M over 1M input tokens -> 99.0 USD, `ModelsDevBundled`) and `models_dev_partition_live_rows_price_the_live_catalog_rate` (10K tokens -> 0.99 USD; the old 0.05 came from a bundled 5.0/M row). Evidence: - nextest -p codewhale-tui -E 'test(pricing::tests::)' — 85 passed, 0 failed - nextest -p codewhale-tui -E 'test(tui::model_picker::) | test(models_dev_live) | test(provider_lake::)' — 81 passed, 0 failed Still open on #6396: items 5 (whole-row shadowing), 9 (context/output budgeting), 11 (the singular `codewhale models` command still builds `ModelRegistry::default()`), 13 (corrections-shrink) and the installed acceptance.
This commit is contained in:
1 parent
082d347252
commit
22a6695ae8
1 file changed
+24
-38
+24
-38
@@ -1741,7 +1741,7 @@ fn verified_catalog_offering(
|
||||
recorded_at: DateTime<Utc>,
|
||||
allow_cloud_catalog: bool,
|
||||
) -> VerifiedOffering {
|
||||
let Some(offering) = crate::provider_lake::catalog_offering_for_model_identity(
|
||||
let Some(mut offering) = crate::provider_lake::catalog_offering_for_model_identity(
|
||||
provider,
|
||||
provider_identity,
|
||||
catalog_model,
|
||||
@@ -1784,24 +1784,18 @@ fn verified_catalog_offering(
|
||||
.map(VerifiedOffering::Usable)
|
||||
.unwrap_or(VerifiedOffering::Absent);
|
||||
}
|
||||
// Models.dev is a capabilities catalog. A live overlay from that fetch
|
||||
// must never be treated as a rate source — leftover `cost` fields are
|
||||
// not provider prices, and `https://api.codewhale.net/session` 503
|
||||
// (`control_plane_not_attached`) is not a healthy live price list
|
||||
// (#5241). Prefer the bundled snapshot (curated in-repo rates, when
|
||||
// present) and otherwise ignore live cost so hand/bundled fallbacks
|
||||
// can restore a usable session total.
|
||||
if matches!(
|
||||
offering.pricing_source(),
|
||||
codewhale_config::catalog::CatalogSource::ModelsDevLive { .. }
|
||||
) || (!cloud_price
|
||||
&& crate::provider_lake::live_catalog_origin(provider, catalog_model)
|
||||
== Some(crate::provider_lake::LiveSource::ModelsDev))
|
||||
{
|
||||
let offering =
|
||||
crate::provider_lake::bundled_catalog_offering_for_model(provider, catalog_model)
|
||||
.unwrap_or_else(|| capabilities_only_offering(offering));
|
||||
return VerifiedOffering::Usable(offering);
|
||||
let models_dev_live_fetched_at = match offering.pricing_source() {
|
||||
codewhale_config::catalog::CatalogSource::Live { fetched_at, .. }
|
||||
if crate::provider_lake::live_catalog_origin(provider, catalog_model)
|
||||
== Some(crate::provider_lake::LiveSource::ModelsDev) =>
|
||||
{
|
||||
Some(*fetched_at)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
if let Some(fetched_at) = models_dev_live_fetched_at {
|
||||
offering.cost_source =
|
||||
Some(codewhale_config::catalog::CatalogSource::ModelsDevLive { fetched_at });
|
||||
}
|
||||
let Some(pricing) = OfferingPricing::from_catalog_offering_at(
|
||||
&offering,
|
||||
@@ -1835,15 +1829,6 @@ fn verified_catalog_offering(
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop any cost on a Models.dev live overlay so leftover price fields cannot
|
||||
/// be billed as `provider_live` (#5241).
|
||||
fn capabilities_only_offering(
|
||||
mut offering: codewhale_config::catalog::CatalogOffering,
|
||||
) -> codewhale_config::catalog::CatalogOffering {
|
||||
offering.cost = None;
|
||||
offering
|
||||
}
|
||||
|
||||
/// Project a hand-sourced provider row into an audit.
|
||||
///
|
||||
/// A hand row always publishes input, cache-read, and output rates. Cache-write
|
||||
@@ -5219,7 +5204,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn models_dev_live_cost_is_capabilities_only_and_falls_back_to_bundled_rates() {
|
||||
fn models_dev_live_rates_price_the_published_estimate() {
|
||||
let _live = crate::provider_lake::lock_live_snapshot();
|
||||
crate::provider_lake::clear_live_snapshot();
|
||||
let now = Utc::now();
|
||||
@@ -5260,12 +5245,12 @@ mod tests {
|
||||
audit.unpriced_reason,
|
||||
Some(UnpricedReason::UnverifiedLivePricing)
|
||||
);
|
||||
assert_eq!(audit.provenance, Some(PricingProvenance::ProviderDocs));
|
||||
assert_eq!(audit.provenance, Some(PricingProvenance::ModelsDevBundled));
|
||||
assert_eq!(audit.live_pricing_defect, None);
|
||||
let estimate = audit.estimate.expect("priced");
|
||||
assert!(
|
||||
(estimate.usd - 0.14).abs() < 1e-12,
|
||||
"models.dev leftover cost must not be billed: {}",
|
||||
(estimate.usd - 99.0).abs() < 1e-12,
|
||||
"live models.dev input rate must price the estimate: {}",
|
||||
estimate.usd
|
||||
);
|
||||
}
|
||||
@@ -5429,11 +5414,12 @@ mod tests {
|
||||
|
||||
/// The fixture deliberately stamps `Live` rather than the `ModelsDevLive`
|
||||
/// the refresh now emits: this pins the *second*, independent check — the
|
||||
/// live partition the row sits in — which is what still catches a row
|
||||
/// mislabelled by an older publisher or a stale on-disk cache. Do not
|
||||
/// "correct" the source here; that would delete this belt's only coverage.
|
||||
/// live partition the row sits in — which is what decides the price basis
|
||||
/// of a row mislabelled by an older publisher or a stale on-disk cache. Do
|
||||
/// not "correct" the source here; that would delete this belt's only
|
||||
/// coverage.
|
||||
#[test]
|
||||
fn models_dev_live_overlay_does_not_replace_bundled_catalog_rates() {
|
||||
fn models_dev_partition_live_rows_price_the_live_catalog_rate() {
|
||||
let _live = crate::provider_lake::lock_live_snapshot();
|
||||
crate::provider_lake::clear_live_snapshot();
|
||||
let now = Utc::now();
|
||||
@@ -5476,8 +5462,8 @@ mod tests {
|
||||
assert_eq!(audit.live_pricing_defect, None);
|
||||
let estimate = audit.estimate.expect("priced");
|
||||
assert!(
|
||||
(estimate.usd - 0.05).abs() < 1e-12,
|
||||
"bundled OpenAI rate must win over models.dev leftover cost: {}",
|
||||
(estimate.usd - 0.99).abs() < 1e-12,
|
||||
"the models.dev live rate must price the estimate: {}",
|
||||
estimate.usd
|
||||
);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user