mirror of
https://github.com/codewhale-hq/Codewhale.git
synced 2026-11-07 17:09:13 -10:00
feat(CW-7503): spend in proportion to risk, with a money cap that says whether it applied
#7: a docs typo and a billing migration were reviewed with the same fixed effort. `tools/review_triage.rs` now sets the budget from the diff alone, before any model call: - triage is a path and size read: sensitive paths (auth, billing, payment, migrations, crypto, secrets, permission/policy, sandbox, locking, deletion, schema) or a change of 400+ lines / 20+ files is high risk; API, route, server, config, database, storage and contract paths or 80+ lines / 6+ files is medium; everything else is low. Every reason is kept, so a receipt can explain its own tier instead of asserting one. - budget: low risk gets one step and no repository tools (the diff still gets reviewed), medium gets 4 steps and 6 tool calls, high gets 6 steps and 12 calls, with an output-token ceiling per tier. - the money cap is `--money-cap-usd` with `--input-usd-per-mtok` and `--output-usd-per-mtok`. A cap without a price cannot be enforced, so `money_limit_supported` stays false and the receipt reports the cap as unsupported rather than implying it ran out. The core checks the cap before each model step and again at the end. - a stop is named: money_cap, step_budget, tool_budget, output_limit or model_failure, and the summary carries a partial-review note naming the limit that stopped it and saying plainly that unreached files were not reviewed and are not thereby correct. It is never a silent truncation. Triage, budget, stop and spend travel on the tool metadata and the CLI JSON, so the same review can be audited after the fact. Checks: scripts/dev-cargo.sh check -p codewhale-tui --lib clean; scripts/dev-cargo.sh test -p codewhale-tui --lib tools::review = 80 passed, 0 failed (6 new triage tests: docs-low vs billing-migration-high, size and sensitive paths raising the tier independently, a cap without a price reported unsupported, cost over every billable counter, and the partial note naming its limit). Not claimed: cost per corpus pull request is not yet shown to fall with no recall loss, because the model-backed baseline is blocked on an approved eval route (see #1). The free deterministic-floor baseline is unaffected by this slice, since it never calls the core. Refs CW-7503, codewhale-agent #7
This commit is contained in:
1 parent
4194b0e755
commit
0ab0d4ac29
5 files changed
+467
-7
No files matched your search
+53
-2
@@ -1472,6 +1472,17 @@ struct ReviewArgs {
|
||||
/// additional model requests; the default preserves single-pass behavior.
|
||||
#[arg(long, default_value_t = 1)]
|
||||
max_passes: usize,
|
||||
/// Per-review money cap in USD. It is only enforceable together with
|
||||
/// `--input-usd-per-mtok` and `--output-usd-per-mtok`; without both, the
|
||||
/// receipt reports the cap as unsupported instead of implying it applied.
|
||||
#[arg(long)]
|
||||
money_cap_usd: Option<f64>,
|
||||
/// Input price per million tokens for this exact route, in USD
|
||||
#[arg(long, requires_all = ["money_cap_usd"])]
|
||||
input_usd_per_mtok: Option<f64>,
|
||||
/// Output price per million tokens for this exact route, in USD
|
||||
#[arg(long, requires_all = ["money_cap_usd"])]
|
||||
output_usd_per_mtok: Option<f64>,
|
||||
/// Write a durable pre-push review receipt after a successful review
|
||||
#[arg(long, default_value_t = false)]
|
||||
write_receipt: bool,
|
||||
@@ -10450,6 +10461,20 @@ async fn run_review(config: &Config, args: ReviewArgs) -> Result<()> {
|
||||
};
|
||||
let mut review_anchors = crate::tools::review_core::ReviewAnchorReport::default();
|
||||
let mut review_tools_used: Vec<String> = Vec::new();
|
||||
let review_triage = crate::tools::review_triage::triage_review(&diff);
|
||||
let review_rates = match (args.input_usd_per_mtok, args.output_usd_per_mtok) {
|
||||
(Some(input), Some(output)) => Some(crate::tools::review_triage::SpendRates {
|
||||
input_usd_per_million: input,
|
||||
output_usd_per_million: output,
|
||||
}),
|
||||
_ => None,
|
||||
};
|
||||
let review_budget = crate::tools::review_triage::budget_for(&review_triage).with_money_cap(
|
||||
args.money_cap_usd,
|
||||
review_rates.map(|rates| (rates.input_usd_per_million, rates.output_usd_per_million)),
|
||||
);
|
||||
let mut review_stop: Option<crate::tools::review_triage::ReviewStop> = None;
|
||||
let mut review_spend_usd = None;
|
||||
for (index, user_prompt) in prompts.into_iter().enumerate() {
|
||||
let reasoning_effort = route.reasoning_effort.and_then(|effort| {
|
||||
review_reasoning_effort_value_for_prompt(
|
||||
@@ -10461,8 +10486,15 @@ async fn run_review(config: &Config, args: ReviewArgs) -> Result<()> {
|
||||
});
|
||||
let mut request = crate::tools::review_core::ReviewCoreRequest::new(&diff, user_prompt);
|
||||
request.workspace = review_tool_workspace.clone();
|
||||
request.max_output_tokens = Some(client.effective_max_output_tokens(&request_route.model));
|
||||
request.budget = review_budget.tool_budget();
|
||||
request.max_output_tokens = Some(
|
||||
client
|
||||
.effective_max_output_tokens(&request_route.model)
|
||||
.min(review_budget.max_output_tokens),
|
||||
);
|
||||
request.reasoning_effort = reasoning_effort;
|
||||
request.spend_rates = review_rates;
|
||||
request.money_cap_usd = review_budget.money_cap_usd.filter(|_| review_budget.money_limit_supported);
|
||||
let result =
|
||||
match crate::tools::review_core::run_review_core(&client, &model, request).await {
|
||||
Ok(result) => result,
|
||||
@@ -10492,6 +10524,8 @@ async fn run_review(config: &Config, args: ReviewArgs) -> Result<()> {
|
||||
review_anchors.claims += result.anchors.claims;
|
||||
review_anchors.anchored += result.anchors.anchored;
|
||||
review_anchors.unanchored += result.anchors.unanchored;
|
||||
review_stop = result.stop.clone().or(review_stop);
|
||||
review_spend_usd = result.spend_usd.or(review_spend_usd);
|
||||
if codewhale_models::is_incomplete_stop_reason(review_stop_reason.as_deref()) {
|
||||
// #6285: budget exhaustion is an infrastructure outcome, not a
|
||||
// review verdict. The PR was never judged, so the message must not
|
||||
@@ -10531,7 +10565,10 @@ async fn run_review(config: &Config, args: ReviewArgs) -> Result<()> {
|
||||
output = pass_output;
|
||||
}
|
||||
}
|
||||
let (structured, coverage) = if let Some(accumulator) = accumulator {
|
||||
let partial_note = review_stop.as_ref().map(|stop| {
|
||||
crate::tools::review_triage::partial_review_note(stop, &review_triage, &review_budget)
|
||||
});
|
||||
let (mut structured, coverage) = if let Some(accumulator) = accumulator {
|
||||
let (review, content, coverage) = match accumulator.finish(&diff) {
|
||||
Ok(complete) => complete,
|
||||
Err(error) => {
|
||||
@@ -10548,6 +10585,16 @@ async fn run_review(config: &Config, args: ReviewArgs) -> Result<()> {
|
||||
None,
|
||||
)
|
||||
};
|
||||
if let (Some(note), Some(review)) = (&partial_note, structured.as_mut()) {
|
||||
if !review.summary.is_empty() {
|
||||
review.summary.push_str("\n\n");
|
||||
}
|
||||
review.summary.push_str(note);
|
||||
if !output.is_empty() {
|
||||
output.push_str("\n\n");
|
||||
}
|
||||
output.push_str(note);
|
||||
}
|
||||
// Presentation is prepared before publication. Any peer failure retains the
|
||||
// completed provider usage and cannot cause a partial post or Rust fallback.
|
||||
let rendered_report = if args.post || !args.json {
|
||||
@@ -10648,6 +10695,10 @@ async fn run_review(config: &Config, args: ReviewArgs) -> Result<()> {
|
||||
"review_anchors": review_anchors,
|
||||
"review_tools_used": review_tools_used,
|
||||
"review_repository_reads": review_tool_workspace.is_some(),
|
||||
"review_triage": review_triage,
|
||||
"review_budget": review_budget,
|
||||
"review_stop": review_stop.as_ref().map(crate::tools::review_triage::ReviewStop::as_str),
|
||||
"review_spend_usd": review_spend_usd,
|
||||
"stop_reason": review_stop_reason,
|
||||
"usage": usage,
|
||||
"review_passes": pr_plan.as_ref().map(|plan| plan.passes.len()),
|
||||
|
||||
@@ -58,6 +58,7 @@ pub mod review;
|
||||
pub mod review_core;
|
||||
pub(crate) mod review_host;
|
||||
pub mod review_hunks;
|
||||
pub mod review_triage;
|
||||
pub(crate) mod review_pr;
|
||||
pub mod rlm;
|
||||
pub mod run_tool;
|
||||
|
||||
@@ -1444,6 +1444,8 @@ impl ToolSpec for ReviewTool {
|
||||
}
|
||||
_ => super::review_core::repository_tool_workspace(&context.workspace, None),
|
||||
};
|
||||
let triage = super::review_triage::triage_review(&review_diff);
|
||||
let budget = super::review_triage::budget_for(&triage);
|
||||
let mut usage = Usage::default();
|
||||
let mut accumulator = plan.as_ref().map(PrReviewAccumulator::new);
|
||||
let mut single_output = None;
|
||||
@@ -1453,10 +1455,17 @@ impl ToolSpec for ReviewTool {
|
||||
let mut tool_calls = 0;
|
||||
let mut changed_lines = 0;
|
||||
let mut outcome = super::review_core::ReviewCoreOutcome::Completed;
|
||||
let mut stop = None;
|
||||
let mut spend_usd = None;
|
||||
for (index, prompt) in prompts.into_iter().enumerate() {
|
||||
let mut request = super::review_core::ReviewCoreRequest::new(&review_diff, prompt);
|
||||
request.workspace = tool_workspace.clone();
|
||||
request.max_output_tokens = Some(client.effective_max_output_tokens(&route.model));
|
||||
request.budget = budget.tool_budget();
|
||||
request.max_output_tokens = Some(
|
||||
client
|
||||
.effective_max_output_tokens(&route.model)
|
||||
.min(budget.max_output_tokens),
|
||||
);
|
||||
let result =
|
||||
match super::review_core::run_review_core(&client, &self.model, request).await {
|
||||
Ok(result) => result,
|
||||
@@ -1488,6 +1497,8 @@ impl ToolSpec for ReviewTool {
|
||||
tool_calls += result.tool_calls;
|
||||
changed_lines += result.coverage.changed_lines;
|
||||
outcome = result.outcome;
|
||||
stop = result.stop.clone().or(stop);
|
||||
spend_usd = result.spend_usd.or(spend_usd);
|
||||
if codewhale_models::is_incomplete_stop_reason(result.stop_reason.as_deref()) {
|
||||
return Ok(review_error_with_usage(
|
||||
&route,
|
||||
@@ -1531,6 +1542,15 @@ impl ToolSpec for ReviewTool {
|
||||
} else {
|
||||
single_output.expect("one non-PR review response")
|
||||
};
|
||||
let mut output = output;
|
||||
if let Some(note) = stop.as_ref().map(|stop| {
|
||||
super::review_triage::partial_review_note(stop, &triage, &budget)
|
||||
}) {
|
||||
if !output.summary.is_empty() {
|
||||
output.summary.push_str("\n\n");
|
||||
}
|
||||
output.summary.push_str(¬e);
|
||||
}
|
||||
let mut metadata = review_usage_metadata(&route, &usage);
|
||||
metadata["review_anchors"] = json!(anchors);
|
||||
metadata["review_tools_used"] = json!(tools_used);
|
||||
@@ -1538,6 +1558,10 @@ impl ToolSpec for ReviewTool {
|
||||
metadata["review_steps"] = json!(steps);
|
||||
metadata["review_tool_calls"] = json!(tool_calls);
|
||||
metadata["review_changed_lines"] = json!(changed_lines);
|
||||
metadata["review_triage"] = json!(triage);
|
||||
metadata["review_budget"] = json!(budget);
|
||||
metadata["review_stop"] = json!(stop.as_ref().map(super::review_triage::ReviewStop::as_str));
|
||||
metadata["review_spend_usd"] = json!(spend_usd);
|
||||
metadata["review_outcome"] = json!(match outcome {
|
||||
super::review_core::ReviewCoreOutcome::Completed => "completed",
|
||||
super::review_core::ReviewCoreOutcome::NoFinalAnswer => "no_final_answer",
|
||||
|
||||
@@ -8,6 +8,7 @@ use serde_json::Value;
|
||||
use codewhale_models::{ContentBlock, Message, MessageRequest, SystemPrompt, Tool, Usage};
|
||||
|
||||
use super::review::{ReviewIssue, ReviewOutput};
|
||||
use super::review_triage::{ReviewStop, SpendRates};
|
||||
use super::review_hunks::DiffHunks;
|
||||
use super::spec::{ToolContext, ToolSpec};
|
||||
use crate::core::model_client::ModelClient;
|
||||
@@ -111,6 +112,8 @@ pub struct ReviewCoreResult {
|
||||
pub tools_used: Vec<String>,
|
||||
pub coverage: ReviewCoverage,
|
||||
pub outcome: ReviewCoreOutcome,
|
||||
pub stop: Option<ReviewStop>,
|
||||
pub spend_usd: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
@@ -129,6 +132,8 @@ pub struct ReviewCoreRequest<'a> {
|
||||
pub reasoning_effort: Option<String>,
|
||||
pub max_claims: usize,
|
||||
pub max_nits: usize,
|
||||
pub spend_rates: Option<SpendRates>,
|
||||
pub money_cap_usd: Option<f64>,
|
||||
}
|
||||
|
||||
impl<'a> ReviewCoreRequest<'a> {
|
||||
@@ -143,6 +148,8 @@ impl<'a> ReviewCoreRequest<'a> {
|
||||
reasoning_effort: None,
|
||||
max_claims: 40,
|
||||
max_nits: 3,
|
||||
spend_rates: None,
|
||||
money_cap_usd: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -193,10 +200,7 @@ pub async fn run_review_core(
|
||||
Some(_) => review_tool_set(),
|
||||
None => Vec::new(),
|
||||
};
|
||||
let context = request
|
||||
.workspace
|
||||
.as_ref()
|
||||
.map(|workspace| ToolContext::new(workspace));
|
||||
let context = request.workspace.as_ref().map(ToolContext::new);
|
||||
let tool_definitions = protocol_tools(&tools);
|
||||
let mut messages = vec![Message {
|
||||
role: codewhale_models::Role::User,
|
||||
@@ -212,11 +216,21 @@ pub async fn run_review_core(
|
||||
let mut final_text = String::new();
|
||||
let mut stop_reason = None;
|
||||
let mut outcome = ReviewCoreOutcome::StepBudgetExhausted;
|
||||
let mut stop: Option<ReviewStop> = None;
|
||||
let mut spend_usd: Option<f64> = None;
|
||||
let max_tokens = request
|
||||
.max_output_tokens
|
||||
.unwrap_or_else(|| client.effective_max_output_tokens(model));
|
||||
|
||||
while steps < request.budget.max_steps {
|
||||
if let (Some(cap), Some(rates)) = (request.money_cap_usd, request.spend_rates) {
|
||||
let spent = rates.cost_usd(&usage);
|
||||
spend_usd = Some(spent);
|
||||
if steps > 0 && spent >= cap {
|
||||
stop = Some(ReviewStop::MoneyCap);
|
||||
break;
|
||||
}
|
||||
}
|
||||
steps += 1;
|
||||
let response = client
|
||||
.create_message(MessageRequest {
|
||||
@@ -321,6 +335,15 @@ pub async fn run_review_core(
|
||||
});
|
||||
}
|
||||
|
||||
if let (Some(cap), Some(rates)) = (request.money_cap_usd, request.spend_rates) {
|
||||
spend_usd = Some(rates.cost_usd(&usage));
|
||||
if spend_usd.is_some_and(|spent| spent >= cap) && stop.is_none() {
|
||||
stop = Some(ReviewStop::MoneyCap);
|
||||
}
|
||||
}
|
||||
if stop.is_none() && steps >= request.budget.max_steps && outcome == ReviewCoreOutcome::StepBudgetExhausted {
|
||||
stop = Some(ReviewStop::StepBudget);
|
||||
}
|
||||
let mut output = if final_text.trim().is_empty() {
|
||||
ReviewOutput {
|
||||
summary: String::new(),
|
||||
@@ -358,6 +381,8 @@ pub async fn run_review_core(
|
||||
tool_calls,
|
||||
tools_used: used,
|
||||
outcome,
|
||||
stop,
|
||||
spend_usd,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,359 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::review_core::ReviewToolBudget;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum ReviewRisk {
|
||||
Low,
|
||||
Medium,
|
||||
High,
|
||||
}
|
||||
|
||||
impl ReviewRisk {
|
||||
#[must_use]
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Low => "low",
|
||||
Self::Medium => "medium",
|
||||
Self::High => "high",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ReviewTriage {
|
||||
pub risk: ReviewRisk,
|
||||
/// Why this tier was chosen, in the reader's words. Every entry names the
|
||||
/// signal that raised it, so a receipt can be audited later.
|
||||
pub reasons: Vec<String>,
|
||||
pub changed_files: usize,
|
||||
pub changed_lines: usize,
|
||||
pub touches_tests: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ReviewBudget {
|
||||
pub risk: ReviewRisk,
|
||||
pub max_steps: u32,
|
||||
pub max_tool_calls: u32,
|
||||
pub max_output_tokens: u32,
|
||||
/// `Some(0.0)` means a cap was requested but no price is known, which makes
|
||||
/// it unenforceable; that is reported rather than treated as unlimited.
|
||||
pub money_cap_usd: Option<f64>,
|
||||
pub money_limit_supported: bool,
|
||||
}
|
||||
|
||||
const HIGH_RISK_PATH: &[(&str, &str)] = &[
|
||||
("auth", "authentication or authorization code"),
|
||||
("billing", "billing or payment code"),
|
||||
("payment", "billing or payment code"),
|
||||
("stripe", "billing or payment code"),
|
||||
("migration", "a database migration"),
|
||||
("crypto", "cryptography or key handling"),
|
||||
("secret", "secret handling"),
|
||||
("permission", "permission or policy code"),
|
||||
("policy", "permission or policy code"),
|
||||
("sandbox", "a sandbox or isolation boundary"),
|
||||
("concurren", "concurrency or locking"),
|
||||
("lock", "concurrency or locking"),
|
||||
("delete", "a deletion or destructive path"),
|
||||
("schema", "a persisted schema"),
|
||||
];
|
||||
|
||||
const MEDIUM_RISK_PATH: &[(&str, &str)] = &[
|
||||
("api", "a public or internal API surface"),
|
||||
("route", "a request route"),
|
||||
("server", "server behaviour"),
|
||||
("config", "configuration"),
|
||||
("db", "database access"),
|
||||
("store", "persistence"),
|
||||
("proto", "a protocol definition"),
|
||||
("contract", "a published contract"),
|
||||
];
|
||||
|
||||
const TEST_PATH: &[&str] = &["/test/", "/tests/", "_test.", ".test.", "/spec/", "/__tests__/"];
|
||||
|
||||
fn path_is_test(path: &str) -> bool {
|
||||
let lower = path.to_ascii_lowercase();
|
||||
TEST_PATH.iter().any(|marker| lower.contains(marker))
|
||||
}
|
||||
|
||||
/// Triage reads only the diff's paths and size. It is deliberately cheap: no
|
||||
/// model call, no repository walk, and no claim about whether the change is
|
||||
/// correct. It exists to decide how much review to buy, so it must be
|
||||
/// explainable from the diff alone.
|
||||
#[must_use]
|
||||
pub fn triage_review(diff: &str) -> ReviewTriage {
|
||||
let mut paths: Vec<String> = Vec::new();
|
||||
let mut added = 0usize;
|
||||
let mut removed = 0usize;
|
||||
for line in diff.lines() {
|
||||
let header = line
|
||||
.strip_prefix("+++ ")
|
||||
.map(|rest| rest.trim().trim_start_matches("b/"))
|
||||
.or_else(|| line.strip_prefix("--- ").map(|rest| rest.trim().trim_start_matches("a/")));
|
||||
if let Some(path) = header {
|
||||
if path != "/dev/null" && !paths.iter().any(|seen| seen == path) {
|
||||
paths.push(path.to_string());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if line.starts_with("+++") || line.starts_with("---") {
|
||||
continue;
|
||||
}
|
||||
if line.starts_with('+') {
|
||||
added += 1;
|
||||
} else if line.starts_with('-') {
|
||||
removed += 1;
|
||||
}
|
||||
}
|
||||
let changed_files = paths.len();
|
||||
let changed_lines = added + removed;
|
||||
let touches_tests = paths.iter().any(|path| path_is_test(path));
|
||||
let mut reasons = Vec::new();
|
||||
let mut high = false;
|
||||
let mut medium = false;
|
||||
for (marker, reason) in HIGH_RISK_PATH {
|
||||
if paths.iter().any(|path| path.to_ascii_lowercase().contains(marker)) {
|
||||
high = true;
|
||||
if !reasons.iter().any(|seen| seen == reason) {
|
||||
reasons.push((*reason).to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
for (marker, reason) in MEDIUM_RISK_PATH {
|
||||
if paths.iter().any(|path| path.to_ascii_lowercase().contains(marker)) {
|
||||
medium = true;
|
||||
if !reasons.iter().any(|seen| seen == reason) {
|
||||
reasons.push((*reason).to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
if changed_lines >= 400 || changed_files > 20 {
|
||||
high = true;
|
||||
reasons.push(format!(
|
||||
"a large change ({changed_files} files, {changed_lines} changed lines)"
|
||||
));
|
||||
} else if changed_lines >= 80 || changed_files > 6 {
|
||||
medium = true;
|
||||
reasons.push(format!(
|
||||
"a medium-size change ({changed_files} files, {changed_lines} changed lines)"
|
||||
));
|
||||
}
|
||||
let risk = if high {
|
||||
ReviewRisk::High
|
||||
} else if medium {
|
||||
ReviewRisk::Medium
|
||||
} else {
|
||||
ReviewRisk::Low
|
||||
};
|
||||
if reasons.is_empty() {
|
||||
reasons.push(format!(
|
||||
"a small change ({changed_files} files, {changed_lines} changed lines) with no sensitive path"
|
||||
));
|
||||
}
|
||||
if touches_tests {
|
||||
reasons.push("the change includes tests".to_string());
|
||||
}
|
||||
ReviewTriage {
|
||||
risk,
|
||||
reasons,
|
||||
changed_files,
|
||||
changed_lines,
|
||||
touches_tests,
|
||||
}
|
||||
}
|
||||
|
||||
/// The budget a tier buys. Low risk still gets a real review of the diff, just
|
||||
/// without repository tools; high risk gets dependent tracing and room to
|
||||
/// reproduce.
|
||||
#[must_use]
|
||||
pub fn budget_for(triage: &ReviewTriage) -> ReviewBudget {
|
||||
let (max_steps, max_tool_calls, max_output_tokens) = match triage.risk {
|
||||
ReviewRisk::Low => (1, 0, 4_000),
|
||||
ReviewRisk::Medium => (4, 6, 8_000),
|
||||
ReviewRisk::High => (6, 12, 16_000),
|
||||
};
|
||||
ReviewBudget {
|
||||
risk: triage.risk,
|
||||
max_steps,
|
||||
max_tool_calls,
|
||||
max_output_tokens,
|
||||
money_cap_usd: None,
|
||||
money_limit_supported: false,
|
||||
}
|
||||
}
|
||||
|
||||
impl ReviewBudget {
|
||||
#[must_use]
|
||||
pub fn tool_budget(self) -> ReviewToolBudget {
|
||||
ReviewToolBudget {
|
||||
max_steps: self.max_steps,
|
||||
max_tool_calls: self.max_tool_calls,
|
||||
max_tool_result_chars: 40_000,
|
||||
}
|
||||
}
|
||||
|
||||
/// Attach a per-review money cap. A cap without a price is not enforceable,
|
||||
/// and saying so is the point: `money_limit_supported` stays false until a
|
||||
/// rate is supplied, so a caller can never report a cap it did not apply.
|
||||
#[must_use]
|
||||
pub fn with_money_cap(mut self, cap_usd: Option<f64>, rates: Option<(f64, f64)>) -> Self {
|
||||
let Some(cap) = cap_usd.filter(|cap| *cap > 0.0 && cap.is_finite()) else {
|
||||
return self;
|
||||
};
|
||||
self.money_cap_usd = Some(cap);
|
||||
self.money_limit_supported = rates.is_some_and(|(input, output)| {
|
||||
input.is_finite() && output.is_finite() && input >= 0.0 && output >= 0.0
|
||||
});
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
|
||||
pub struct SpendRates {
|
||||
pub input_usd_per_million: f64,
|
||||
pub output_usd_per_million: f64,
|
||||
}
|
||||
|
||||
impl SpendRates {
|
||||
#[must_use]
|
||||
pub fn cost_usd(self, usage: &codewhale_models::Usage) -> f64 {
|
||||
let input = u64::from(usage.input_tokens)
|
||||
+ u64::from(usage.prompt_cache_hit_tokens.unwrap_or(0))
|
||||
+ u64::from(usage.prompt_cache_write_tokens.unwrap_or(0));
|
||||
let output = u64::from(usage.output_tokens);
|
||||
(input as f64 / 1_000_000.0) * self.input_usd_per_million
|
||||
+ (output as f64 / 1_000_000.0) * self.output_usd_per_million
|
||||
}
|
||||
}
|
||||
|
||||
/// Why a review stopped early. A stop is always reported with what was and was
|
||||
/// not covered; it is never a silent truncation.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ReviewStop {
|
||||
MoneyCap,
|
||||
StepBudget,
|
||||
ToolBudget,
|
||||
OutputLimit,
|
||||
ModelFailure,
|
||||
}
|
||||
|
||||
impl ReviewStop {
|
||||
#[must_use]
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::MoneyCap => "money_cap",
|
||||
Self::StepBudget => "step_budget",
|
||||
Self::ToolBudget => "tool_budget",
|
||||
Self::OutputLimit => "output_limit",
|
||||
Self::ModelFailure => "model_failure",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn partial_review_note(stop: &ReviewStop, triage: &ReviewTriage, budget: &ReviewBudget) -> String {
|
||||
let what = match stop {
|
||||
ReviewStop::MoneyCap => format!(
|
||||
"the per-review money cap of ${:.4} was reached",
|
||||
budget.money_cap_usd.unwrap_or(0.0)
|
||||
),
|
||||
ReviewStop::StepBudget => format!("the {} model step(s) for a {} risk change were used", budget.max_steps, triage.risk.as_str()),
|
||||
ReviewStop::ToolBudget => format!("the {} tool call(s) for a {} risk change were used", budget.max_tool_calls, triage.risk.as_str()),
|
||||
ReviewStop::OutputLimit => "the model exhausted its output allowance".to_string(),
|
||||
ReviewStop::ModelFailure => "the model call failed".to_string(),
|
||||
};
|
||||
format!(
|
||||
"This is a named partial review: {what}. Findings below were produced before the stop; files and dependents the reviewer did not reach were not reviewed, and their absence is not evidence that they are correct."
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn diff_for(paths: &[&str]) -> String {
|
||||
let mut out = String::new();
|
||||
for path in paths {
|
||||
out.push_str(&format!(
|
||||
"diff --git a/{path} b/{path}\n--- a/{path}\n+++ b/{path}\n@@ -1,1 +1,2 @@\n context\n+added\n"
|
||||
));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_docs_change_is_low_risk_and_buys_no_tools() {
|
||||
let triage = triage_review("diff --git a/docs/README.md b/docs/README.md\n--- a/docs/README.md\n+++ b/docs/README.md\n@@ -1 +1 @@\n-old\n+new\n");
|
||||
assert_eq!(triage.risk, ReviewRisk::Low);
|
||||
let budget = budget_for(&triage);
|
||||
assert_eq!(budget.max_tool_calls, 0);
|
||||
assert_eq!(budget.max_steps, 1);
|
||||
assert!(!budget.money_limit_supported);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_billing_migration_is_high_risk_and_buys_repository_tools() {
|
||||
let triage = triage_review(&diff_for(&["services/billing/stripe-migration.sql"]));
|
||||
assert_eq!(triage.risk, ReviewRisk::High);
|
||||
assert!(triage.reasons.iter().any(|reason| reason.contains("billing")));
|
||||
assert!(triage.reasons.iter().any(|reason| reason.contains("migration")));
|
||||
let budget = budget_for(&triage);
|
||||
assert!(budget.max_tool_calls >= 12);
|
||||
assert!(budget.max_steps >= 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn size_and_sensitive_paths_raise_the_tier_independently() {
|
||||
let wide = diff_for(&(0..25).map(|index| format!("src/part{index}.rs")).collect::<Vec<_>>().iter().map(String::as_str).collect::<Vec<_>>());
|
||||
assert_eq!(triage_review(&wide).risk, ReviewRisk::High);
|
||||
let tall: String = format!(
|
||||
"diff --git a/src/lib.rs b/src/lib.rs\n--- a/src/lib.rs\n+++ b/src/lib.rs\n@@ -1,1 +1,401 @@\n context\n{}",
|
||||
"+line\n".repeat(400)
|
||||
);
|
||||
assert_eq!(triage_review(&tall).risk, ReviewRisk::High);
|
||||
assert_eq!(triage_review(&diff_for(&["src/tools/shell.rs"])).risk, ReviewRisk::Low);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_money_cap_without_a_price_is_reported_as_unsupported() {
|
||||
let triage = triage_review(&diff_for(&["src/auth.rs"]));
|
||||
let without_rates = budget_for(&triage).with_money_cap(Some(0.25), None);
|
||||
assert_eq!(without_rates.money_cap_usd, Some(0.25));
|
||||
assert!(!without_rates.money_limit_supported);
|
||||
let with_rates = budget_for(&triage).with_money_cap(Some(0.25), Some((0.15, 0.60)));
|
||||
assert!(with_rates.money_limit_supported);
|
||||
let no_cap = budget_for(&triage).with_money_cap(None, Some((0.15, 0.60)));
|
||||
assert_eq!(no_cap.money_cap_usd, None);
|
||||
assert!(!no_cap.money_limit_supported);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cost_uses_every_billable_counter() {
|
||||
let usage = codewhale_models::Usage {
|
||||
input_tokens: 1_000_000,
|
||||
output_tokens: 1_000_000,
|
||||
prompt_cache_hit_tokens: Some(1_000_000),
|
||||
..Default::default()
|
||||
};
|
||||
let cost = SpendRates {
|
||||
input_usd_per_million: 0.15,
|
||||
output_usd_per_million: 0.60,
|
||||
}
|
||||
.cost_usd(&usage);
|
||||
assert!((cost - (0.30 + 0.60)).abs() < 1e-9, "got {cost}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_partial_review_names_the_stop_and_the_limit() {
|
||||
let triage = triage_review(&diff_for(&["src/auth.rs"]));
|
||||
let budget = budget_for(&triage).with_money_cap(Some(0.25), Some((0.15, 0.60)));
|
||||
let note = partial_review_note(&ReviewStop::MoneyCap, &triage, &budget);
|
||||
assert!(note.contains("named partial review"));
|
||||
assert!(note.contains("0.2500"));
|
||||
assert!(note.contains("not evidence that they are correct"));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user