mirror of
https://github.com/codewhale-hq/Codewhale.git
synced 2026-11-09 17:10:44 -10:00
fix(tools): bound JSONPath projection work and serialized results [skip ci]
Refs #6554; CW-6943. Reuse the existing shared parser and CountingWriter before result clones. Bound 32 raw fields, 4096 path bytes, 128 steps, 1024 intermediate matches and 1 MiB aggregate serialized projected fields including escaped keys and delimiters. Verification on unchanged 2312 recorded inputs: 27 existing Engine handle/fetch cases; 22 actual ordinary-Engine ToolSpec observations; 2 ordinary CLI journeys using real RLM FINAL handles (exact success and 1025-match refusal); 18 unchanged included-source controls reused. Ordinary CLI build and focused rustfmt pass. Fetch success controls seed the public production cache; no origin HTTP, installed-package or final-release claim. Existing Engine lib-test compact-unwind warning retained. Receipts: CW/artifacts/engine-0103-execution-20261102/jsonpath/RECEIPT.json.
This commit is contained in:
1 parent
19020857fa
commit
b8c5417927
4 files changed
+91
-10
No files matched your search
@@ -160,6 +160,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Fixed
|
||||
|
||||
- JSONPath projections bound path length, traversal steps and intermediate matches before cloning results. Direct fetches share the same parser and enforce one aggregate serialized-field budget; oversized selections refuse with narrower-selection guidance.
|
||||
|
||||
- Long tool output and diffs use bounded display windows and a byte-limited row cache. Retained details load verified byte ranges and label what search and copy include; explicit source copy keeps stored text with control-sequence cleanup. Oversized Unicode is shown safely without changing question answers; long answers retain their exact bytes in the existing owned output artifact before bounded model-context projection. New pager notices are translated across all 15 complete language packs.
|
||||
- Managed Fleet validates overlapping write roots with a component-sorted scan and deduplicates roots once. Large valid request bodies no longer spend quadratic CPU before admission; existing write-scope refusals and execution limits remain in force (#6554).
|
||||
- MCP transports, connection pools and child Engines retain the owning host credential context for placeholders, headers, bearer tokens and OAuth setup. Missing injected values never fall back to process secrets. Reviewed plugins keep their captured environment authority, whose Debug output now masks values. Durable OAuth token custody and full subprocess isolation remain separate work (#5637).
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
//! (`format = "markdown"`); pass `format = "raw"` to keep the bytes intact
|
||||
//! when the model wants to do its own parsing.
|
||||
|
||||
use super::handle::query_jsonpath;
|
||||
use super::handle::{JSONPATH_MAX_BYTES, JSONPATH_MAX_PATH_BYTES, query_jsonpath_bounded};
|
||||
use super::pdf::PdfTextCommand;
|
||||
use super::spec::{
|
||||
ApprovalRequirement, ToolCapability, ToolContext, ToolError, ToolResult, ToolSpec, optional_u64,
|
||||
@@ -27,6 +27,7 @@ use std::collections::BTreeMap;
|
||||
use std::time::Duration;
|
||||
|
||||
const FETCH_ACCEPT: &str = "text/html,text/markdown,text/plain,application/json,application/pdf,image/*,audio/*,video/*,*/*;q=0.5";
|
||||
const MAX_JSON_FIELDS: usize = 32;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum Format {
|
||||
@@ -132,8 +133,9 @@ impl ToolSpec for FetchUrlTool {
|
||||
},
|
||||
"fields": {
|
||||
"type": "array",
|
||||
"items": { "type": "string" },
|
||||
"description": "Optional JSONPath projections for JSON responses. Supports $, .field, [index], [*], and ['field']; returns matches under `fields`."
|
||||
"maxItems": MAX_JSON_FIELDS,
|
||||
"items": { "type": "string", "maxLength": JSONPATH_MAX_PATH_BYTES },
|
||||
"description": "Optional JSONPath projections for JSON responses. Supports $, .field, [index], [*], and ['field']; returns matches under `fields`. At most 32 paths of 4096 bytes each, 128 steps and 1024 intermediate matches per path, and 1 MiB of aggregate projected fields; narrow the selection on refusal."
|
||||
}
|
||||
},
|
||||
"required": ["url"]
|
||||
@@ -515,6 +517,11 @@ fn parse_fields(input: &Value) -> Result<Vec<String>, ToolError> {
|
||||
let Some(values) = values.as_array() else {
|
||||
return Err(ToolError::invalid_input("`fields` must be an array"));
|
||||
};
|
||||
if values.len() > MAX_JSON_FIELDS {
|
||||
return Err(ToolError::invalid_input(
|
||||
"`fields` accepts at most 32 paths",
|
||||
));
|
||||
}
|
||||
let mut fields = Vec::new();
|
||||
for value in values {
|
||||
let Some(field) = value.as_str() else {
|
||||
@@ -523,7 +530,12 @@ fn parse_fields(input: &Value) -> Result<Vec<String>, ToolError> {
|
||||
));
|
||||
};
|
||||
let field = field.trim();
|
||||
if !field.is_empty() {
|
||||
if field.len() > JSONPATH_MAX_PATH_BYTES {
|
||||
return Err(ToolError::invalid_input(
|
||||
"`fields` JSONPath exceeds 4096 bytes",
|
||||
));
|
||||
}
|
||||
if !field.is_empty() && !fields.iter().any(|existing| existing == field) {
|
||||
fields.push(field.to_string());
|
||||
}
|
||||
}
|
||||
@@ -547,10 +559,30 @@ fn project_json_fields(
|
||||
ToolError::execution_failed(format!("response body is not valid JSON for `fields`: {e}"))
|
||||
})?;
|
||||
let mut out = BTreeMap::new();
|
||||
let mut projected_bytes = 2usize;
|
||||
for field in fields {
|
||||
let matches = query_jsonpath(&body_json, field).map_err(|e| {
|
||||
ToolError::invalid_input(format!("invalid JSONPath `{field}` in `fields`: {e}"))
|
||||
if out.contains_key(field) {
|
||||
continue;
|
||||
}
|
||||
let mut counter = codewhale_util::utils::CountingWriter::new();
|
||||
serde_json::to_writer(&mut counter, field).map_err(|error| {
|
||||
ToolError::execution_failed(format!("JSON projection measurement failed: {error}"))
|
||||
})?;
|
||||
projected_bytes = projected_bytes
|
||||
.saturating_add(counter.count())
|
||||
.saturating_add(1 + usize::from(!out.is_empty()));
|
||||
let remaining = JSONPATH_MAX_BYTES
|
||||
.checked_sub(projected_bytes)
|
||||
.ok_or_else(|| {
|
||||
ToolError::invalid_input(
|
||||
"`fields` projection exceeds 1 MiB; request fewer or narrower fields",
|
||||
)
|
||||
})?;
|
||||
let (matches, bytes) =
|
||||
query_jsonpath_bounded(&body_json, field, remaining).map_err(|e| {
|
||||
ToolError::invalid_input(format!("invalid JSONPath `{field}` in `fields`: {e}"))
|
||||
})?;
|
||||
projected_bytes = projected_bytes.saturating_add(bytes);
|
||||
out.insert(field.clone(), matches);
|
||||
}
|
||||
Ok(Some(out))
|
||||
|
||||
@@ -31,6 +31,10 @@ const HANDLE_STORE_MAX_BYTES: usize = 64 * 1024 * 1024;
|
||||
const DEFAULT_MAX_CHARS: usize = 12_000;
|
||||
const HARD_MAX_CHARS: usize = 50_000;
|
||||
const REPR_PREVIEW_CHARS: usize = 160;
|
||||
pub(super) const JSONPATH_MAX_PATH_BYTES: usize = 4096;
|
||||
pub(super) const JSONPATH_MAX_BYTES: usize = 1024 * 1024;
|
||||
const JSONPATH_MAX_MATCHES: usize = 1024;
|
||||
const JSONPATH_MAX_STEPS: usize = 128;
|
||||
|
||||
pub type SharedHandleStore = Arc<Mutex<HandleStore>>;
|
||||
|
||||
@@ -311,7 +315,8 @@ impl ToolSpec for HandleReadTool {
|
||||
},
|
||||
"jsonpath": {
|
||||
"type": "string",
|
||||
"description": "Small JSONPath subset: $, .field, [index], [*], and ['field']."
|
||||
"maxLength": JSONPATH_MAX_PATH_BYTES,
|
||||
"description": "Small JSONPath subset: $, .field, [index], [*], and ['field']. Refuses paths over 4096 bytes, over 128 steps, over 1024 intermediate matches, or results over 1 MiB; narrow the path on refusal."
|
||||
},
|
||||
"introspect": {
|
||||
"type": "boolean",
|
||||
@@ -490,6 +495,11 @@ fn parse_projection(input: &Value) -> Result<Projection, ToolError> {
|
||||
"handle_read: jsonpath must not be empty",
|
||||
));
|
||||
}
|
||||
if path.len() > JSONPATH_MAX_PATH_BYTES {
|
||||
return Err(ToolError::invalid_input(
|
||||
"handle_read: JSONPath exceeds 4096 bytes",
|
||||
));
|
||||
}
|
||||
return Ok(Projection::JsonPath(path.to_string()));
|
||||
}
|
||||
if let Some(slice) = input.get("slice") {
|
||||
@@ -735,13 +745,29 @@ fn record_text(record: &HandleRecord) -> std::borrow::Cow<'_, str> {
|
||||
}
|
||||
|
||||
pub fn query_jsonpath(root: &Value, path: &str) -> Result<Vec<Value>, String> {
|
||||
query_jsonpath_bounded(root, path, JSONPATH_MAX_BYTES).map(|(values, _)| values)
|
||||
}
|
||||
|
||||
pub(super) fn query_jsonpath_bounded(
|
||||
root: &Value,
|
||||
path: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<(Vec<Value>, usize), String> {
|
||||
if path.len() > JSONPATH_MAX_PATH_BYTES {
|
||||
return Err("jsonpath exceeds 4096 bytes".to_string());
|
||||
}
|
||||
if !path.starts_with('$') {
|
||||
return Err("jsonpath must start with `$`".to_string());
|
||||
}
|
||||
let mut idx = 1usize;
|
||||
let bytes = path.as_bytes();
|
||||
let mut current = vec![root];
|
||||
let mut steps = 0usize;
|
||||
while idx < bytes.len() {
|
||||
steps += 1;
|
||||
if steps > JSONPATH_MAX_STEPS {
|
||||
return Err("jsonpath exceeds 128 steps; use a narrower projection".to_string());
|
||||
}
|
||||
match bytes[idx] {
|
||||
b'.' => {
|
||||
idx += 1;
|
||||
@@ -780,16 +806,35 @@ pub fn query_jsonpath(root: &Value, path: &str) -> Result<Vec<Value>, String> {
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(current.into_iter().cloned().collect())
|
||||
let mut counter = codewhale_util::utils::CountingWriter::new();
|
||||
serde_json::to_writer(&mut counter, ¤t)
|
||||
.map_err(|error| format!("jsonpath result measurement failed: {error}"))?;
|
||||
let max_bytes = max_bytes.min(JSONPATH_MAX_BYTES);
|
||||
if counter.count() > max_bytes {
|
||||
return Err(format!(
|
||||
"jsonpath result exceeds the {max_bytes}-byte projection budget; use a narrower projection"
|
||||
));
|
||||
}
|
||||
Ok((current.into_iter().cloned().collect(), counter.count()))
|
||||
}
|
||||
|
||||
fn apply_bracket_token<'a>(values: Vec<&'a Value>, token: &str) -> Result<Vec<&'a Value>, String> {
|
||||
if token == "*" {
|
||||
let mut out = Vec::new();
|
||||
let mut push = |item: &'a Value| -> Result<(), String> {
|
||||
if out.len() == JSONPATH_MAX_MATCHES {
|
||||
return Err(
|
||||
"jsonpath exceeds 1024 intermediate matches; use a narrower projection"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
out.push(item);
|
||||
Ok(())
|
||||
};
|
||||
for value in values {
|
||||
match value {
|
||||
Value::Array(items) => out.extend(items),
|
||||
Value::Object(map) => out.extend(map.values()),
|
||||
Value::Array(items) => items.iter().try_for_each(&mut push)?,
|
||||
Value::Object(map) => map.values().try_for_each(&mut push)?,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,6 +160,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Fixed
|
||||
|
||||
- JSONPath projections bound path length, traversal steps and intermediate matches before cloning results. Direct fetches share the same parser and enforce one aggregate serialized-field budget; oversized selections refuse with narrower-selection guidance.
|
||||
|
||||
- Long tool output and diffs use bounded display windows and a byte-limited row cache. Retained details load verified byte ranges and label what search and copy include; explicit source copy keeps stored text with control-sequence cleanup. Oversized Unicode is shown safely without changing question answers; long answers retain their exact bytes in the existing owned output artifact before bounded model-context projection. New pager notices are translated across all 15 complete language packs.
|
||||
|
||||
- Managed Fleet validates overlapping write roots with a component-sorted scan and deduplicates roots once. Large valid request bodies no longer spend quadratic CPU before admission; existing write-scope refusals and execution limits remain in force (#6554).
|
||||
|
||||
Reference in new issue
Block a user