mirror of
https://github.com/codewhale-hq/Codewhale.git
synced 2026-11-09 09:59:45 -10:00
fix: pin legal parallel completion observations without hiding outcomes
Repeat replay exposed swapped adjacent activity completions from independent parallel reads. Extend the existing named completion normalization to adjacent operation_activity_completed only, keyed by established span_id; preserve exact outcomes/errors, span links, count and every start/error/other-event boundary. Regression controls reject wrong outcomes/span links and prove boundaries are not crossed. Recorded real DeepSeek fixture outputs now reach loopback POST /v1/chat/completions and retain reasoning/usage/error/stray-delta facts. Correct inaccurate Plan read-only catalog fixture prose without changing captured bytes.
At ffc6b3dbe4 governed all-feature build completed; test result FAILED:10passed/2failed: actual parallel observation race plus intended DeepSeek golden drift after constructor fix. All7hardening controls passed; three other families matched. Frozen-binary corrected SSE recording1passed/0failed (11cases). Original failing diff/logs preserved under conformance-qualification. Final recompile/repeat/default/negative-control qualification pending; no release/platform/provider claims.
Signed-off-by: CodeWhale Bot <[email protected]>
This commit is contained in:
1 parent
ffc6b3dbe4
commit
f3bf08e58a
5 files changed
+84
-12
No files matched your search
@@ -14,6 +14,8 @@
|
||||
//! system prompt and tool catalog bodies are replaced by a marker because the
|
||||
//! prompt family owns those bytes; runs of adjacent `tool_call_complete`
|
||||
//! events are ordered by tool call id because parallel completions race.
|
||||
//! Adjacent `operation_activity_completed` observations are likewise ordered
|
||||
//! by their established span id. No start, error, or other event is crossed.
|
||||
|
||||
use std::collections::{BTreeMap, VecDeque};
|
||||
use std::path::Path;
|
||||
@@ -453,21 +455,68 @@ pub(super) fn normalize_events(events: &[Event], masker: &mut Masker) -> Vec<Val
|
||||
fn order_parallel_completions(lines: &mut [Value]) {
|
||||
let mut start = 0;
|
||||
while start < lines.len() {
|
||||
let (kind, key) = match lines[start]["event"].as_str() {
|
||||
Some("tool_call_complete") => ("tool_call_complete", "tool_call_id"),
|
||||
Some("operation_activity_completed") => ("operation_activity_completed", "span_id"),
|
||||
_ => {
|
||||
start += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let mut end = start;
|
||||
while end < lines.len() && lines[end]["event"] == "tool_call_complete" {
|
||||
while end < lines.len() && lines[end]["event"] == kind {
|
||||
end += 1;
|
||||
}
|
||||
if end - start > 1 {
|
||||
lines[start..end].sort_by(|left, right| {
|
||||
left["tool_call_id"]
|
||||
.as_str()
|
||||
.cmp(&right["tool_call_id"].as_str())
|
||||
});
|
||||
lines[start..end].sort_by(|left, right| left[key].as_str().cmp(&right[key].as_str()));
|
||||
}
|
||||
start = end.max(start + 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parallel_completion_projection_keeps_outcomes_spans_and_causal_boundaries() {
|
||||
let first = json!({"event": "operation_activity_completed", "span_id": "call_a#1", "outcome": "succeeded"});
|
||||
let second = json!({"event": "operation_activity_completed", "span_id": "call_b#2", "outcome": "failed"});
|
||||
let boundary = json!({"event": "error", "message": "exact error bytes"});
|
||||
let start = json!({"event": "operation_activity_started", "span_id": "call_a#1"});
|
||||
let original = vec![
|
||||
start.clone(),
|
||||
second.clone(),
|
||||
first.clone(),
|
||||
boundary.clone(),
|
||||
second.clone(),
|
||||
start.clone(),
|
||||
first.clone(),
|
||||
];
|
||||
let expected = vec![
|
||||
start.clone(),
|
||||
first.clone(),
|
||||
second.clone(),
|
||||
boundary,
|
||||
second,
|
||||
start,
|
||||
first,
|
||||
];
|
||||
let mut normalized = original.clone();
|
||||
order_parallel_completions(&mut normalized);
|
||||
assert_eq!(normalized, expected);
|
||||
assert_eq!(normalized.len(), original.len());
|
||||
|
||||
let mut wrong_outcome = original.clone();
|
||||
wrong_outcome[1]["outcome"] = json!("succeeded");
|
||||
order_parallel_completions(&mut wrong_outcome);
|
||||
assert_ne!(wrong_outcome, expected, "wrong outcome was normalized away");
|
||||
|
||||
let mut wrong_span = original;
|
||||
wrong_span[1]["span_id"] = json!("call_other#2");
|
||||
order_parallel_completions(&mut wrong_span);
|
||||
assert_ne!(
|
||||
wrong_span, expected,
|
||||
"wrong span relationship was normalized away"
|
||||
);
|
||||
}
|
||||
|
||||
fn check_invariants(case: &Value, workspace: &Path, provider: &ScriptedProvider) -> Vec<String> {
|
||||
let mut violations = Vec::new();
|
||||
for invariant in case["invariants"].as_array().into_iter().flatten() {
|
||||
|
||||
+8
-1
@@ -10,7 +10,8 @@ written in the abstract.
|
||||
Everything here is language-neutral: JSON, JSONL, raw SSE bytes, and POSIX
|
||||
shell. No fixture was captured from a real provider or contains a secret.
|
||||
|
||||
The Rust runner is `crates/tui/src/conformance/` (one test per family):
|
||||
The Rust runner is `crates/tui/src/conformance/` (one golden test per family,
|
||||
plus hardening controls):
|
||||
|
||||
```sh
|
||||
cargo test -p codewhale-tui --lib -- conformance::
|
||||
@@ -78,6 +79,9 @@ Each line is `codewhale_protocol::EventMsg` as serialized (`"event"` tag), with:
|
||||
- `tool_catalog` and `system_prompt` bodies → `"<pinned by the prompt family>"`;
|
||||
- runs of adjacent `tool_call_complete` events ordered by `tool_call_id`
|
||||
(parallel completions race);
|
||||
- runs of adjacent `operation_activity_completed` observations ordered by
|
||||
the established `span_id` for the same reason; outcomes, span relationships,
|
||||
and event counts stay exact, and no start/error/other event is crossed;
|
||||
- keys sorted.
|
||||
|
||||
The last line is `{"harness_summary": {model_requests, non_streaming_requests,
|
||||
@@ -179,6 +183,9 @@ background observers, cancellation and process-tree teardown are not covered.
|
||||
The turns do not qualify persistence/resume, nested multi-tool cardinality,
|
||||
non-draining client backpressure, or every client projection. Slash-command,
|
||||
script-tool, and migrated adapter calls need their own fixtures before flipping.
|
||||
The Plan prefix records its advertised catalog and frozen prompt bytes; it
|
||||
does not prove read-only admission. At this baseline it still advertises
|
||||
`bash`, `edit` and `write`; full-turn Plan refusal needs separate evidence.
|
||||
|
||||
The prefix runner names unavailable host-probed tools explicitly. A skipped
|
||||
definition (`code_execution`, `image_ocr`, `js_execution`, `pandoc_convert`) is
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"description": "Plan mode: the read-only tool surface and the Plan prompt layer.",
|
||||
"description": "Plan mode: pins the advertised catalog and frozen prefix. The current catalog still advertises bash/edit/write; read-only admission and appended Plan instructions are not proved by a prefix fixture.",
|
||||
"model": "deepseek-v4-pro",
|
||||
"mode": "plan",
|
||||
"approval_mode": "suggest",
|
||||
|
||||
+9
-2
@@ -1,2 +1,9 @@
|
||||
{"request":[]}
|
||||
{"type":"open_failure","detail":"DeepSeek model \"deepseek-v4-pro\" uses ChatCompletions, but this client is bound to Responses; resolve a new model route before sending"}
|
||||
{"request":[{"method":"POST","path":"/v1/chat/completions"}]}
|
||||
{"type":"message_start","message":{"id":"","type":"message","role":"assistant","content":[],"model":"deepseek-v4-pro","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}}
|
||||
{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}
|
||||
{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Partial"}}
|
||||
{"type":"error","error":{"message":"Model not exist.","type":"invalid_request_error","code":"invalid_request_error"}}
|
||||
{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" after the error"}}
|
||||
{"type":"content_block_stop","index":0}
|
||||
{"type":"message_delta","delta":{"stop_reason":"stop","stop_sequence":null}}
|
||||
{"type":"message_stop"}
|
||||
+11
-2
@@ -1,2 +1,11 @@
|
||||
{"request":[]}
|
||||
{"type":"open_failure","detail":"DeepSeek model \"deepseek-v4-pro\" uses ChatCompletions, but this client is bound to Responses; resolve a new model route before sending"}
|
||||
{"request":[{"method":"POST","path":"/v1/chat/completions"}]}
|
||||
{"type":"message_start","message":{"id":"","type":"message","role":"assistant","content":[],"model":"deepseek-v4-pro","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":0,"output_tokens":0}}}
|
||||
{"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":""}}
|
||||
{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"Compare the two numbers. "}}
|
||||
{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"9.11 < 9.9."}}
|
||||
{"type":"content_block_stop","index":0}
|
||||
{"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}
|
||||
{"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"9.9 is larger."}}
|
||||
{"type":"content_block_stop","index":1}
|
||||
{"type":"message_delta","delta":{"stop_reason":"stop","stop_sequence":null},"usage":{"input_tokens":20,"output_tokens":30,"prompt_cache_hit_tokens":12,"prompt_cache_miss_tokens":8,"reasoning_tokens":22}}
|
||||
{"type":"message_stop"}
|
||||
Reference in new issue
Block a user